From 8857302347afacff6673fcabadde713126c012ca Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 4 Jun 2014 14:41:18 -0700 Subject: [PATCH 001/631] check-starttls and process-google-starttls-domains.py --- check-starttls.py | 94 ++++++++++++++++++++++++++++++ process-google-starttls-domains.py | 18 ++++++ 2 files changed, 112 insertions(+) create mode 100755 check-starttls.py create mode 100755 process-google-starttls-domains.py diff --git a/check-starttls.py b/check-starttls.py new file mode 100755 index 000000000..15bc7ce64 --- /dev/null +++ b/check-starttls.py @@ -0,0 +1,94 @@ +#!/usr/bin/python +import sys +import os +import errno +import smtplib +import socket +import subprocess +import re + +import dns.resolver +from M2Crypto import X509 + +def mkdirp(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def extract_names(pem): + leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) + + """Extracts a list of DNS names associated with the leaf cert.""" + subj = leaf.get_subject() + # Certs have a "subject" identified by a Distingushed Name (DN). + # Host certs should also have a Common Name (CN) with a DNS name. + common_names = subj.get_entries_by_nid(subj.nid['CN']) + common_names = [name.get_data().as_text() for name in common_names] + try: + # The SAN extension allows one cert to cover multiple domains + # and permits DNS wildcards. + # http://www.digicert.com/subject-alternative-name.htm + # The field is a comma delimited list, e.g.: + # >>> twitter_cert.get_ext('subjectAltName').get_value() + # 'DNS:www.twitter.com, DNS:twitter.com' + alt_names = leaf.get_ext('subjectAltName').get_value() + alt_names = alt_names.split(',') + alt_names = [name.partition(':') for name in alt_names] + alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] + except: + alt_names = [] + return set(common_names + alt_names) + +def tls_connect(mx_host, mail_domain): + # smtplib doesn't let us access certificate information, + # so shell out to openssl. + output = subprocess.check_output( + """openssl s_client \ + -CApath /usr/share/ca-certificates/mozilla/ \ + -starttls smtp -connect %s:25 -showcerts 0: + print "iiii ", len(cert) + print extract_names(cert[0]) + #lines = output.split("\n") + #for i in range(0, len(lines)): + #line = lines[i] + #if re.search("Subject:.* CN=(.*)", line): + #m = re.search("Subject:.* CN=(.*)", line) + #print "CN=", m.group(1) + #elif re.search("Subject Alternative Name:", line): + #dns = re.findall("DNS:([^,]*),", lines[i+1]) + #for d in dns: + #print d + +# try: +# smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) +# smtpserver.ehlo() +# smtpserver.starttls() +# print "Success: %s" % mx_host +# except socket.error as e: +# print "Connection to %s failed: %s" % (mx_host, e.strerror) +# pass + +def check(mail_domain): + mkdirp(mail_domain) + answers = dns.resolver.query(mail_domain, 'MX') + for rdata in answers: + mx_host = str(rdata.exchange) + print 'Host', rdata.exchange, 'has preference', rdata.preference + tls_connect(mx_host, mail_domain) + +if len(sys.argv) == 1: + print("Please pass at least one mail domain as an argument") + +for domain in sys.argv[1:]: + check(domain) diff --git a/process-google-starttls-domains.py b/process-google-starttls-domains.py new file mode 100755 index 000000000..0a0201875 --- /dev/null +++ b/process-google-starttls-domains.py @@ -0,0 +1,18 @@ +#!/usr/bin/python +import csv +import codecs +import sys +from collections import defaultdict + +csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') +d = defaultdict(set) +for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in csvreader: + if direction == "outbound": + try: + d[address_suffix].add(float(fraction_encrypted)) + except ValueError: + pass + +for address_suffix, fraction_encrypted in d.iteritems(): + if min(fraction_encrypted) >= 0.50: + print min(fraction_encrypted), address_suffix From f0b9ef2716f356dbf033f4aec71c365103429a8d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 10:43:01 -0700 Subject: [PATCH 002/631] Add a Vagrantfile and the list of golden domains. --- Vagrantfile | 27 +++++++++++++++++++++++++++ golden-domains.txt | 22 ++++++++++++++++++++++ vagrant-bootstrap.sh | 10 ++++++++++ 3 files changed, 59 insertions(+) create mode 100644 Vagrantfile create mode 100644 golden-domains.txt create mode 100755 vagrant-bootstrap.sh diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..e485988f1 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,27 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "hashicorp/precise32" + + config.vm.define "sender" do |sender| + sender.vm.network "private_network", ip: "192.168.33.5" + end + config.vm.define "valid" do |valid| + valid.vm.network "private_network", ip: "192.168.33.7" + end + config.vm.provision :shell, path: "vagrant-bootstrap.sh" + + config.vm.provider "virtualbox" do |vb| + # vb.gui = true + vb.customize ["modifyvm", :id, "--memory", "256"] + end +end diff --git a/golden-domains.txt b/golden-domains.txt new file mode 100644 index 000000000..9a0c95948 --- /dev/null +++ b/golden-domains.txt @@ -0,0 +1,22 @@ +ymail.com +yandex.ru +yahoo.co.uk +wp.pl +web.de +vtext.com +ukr.net +t-online.de +sompo-japan.co.jp +sbcglobal.net +salesforce.com +rogers.com +rocketmail.com +rambler.ru +marktplaats.nl +interia.pl +gmx.net +gmx.de +facebook.com +craigslist.org +bigpond.com +aol.com diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh new file mode 100755 index 000000000..ac9837198 --- /dev/null +++ b/vagrant-bootstrap.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export DEBIAN_FRONTEND=noninteractive + +apt-get update -q +apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postfix +cat > /etc/hosts << Date: Thu, 5 Jun 2014 11:05:08 -0700 Subject: [PATCH 003/631] Add design doc --- README.md | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..4728ea352 --- /dev/null +++ b/README.md @@ -0,0 +1,188 @@ +# STARTTLS Everywhere + +## Authors + +Jacob Hoffman-Andrews , Peter Eckersley + +## Background + +Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. + +To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. + +Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. + +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. + +STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. + +## Goals + +* Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. +* Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. +* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring + +## Non-goals + +* Prevent fully-targeted exploits of vulnerabilities on endpoints or on mail hosts. +* Refuse delivery on the recipient side if sender does not negotiate TLS (this may be a future project). +* Develop a fully-decentralized solution. +* Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. + +## Threat model + +Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. + +## Detailed design + +Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: + + (a) a configuration file format to convey STARTTLS support for recipient domains, + + (b) Python code (config-generator) to transform (a) into configuration files for popular MTAs., and + + (c) a method to create and securely distribute files of type (a) for major email domains that that agree to be included, plus any other domains that proactively request to be included. + +## File Format + +The basic file format will be JSON with comments ([](http://blog.getify.com/json-comments/))http://blog.getify.com/json-comments/). Example: + +* { +* // Canonical URL [](https://eff.org/starttls-everywhere/config)[https://eff.org/s](https://eff.org/that-email-thing)tarttls-everywhere/config -- redirects to latest version +* "timestamp": 1401093333 +* "author": "Electronic Frontier Foundation [](https://eff.org)https://eff.org", +* "expires": 1401414363, // epoch seconds +* "nexthop-domains": { +* "gmail.com": { +* "accept-mx-domains": ["google.com", "gmail.com"] +* } +* "yahoo.com": { +* "accept-mx-domains": ["yahoodns.net"] +* } +* "eff.org": { +* "accept-mx-domains": ["eff.org"] +* } +* } +* "mx-domains": { +* "eff.org": { +* "require-tls": true, +* "min-tls-version": "TLSv1.1", +* "enforce-mode": "enforce" +* } +* "google.com": { +* "require-valid-certificate": true, +* "min-tls-version": "TLSv1.1", +* "accept-pinset": "google", +* "enforce-mode": "log-only", +* // error-notification domains * +* "error-notification": "[](https://googlemail.com/post/reports/here)https://g[o](https://g)[o](https://go)[g](https://goo)[l](https://goog)[e](https://googl)[m](https://google)[a](https://googlem)[il](https://googlema)[.co](https://googlemail)[m](https://googlemail.co)[/](https://googlemail.com)post/[r](https://googlemail.com/xhr/)[e](https://googlemail.com/xhr/r)[por](https://googlemail.com/xhr/re)[t](https://googlemail.com/xhr/repor)[s](https://googlemail.com/xhr/report)[/](https://googlemail.com/xhr/reports/go)[he](https://googlemail.com/xhr/reports/go/)[r](https://googlemail.com/xhr/reports/go/he)[e](https://googlemail.com/xhr/reports/go/her)["](https://googlemail.com/xhr/reports/go/here) +* }, +* "yahoodns.net": { +* "require-valid-certificate": true, +* } +* } +* // Similar to +* // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json +* "pinsets": [ +* { +* "name": "google", +* "static_spki_hashes": [ +* "GoogleBackup2048", +* "GoogleG2" +* ] +* } +* ], +* "spki_hashes": { + +* Are base64 encoded hashes already widely used/defined? If not, I'd lean towards hex here in this JSON file, since we hopefully have gzip to reduce the encoding overhead anyway, and this file should optimize for admin ease of use. +* These are what's used for certificate pinning in Chrome. Most likely we'll have people use the same tooling to generate these SPKI hashes for us, so I think using the same format and encoding makes sense. It's also easy on admins: "We'll just use the same set of pins we use for HTTPS." +* The other thing I forgot to mention: There isn't, AFAIK, a good way to get the SPKI hash other than using the Chrome tool. It's not one of the fields output by openssl x509, for example. + +* "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", +* "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" +* } +* } + +A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. + +The _timestamp_ field is an integer number of epoch seconds. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. + +There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. + +Option 1: Plain JSON distributed with a signature using gpg --clearsign. Config-generator should validate the signature against a known GPG public key before extracting. The public key is part of the permanent system configuration, like the fetch URL. + +Option 2: Git is a revision control system built on top of an authenticated, history-preserving file system. Let's use it as an authenticated, history preserving file system: valid versions of recipient policy files may be fetched and verified via signed git tags. [Here's an example shell recipe to do this.](https://gist.github.com/jsha/6230206e89759cc6e00d) + +Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. + +**nexthop-domains** + +The _nexthop-domains _field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _nexthop-domains _section. + +Currently the only property defined for _nexthop-domains _is _accept-mx-domains_, a list. If an MX lookup for a listed nexthop domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. + +The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. + +**mx-domains** + +The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain _can be a subdomain of any other _mx-domain _in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. + +Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. + +If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ + +_Require-valid-certificate _defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: + +1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. +2. The certificate is unexpired. +3. There is a valid chain from the certificate to a root certificate included in [Mozilla's trust store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/included/) (available as [Debian package ca-certificates](https://packages.debian.org/sid/ca-certificates)). + +The _accept-pinset_ field references an entry in the pinsets list, which has the same format and semantics as [Chrome's pinning list](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json). Most _mx-domain_s should specify a pinset that describes trust roots rather than leaf certificates, but both are possible. Pinning will only be added at the request of mail operators because it requires operators be careful when issuing new leaf certificates. + +## Pinning and hostname verification + +Like Chrome (and soon Firefox) we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. + +The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. + +We do not attempt to solve the self-signed certificate problem. For mail hosts with self-signed certificates, we can require TLS but will not require validation of the certificates. Such hosts should be encouraged to upgrade to a CA-signed certificate that can be validated by senders. + +## Creating configuration + +We have three options for creating the configuration file: + +1. Ask mail operators to submit policies for their domains which we incorporate. +2. Manually curate a set of policies for the top N mail domains. +3. Programmatically create a set of policies by connecting to the top N mail domains. + +For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. + +For option (3) we'd be likely to pull in bad policies that could result in failed delivery. + +We'll initially launch a demo using option (2), do some initial deployments to prove viability and delivery rate impact, and then start reaching out to operators to do option (1). + +## Distribution + +The configuration file will be provided at a long-term maintained URL. It will be signed using a key held offline on an airgapped machine or smartcard. + +Since recipient mail servers may abruptly stop supporting TLS, we will request that mail operators set up auto-updating of the configuration file, with signature verification. This allows us to minimize the delivery impact of such events. However, config-generator should not auto-update its own code, since that would amount to auto-deployment of third party code, which some operators may not wish to do. + +We may choose to implement a form of immutable log along the lines of certificate transparency. This would be appealing if we chose to use this mechanism to distribute expected leaf keys as a primary authentication mechanism, but as described in "Pinning and hostname verification," that's not a viable option. Instead we will rely on the CA ecosystem to do primary authentication, so an immutable log for this system is probably overkill, engineering-wise. + +## Python code + +Config-generator should parse input JSON and produce output configs for various mail servers. It should not be possible for any input JSON to cause arbitrary code execution or even any MTA config directives beyond the ones that specifically impact the decision to deliver or bounce based on TLS support. For instance, it must not be possible for config-generator to output a directive to forward mail from one domain to another. Config-generator will have the option to directly pull the latest config from a URL, or from a file on local disk distributed regularly from another system that has outside network access. + +Config-generator will be manually updated by mail operators. + +## Testing + +We will create a reproducible test configuration that can be run locally and exercises each of the major cases: Enforce mode vs log mode; Enforced TLS negotiation, enforced MX hostname match, and enforced valid certificates. + +Additionally, for ongoing monitoring of third-party deployments, we will create a canary mail domain that intentionally fails one of the tests but is included in the configuration file. For instance, starttls-canary.org would be listed in the configuration as requiring STARTTLS, but would not actually offer STARTTLS. Each time a mail operator commits to configuring STARTTLS Everywhere, we would request an account on their email domain from which to send automated daily email to starttls-canary.org. We should expect bounces. If such mail is successfully delivered to starttls-canary.org, that would indicate a configuration failure on the sending host, and we would manually notify the operator. + +## Failure reporting + +For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job and send notices only when STARTTLS Everywhere-related failures exceed 0.1% for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. + +For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. From aa417eec15ab6e9442e33cf687238fbc48041d1c Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:10:07 -0700 Subject: [PATCH 004/631] Formatting issue in design doc --- README.md | 106 ++++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 4728ea352..4686d5baf 100644 --- a/README.md +++ b/README.md @@ -45,63 +45,59 @@ Senders need to know which target hosts are known to support STARTTLS, and how t ## File Format -The basic file format will be JSON with comments ([](http://blog.getify.com/json-comments/))http://blog.getify.com/json-comments/). Example: +The basic file format will be JSON with comments (http://blog.getify.com/json-comments/). Example: -* { -* // Canonical URL [](https://eff.org/starttls-everywhere/config)[https://eff.org/s](https://eff.org/that-email-thing)tarttls-everywhere/config -- redirects to latest version -* "timestamp": 1401093333 -* "author": "Electronic Frontier Foundation [](https://eff.org)https://eff.org", -* "expires": 1401414363, // epoch seconds -* "nexthop-domains": { -* "gmail.com": { -* "accept-mx-domains": ["google.com", "gmail.com"] -* } -* "yahoo.com": { -* "accept-mx-domains": ["yahoodns.net"] -* } -* "eff.org": { -* "accept-mx-domains": ["eff.org"] -* } -* } -* "mx-domains": { -* "eff.org": { -* "require-tls": true, -* "min-tls-version": "TLSv1.1", -* "enforce-mode": "enforce" -* } -* "google.com": { -* "require-valid-certificate": true, -* "min-tls-version": "TLSv1.1", -* "accept-pinset": "google", -* "enforce-mode": "log-only", -* // error-notification domains * -* "error-notification": "[](https://googlemail.com/post/reports/here)https://g[o](https://g)[o](https://go)[g](https://goo)[l](https://goog)[e](https://googl)[m](https://google)[a](https://googlem)[il](https://googlema)[.co](https://googlemail)[m](https://googlemail.co)[/](https://googlemail.com)post/[r](https://googlemail.com/xhr/)[e](https://googlemail.com/xhr/r)[por](https://googlemail.com/xhr/re)[t](https://googlemail.com/xhr/repor)[s](https://googlemail.com/xhr/report)[/](https://googlemail.com/xhr/reports/go)[he](https://googlemail.com/xhr/reports/go/)[r](https://googlemail.com/xhr/reports/go/he)[e](https://googlemail.com/xhr/reports/go/her)["](https://googlemail.com/xhr/reports/go/here) -* }, -* "yahoodns.net": { -* "require-valid-certificate": true, -* } -* } -* // Similar to -* // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json -* "pinsets": [ -* { -* "name": "google", -* "static_spki_hashes": [ -* "GoogleBackup2048", -* "GoogleG2" -* ] -* } -* ], -* "spki_hashes": { + { + // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version + "timestamp": 1401093333 + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1401414363, // epoch seconds + "nexthop-domains": { + "gmail.com": { + "accept-mx-domains": ["google.com", "gmail.com"] + } + "yahoo.com": { + "accept-mx-domains": ["yahoodns.net"] + } + "eff.org": { + "accept-mx-domains": ["eff.org"] + } + } + "mx-domains": { + "eff.org": { + "require-tls": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "enforce" + } + "google.com": { + "require-valid-certificate": true, + "min-tls-version": "TLSv1.1", + "accept-pinset": "google", + "enforce-mode": "log-only", + // error-notification domains * + "error-notification": "https://google.com/post/reports/here" + }, + "yahoodns.net": { + "require-valid-certificate": true, + } + } + // Similar to + // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json + "pinsets": [ + { + "name": "google", + "static_spki_hashes": [ + "GoogleBackup2048", + "GoogleG2" + ] + } + ], + "spki_hashes": { + "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", + "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" + } + } -* Are base64 encoded hashes already widely used/defined? If not, I'd lean towards hex here in this JSON file, since we hopefully have gzip to reduce the encoding overhead anyway, and this file should optimize for admin ease of use. -* These are what's used for certificate pinning in Chrome. Most likely we'll have people use the same tooling to generate these SPKI hashes for us, so I think using the same format and encoding makes sense. It's also easy on admins: "We'll just use the same set of pins we use for HTTPS." -* The other thing I forgot to mention: There isn't, AFAIK, a good way to get the SPKI hash other than using the Chrome tool. It's not one of the fields output by openssl x509, for example. - -* "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", -* "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" -* } -* } A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. From 3d9d5607bd51caca2202d3c79f418fcb4c0554c9 Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:34:12 -0700 Subject: [PATCH 005/631] Formatting issue #2 in design doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4686d5baf..d47ba0f70 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co } } // Similar to - // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json + // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json "pinsets": [ { "name": "google", From 4b5b9f164fcab8cb2ee4601cef804211d74bbdc8 Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:51:19 -0700 Subject: [PATCH 006/631] nexthop-domains -> address-domains --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d47ba0f70..cc510ecfa 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Most email transferred between SMTP servers (aka MTAs) is transmitted in the cle To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. -Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. +Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. @@ -52,7 +52,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "timestamp": 1401093333 "author": "Electronic Frontier Foundation https://eff.org", "expires": 1401414363, // epoch seconds - "nexthop-domains": { + "address-domains": { "gmail.com": { "accept-mx-domains": ["google.com", "gmail.com"] } @@ -111,11 +111,11 @@ Option 2: Git is a revision control system built on top of an authenticated, his Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. -**nexthop-domains** +**address-domains** -The _nexthop-domains _field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _nexthop-domains _section. +The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. -Currently the only property defined for _nexthop-domains _is _accept-mx-domains_, a list. If an MX lookup for a listed nexthop domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. +Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. From 6fb51d54227554445ef0772eb66e8d945d4d13e8 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 14:11:08 -0700 Subject: [PATCH 007/631] Example shouldn't include hashes from Chrome source. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cc510ecfa..3ba644d7f 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,11 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" + "accept-pinset": "eff", } "google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", - "accept-pinset": "google", "enforce-mode": "log-only", // error-notification domains * "error-notification": "https://google.com/post/reports/here" @@ -85,16 +85,16 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json "pinsets": [ { - "name": "google", + "name": "eff", "static_spki_hashes": [ - "GoogleBackup2048", - "GoogleG2" + "EFFBackup2048", + "EFF" ] } ], "spki_hashes": { - "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", - "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" + "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "EFF": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" } } From 714cb17dcb8a7f5aa1c78f52e7a1ebec4928fcb1 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 14:26:45 -0700 Subject: [PATCH 008/631] Clarify example usage of pinsets by including a CA --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ba644d7f..604d2dad0 100644 --- a/README.md +++ b/README.md @@ -88,13 +88,14 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "name": "eff", "static_spki_hashes": [ "EFFBackup2048", - "EFF" + "StartCom Class 2 Primary Intermediate Server CA" ] } ], "spki_hashes": { + // Not real SPKI hashes, just examples "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "EFF": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + "StartCom Class 2 Primary Intermediate Server CA": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" } } From ed0c024209a243cf4759647b93dff7281d000293 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 16:01:07 -0700 Subject: [PATCH 009/631] Improved provisioning and certificate checking. --- Vagrantfile | 7 ++--- check-starttls.py | 74 +++++++++++++++++++++++--------------------- vagrant-bootstrap.sh | 21 +++++++++++-- 3 files changed, 60 insertions(+), 42 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index e485988f1..b990d1311 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,18 +5,15 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "hashicorp/precise32" config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" + sender.vm.hostname = "sender.example.com" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" + valid.vm.hostname = "valid-example-recipient.com" end config.vm.provision :shell, path: "vagrant-bootstrap.sh" diff --git a/check-starttls.py b/check-starttls.py index 15bc7ce64..72f0d3430 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -35,7 +35,7 @@ def extract_names(pem): # >>> twitter_cert.get_ext('subjectAltName').get_value() # 'DNS:www.twitter.com, DNS:twitter.com' alt_names = leaf.get_ext('subjectAltName').get_value() - alt_names = alt_names.split(',') + alt_names = alt_names.split(', ') alt_names = [name.partition(':') for name in alt_names] alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] except: @@ -43,48 +43,52 @@ def extract_names(pem): return set(common_names + alt_names) def tls_connect(mx_host, mail_domain): - # smtplib doesn't let us access certificate information, - # so shell out to openssl. - output = subprocess.check_output( - """openssl s_client \ - -CApath /usr/share/ca-certificates/mozilla/ \ - -starttls smtp -connect %s:25 -showcerts /dev/null + """ % mx_host, shell=True) + except subprocess.CalledProcessError: + print "Failed s_client" + return - # Save a copy of the certificate for later analysis - with open(os.path.join(mail_domain, mx_host), "w") as f: - f.write(output) + # Save a copy of the certificate for later analysis + with open(os.path.join(mail_domain, mx_host), "w") as f: + f.write(output) - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) - if len(cert) > 0: - print "iiii ", len(cert) - print extract_names(cert[0]) - #lines = output.split("\n") - #for i in range(0, len(lines)): - #line = lines[i] - #if re.search("Subject:.* CN=(.*)", line): - #m = re.search("Subject:.* CN=(.*)", line) - #print "CN=", m.group(1) - #elif re.search("Subject Alternative Name:", line): - #dns = re.findall("DNS:([^,]*),", lines[i+1]) - #for d in dns: - #print d + cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) + if len(cert) > 0: + names = extract_names(cert[0]) + print "%s certificate -> %s" % (mx_host, ", ".join(names)) + if mx_host in names: + print " MATCH" + else: + print " NOMATCH" -# try: -# smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) -# smtpserver.ehlo() -# smtpserver.starttls() -# print "Success: %s" % mx_host -# except socket.error as e: -# print "Connection to %s failed: %s" % (mx_host, e.strerror) -# pass +def supports_starttls(mx_host): + try: + smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) + smtpserver.ehlo() + smtpserver.starttls() + return True + print "Success: %s" % mx_host + except socket.error as e: + print "Connection to %s failed: %s" % (mx_host, e.strerror) + return False + except smtplib.SMTPException: + print "No STARTTLS support on %s" % mx_host + return False def check(mail_domain): mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: - mx_host = str(rdata.exchange) - print 'Host', rdata.exchange, 'has preference', rdata.preference + mx_host = str(rdata.exchange).rstrip(".") tls_connect(mx_host, mail_domain) if len(sys.argv) == 1: diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index ac9837198..215abacfd 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -3,8 +3,25 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -q -apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postfix -cat > /etc/hosts <<> /etc/hosts < /etc/dnsmasq.conf +# Not sure why restart is necessary, but otherwise dnsmasq doesn't use +# /etc/hosts to answer queries. +/etc/init.d/dnsmasq restart + +if [ "`hostname`" = "sender" ]; then + crontab < Date: Thu, 5 Jun 2014 16:51:38 -0700 Subject: [PATCH 010/631] Start work on forcing TLS to valid --- Vagrantfile | 2 + vagrant-bootstrap.sh | 2 +- vm-postfix-config/dynamicmaps.cf | 6 ++ vm-postfix-config/main.cf | 39 ++++++++ vm-postfix-config/master.cf | 113 +++++++++++++++++++++++ vm-postfix-config/starttls-everywhere.cf | 1 + vm-postfix-config/tls_policy | 1 + 7 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 vm-postfix-config/dynamicmaps.cf create mode 100644 vm-postfix-config/main.cf create mode 100644 vm-postfix-config/master.cf create mode 100644 vm-postfix-config/starttls-everywhere.cf create mode 100644 vm-postfix-config/tls_policy diff --git a/Vagrantfile b/Vagrantfile index b990d1311..ab5d43307 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,4 +21,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # vb.gui = true vb.customize ["modifyvm", :id, "--memory", "256"] end + + config.vm.synced_folder "vm-postfix-config", "/etc/postfix" end diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 215abacfd..9ec1e3d11 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -4,7 +4,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -q apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - postfix dnsmasq mutt + postfix dnsmasq mutt vim # Provide hostnames so the boxes can talk to each other. DNSMasq will also serve # results to each box based on these contents. diff --git a/vm-postfix-config/dynamicmaps.cf b/vm-postfix-config/dynamicmaps.cf new file mode 100644 index 000000000..1c48bdcde --- /dev/null +++ b/vm-postfix-config/dynamicmaps.cf @@ -0,0 +1,6 @@ +# Postfix dynamic maps configuration file. +# +#type location of .so file open function (mkmap func) +#==== ================================ ============= ============ +tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open +sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config/main.cf b/vm-postfix-config/main.cf new file mode 100644 index 000000000..b9f265058 --- /dev/null +++ b/vm-postfix-config/main.cf @@ -0,0 +1,39 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +myhostname = sender.example.com +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = sender.example.com, localhost.example.com, , localhost +relayhost = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all diff --git a/vm-postfix-config/master.cf b/vm-postfix-config/master.cf new file mode 100644 index 000000000..3df29d8a9 --- /dev/null +++ b/vm-postfix-config/master.cf @@ -0,0 +1,113 @@ +# +# Postfix master process configuration file. For details on the format +# of the file, see the master(5) manual page (command: "man 5 master"). +# +# Do not forget to execute "postfix reload" after editing this file. +# +# ========================================================================== +# service type private unpriv chroot wakeup maxproc command + args +# (yes) (yes) (yes) (never) (100) +# ========================================================================== +smtp inet n - - - - smtpd +#smtp inet n - - - 1 postscreen +#smtpd pass - - - - - smtpd +#dnsblog unix - - - - 0 dnsblog +#tlsproxy unix - - - - 0 tlsproxy +#submission inet n - - - - smtpd +# -o syslog_name=postfix/submission +# -o smtpd_tls_security_level=encrypt +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_client_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#smtps inet n - - - - smtpd +# -o syslog_name=postfix/smtps +# -o smtpd_tls_wrappermode=yes +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_client_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#628 inet n - - - - qmqpd +pickup fifo n - - 60 1 pickup +cleanup unix n - - - 0 cleanup +qmgr fifo n - n 300 1 qmgr +#qmgr fifo n - n 300 1 oqmgr +tlsmgr unix - - - 1000? 1 tlsmgr +rewrite unix - - - - - trivial-rewrite +bounce unix - - - - 0 bounce +defer unix - - - - 0 bounce +trace unix - - - - 0 bounce +verify unix - - - - 1 verify +flush unix n - - 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - - - - smtp +relay unix - - - - - smtp +# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 +showq unix n - - - - showq +error unix - - - - - error +retry unix - - - - - error +discard unix - - - - - discard +local unix - n n - - local +virtual unix - n n - - virtual +lmtp unix - - - - - lmtp +anvil unix - - - - 1 anvil +scache unix - - - - 1 scache +# +# ==================================================================== +# Interfaces to non-Postfix software. Be sure to examine the manual +# pages of the non-Postfix software to find out what options it wants. +# +# Many of the following services use the Postfix pipe(8) delivery +# agent. See the pipe(8) man page for information about ${recipient} +# and other message envelope options. +# ==================================================================== +# +# maildrop. See the Postfix MAILDROP_README file for details. +# Also specify in main.cf: maildrop_destination_recipient_limit=1 +# +maildrop unix - n n - - pipe + flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} +# +# ==================================================================== +# +# Recent Cyrus versions can use the existing "lmtp" master.cf entry. +# +# Specify in cyrus.conf: +# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 +# +# Specify in main.cf one or more of the following: +# mailbox_transport = lmtp:inet:localhost +# virtual_transport = lmtp:inet:localhost +# +# ==================================================================== +# +# Cyrus 2.1.5 (Amos Gouaux) +# Also specify in main.cf: cyrus_destination_recipient_limit=1 +# +#cyrus unix - n n - - pipe +# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} +# +# ==================================================================== +# Old example of delivery via Cyrus. +# +#old-cyrus unix - n n - - pipe +# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} +# +# ==================================================================== +# +# See the Postfix UUCP_README file for configuration details. +# +uucp unix - n n - - pipe + flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) +# +# Other external delivery methods. +# +ifmail unix - n n - - pipe + flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) +bsmtp unix - n n - - pipe + flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient +scalemail-backend unix - n n - 2 pipe + flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} +mailman unix - n n - - pipe + flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py + ${nexthop} ${user} + diff --git a/vm-postfix-config/starttls-everywhere.cf b/vm-postfix-config/starttls-everywhere.cf new file mode 100644 index 000000000..98b1bfb3a --- /dev/null +++ b/vm-postfix-config/starttls-everywhere.cf @@ -0,0 +1 @@ +smtp_tls_policy_maps = hash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/tls_policy b/vm-postfix-config/tls_policy new file mode 100644 index 000000000..a40e02ffd --- /dev/null +++ b/vm-postfix-config/tls_policy @@ -0,0 +1 @@ +valid-example-recipient.com encrypt From 7bd06a4d353400307c22672fe54127d034219451 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 5 Jun 2014 17:04:28 -0700 Subject: [PATCH 011/631] Work in progress --- vm-postfix-config/main.cf | 3 +++ vm-postfix-config/starttls-everywhere.cf | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 vm-postfix-config/starttls-everywhere.cf diff --git a/vm-postfix-config/main.cf b/vm-postfix-config/main.cf index b9f265058..d21969005 100644 --- a/vm-postfix-config/main.cf +++ b/vm-postfix-config/main.cf @@ -37,3 +37,6 @@ mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all + +#STARTTLS EVERYWHERE MAGIC STARTS HERE +smtp_tls_policy_maps = hash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/starttls-everywhere.cf b/vm-postfix-config/starttls-everywhere.cf deleted file mode 100644 index 98b1bfb3a..000000000 --- a/vm-postfix-config/starttls-everywhere.cf +++ /dev/null @@ -1 +0,0 @@ -smtp_tls_policy_maps = hash:/etc/postfix/tls_policy From 30938260d4a4c586da082bc44193bb270d86443f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 5 Jun 2014 19:45:21 -0700 Subject: [PATCH 012/631] Split postfix configs across the VMs, and start making them do things --- Vagrantfile | 3 ++- {vm-postfix-config => vm-postfix-config-sender}/dynamicmaps.cf | 0 {vm-postfix-config => vm-postfix-config-sender}/main.cf | 2 +- {vm-postfix-config => vm-postfix-config-sender}/master.cf | 0 vm-postfix-config-sender/tls_policy | 1 + vm-postfix-config/tls_policy | 1 - 6 files changed, 4 insertions(+), 3 deletions(-) rename {vm-postfix-config => vm-postfix-config-sender}/dynamicmaps.cf (100%) rename {vm-postfix-config => vm-postfix-config-sender}/main.cf (95%) rename {vm-postfix-config => vm-postfix-config-sender}/master.cf (100%) create mode 100644 vm-postfix-config-sender/tls_policy delete mode 100644 vm-postfix-config/tls_policy diff --git a/Vagrantfile b/Vagrantfile index ab5d43307..c919f16a0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -10,10 +10,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" sender.vm.hostname = "sender.example.com" + config.vm.synced_folder "vm-postfix-config-sender", "/etc/postfix" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" valid.vm.hostname = "valid-example-recipient.com" + config.vm.synced_folder "vm-postfix-config-valid", "/etc/postfix" end config.vm.provision :shell, path: "vagrant-bootstrap.sh" @@ -22,5 +24,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.customize ["modifyvm", :id, "--memory", "256"] end - config.vm.synced_folder "vm-postfix-config", "/etc/postfix" end diff --git a/vm-postfix-config/dynamicmaps.cf b/vm-postfix-config-sender/dynamicmaps.cf similarity index 100% rename from vm-postfix-config/dynamicmaps.cf rename to vm-postfix-config-sender/dynamicmaps.cf diff --git a/vm-postfix-config/main.cf b/vm-postfix-config-sender/main.cf similarity index 95% rename from vm-postfix-config/main.cf rename to vm-postfix-config-sender/main.cf index d21969005..2d926a657 100644 --- a/vm-postfix-config/main.cf +++ b/vm-postfix-config-sender/main.cf @@ -39,4 +39,4 @@ recipient_delimiter = + inet_interfaces = all #STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = hash:/etc/postfix/tls_policy +smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/master.cf b/vm-postfix-config-sender/master.cf similarity index 100% rename from vm-postfix-config/master.cf rename to vm-postfix-config-sender/master.cf diff --git a/vm-postfix-config-sender/tls_policy b/vm-postfix-config-sender/tls_policy new file mode 100644 index 000000000..7792bad7f --- /dev/null +++ b/vm-postfix-config-sender/tls_policy @@ -0,0 +1 @@ +valid-example-recipient.com encrypt protocols=TLSv1.2 diff --git a/vm-postfix-config/tls_policy b/vm-postfix-config/tls_policy deleted file mode 100644 index a40e02ffd..000000000 --- a/vm-postfix-config/tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com encrypt From fa5acdf674f36c211927fe1695e68d8dfd5ec785 Mon Sep 17 00:00:00 2001 From: jsha Date: Fri, 6 Jun 2014 13:44:03 -0700 Subject: [PATCH 013/631] Simplify SPKI hash usage --- README.md | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 604d2dad0..0cc4f5f05 100644 --- a/README.md +++ b/README.md @@ -68,35 +68,21 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" - "accept-pinset": "eff", + "accept-spki-hashes": [ + "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + ] } "google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", - // error-notification domains * "error-notification": "https://google.com/post/reports/here" }, "yahoodns.net": { "require-valid-certificate": true, } } - // Similar to - // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json - "pinsets": [ - { - "name": "eff", - "static_spki_hashes": [ - "EFFBackup2048", - "StartCom Class 2 Primary Intermediate Server CA" - ] - } - ], - "spki_hashes": { - // Not real SPKI hashes, just examples - "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "StartCom Class 2 Primary Intermediate Server CA": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - } } From ce0a6a1814ba1f0d222088a5a569e8a9bd945e16 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 14:04:38 -0700 Subject: [PATCH 014/631] Add example config.json --- config.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 config.json diff --git a/config.json b/config.json new file mode 100644 index 000000000..d660587f1 --- /dev/null +++ b/config.json @@ -0,0 +1,17 @@ +{ + // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to + // latest version + "timestamp": 1401093333 + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1404677353, // epoch seconds + "address-domains": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "valid-example-recipient.com" ] + } + } + "mx-domains": { + "valid-example-recipient.com": { + "min-tls-version": "TLSv1.1" + } + } +} From 372c96d9fd4fc529ab82804d79904118a3d582e2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 14:06:08 -0700 Subject: [PATCH 015/631] Update Postfix configuration and mail-send-loop --- vagrant-bootstrap.sh | 6 +- vm-postfix-config-sender/main.cf | 4 + vm-postfix-config-sender/tls_policy | 2 +- vm-postfix-config-valid/dynamicmaps.cf | 6 ++ vm-postfix-config-valid/main.cf | 42 +++++++++ vm-postfix-config-valid/master.cf | 113 +++++++++++++++++++++++++ vm-postfix-config-valid/tls_policy | 1 + 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 vm-postfix-config-valid/dynamicmaps.cf create mode 100644 vm-postfix-config-valid/main.cf create mode 100644 vm-postfix-config-valid/master.cf create mode 100644 vm-postfix-config-valid/tls_policy diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 9ec1e3d11..1df09a9a6 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -20,8 +20,8 @@ echo selfmx > /etc/dnsmasq.conf /etc/init.d/dnsmasq restart if [ "`hostname`" = "sender" ]; then - crontab < Date: Fri, 6 Jun 2014 15:54:22 -0700 Subject: [PATCH 016/631] Some tweaks to the config format --- README.md | 21 +++++++++++---------- config.json | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0cc4f5f05..31495a8f5 100644 --- a/README.md +++ b/README.md @@ -49,22 +49,23 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co { // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version - "timestamp": 1401093333 + "timestamp": "2014-06-06T14:30:16+00:00", + // "timestamp": 1401414363, : also acceptable "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1401414363, // epoch seconds - "address-domains": { + "expires": "2014-06-06T14:30:16+00:00", + "acceptable-mxs": { "gmail.com": { - "accept-mx-domains": ["google.com", "gmail.com"] + "accept-mx-domains": ["*.google.com", "*.gmail.com"] } "yahoo.com": { - "accept-mx-domains": ["yahoodns.net"] + "accept-mx-domains": ["*.yahoodns.net"] } "eff.org": { - "accept-mx-domains": ["eff.org"] + "accept-mx-domains": ["*.eff.org"] } } - "mx-domains": { - "eff.org": { + "security-policies": { + "*.eff.org": { "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" @@ -73,13 +74,13 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" ] } - "google.com": { + "*.google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", "error-notification": "https://google.com/post/reports/here" }, - "yahoodns.net": { + "*.yahoodns.net": { "require-valid-certificate": true, } } diff --git a/config.json b/config.json index d660587f1..8d6a73d63 100644 --- a/config.json +++ b/config.json @@ -1,17 +1,17 @@ { - // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to - // latest version - "timestamp": 1401093333 + "comment": "Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version", + "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404677353, // epoch seconds - "address-domains": { - "valid-example-recipient.com": { - "accept-mx-domains": [ "valid-example-recipient.com" ] - } - } - "mx-domains": { - "valid-example-recipient.com": { + "expires": 1404677353, "comment 2:": "epoch seconds", + "tls-policies": { + "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } } + "acceptable-mxs": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "*.valid-example-recipient.com" ] + } + }, + } From fcd1a982010ff6fdba113ae208aad4b93750f83e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 6 Jun 2014 15:54:33 -0700 Subject: [PATCH 017/631] Break ground on a config parser --- config-parser.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 config-parser.py diff --git a/config-parser.py b/config-parser.py new file mode 100755 index 000000000..69a09a745 --- /dev/null +++ b/config-parser.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import sys +import json +from datetime import datetime + +def parse_timestamp(ts): + try: + int(ts) + dt = datetime.fromtimestamp(ts) + return dt + except: + raise ValueError, "Invalid timestamp integer: " + `ts` + + +class Config: + def __init__(self, cfg_file_name = "config.json"): + f = open(cfg_file_name) + cfg = json.loads(f.read()) + for atr, val in cfg.items(): + #print atr,val + # Parse and verify each attribute of the structure + if atr.startswith("comment"): + continue + if atr == "author": + if type(val) not in [str, unicode]: + raise TypeError, "Author must be a string: " + `val` + elif atr == "timestamp": + self.timestamp = parse_timestamp(val) + elif atr == "expires": + self.expires = parse_timestamp(val) + elif atr == "address-domains": + if type(val) != dict: + raise TypeError, "address-domains " + `val` + else: + sys.stderr.write("Uknown attribute: " + `atr` + "\n") + +if __name__ == "__main__": + c = Config() From e534a43d1ad2d9a75a5d585034383077997d6e2d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 16:07:38 -0700 Subject: [PATCH 018/631] Make sender actually attempt TLS on outbound connections. --- vm-postfix-config-sender/main.cf | 3 +-- vm-postfix-config-sender/tls_policy | 2 +- vm-postfix-config-valid/main.cf | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vm-postfix-config-sender/main.cf b/vm-postfix-config-sender/main.cf index c4e994063..cf74d122f 100644 --- a/vm-postfix-config-sender/main.cf +++ b/vm-postfix-config-sender/main.cf @@ -40,7 +40,6 @@ inet_interfaces = all #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy -smtpd_tls_loglevel = 1 -smtpd_tls_received_header = yes smtp_tls_loglevel = 1 +smtp_tls_security_level = may diff --git a/vm-postfix-config-sender/tls_policy b/vm-postfix-config-sender/tls_policy index f8d6a4968..af948c5e7 100644 --- a/vm-postfix-config-sender/tls_policy +++ b/vm-postfix-config-sender/tls_policy @@ -1 +1 @@ -valid-example-recipient.com encrypt protocols=TLSv1.1 +#valid-example-recipient.com encrypt protocols=TLSv1.1 diff --git a/vm-postfix-config-valid/main.cf b/vm-postfix-config-valid/main.cf index 140b6f35d..a5f7ce575 100644 --- a/vm-postfix-config-valid/main.cf +++ b/vm-postfix-config-valid/main.cf @@ -38,5 +38,9 @@ mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all +# STARTLS Everywhere recommended best-practice settings +smtpd_tls_session_cache_timeout = 3600s +smtpd_tls_received_header = yes + #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy From 839c5230483cff26b31ed689ff0408f1b7ad7b01 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 8 Jun 2014 06:22:22 -0700 Subject: [PATCH 019/631] Fix typos --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 8d6a73d63..1a9034545 100644 --- a/config.json +++ b/config.json @@ -7,11 +7,11 @@ "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } - } + }, "acceptable-mxs": { "valid-example-recipient.com": { "accept-mx-domains": [ "*.valid-example-recipient.com" ] } - }, + } } From c033905b162650e41995bec30c9e79bf66642945 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 8 Jun 2014 06:22:32 -0700 Subject: [PATCH 020/631] Now validating most of the config json --- config-parser.py | 58 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/config-parser.py b/config-parser.py index 69a09a745..dbc244b25 100755 --- a/config-parser.py +++ b/config-parser.py @@ -3,6 +3,8 @@ import sys import json from datetime import datetime +import string + def parse_timestamp(ts): try: @@ -12,14 +14,30 @@ def parse_timestamp(ts): except: raise ValueError, "Invalid timestamp integer: " + `ts` +legal = string.letters + string.digits + ".-" +known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list +def looks_like_a_domain(s): + "Return true if string looks like a domain, as best we can tell..." + global known_tlds + try: + domain = s.lower() + assert domain[0].islower() + assert all([c in legal for c in domain]) + tld = s.split(".")[-1] + if tld not in known_tlds: + # XXX perform DNS query to determine that this TLD exists + pass + return True + except: + return False class Config: def __init__(self, cfg_file_name = "config.json"): f = open(cfg_file_name) - cfg = json.loads(f.read()) - for atr, val in cfg.items(): + self.cfg = json.loads(f.read()) + for atr, val in self.cfg.items(): #print atr,val - # Parse and verify each attribute of the structure + # Verify each attribute of the structure if atr.startswith("comment"): continue if atr == "author": @@ -29,11 +47,39 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "address-domains": - if type(val) != dict: - raise TypeError, "address-domains " + `val` + elif atr == "tls-policies": + self.tls_policies = {} + for domain,policies in self.check_tls_policy_domains(val): + if type(policies) != dict: + raise TypeError, domain + "'s policies should be a dict: " + `policies` + self.tls_policies[domain] = {} # being here enforces TLS at all + for policy, value in policies.items(): + if policy == "min-tls-version": + reasonable = ["TLS", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + if not value in reasonable: + raise ValueError, "Not a valid TLS version string: " + `value` + self.tls_policies[domain]["min-tls-version"] = str(value) + elif atr == "acceptable-mxs": + pass else: sys.stderr.write("Uknown attribute: " + `atr` + "\n") + print self.tls_policies + + def check_tls_policy_domains(self, val): + if type(val) != dict: + raise TypeError, "tls-policies should be a dict" + `val` + for domain, policies in val.items(): + try: + assert type(domain) == unicode + d = str(domain) # convert from unicode + except: + raise TypeError, "tls-policy domain not a string" + `domain` + if not d.startswith("*."): + raise ValueError, "tls-policy domains must start with *.; try *."+d + d = d.partition("*.")[2] + if not looks_like_a_domain(d): + raise ValueError, "tls-policy for something that a domain? " + d + yield (d, policies) if __name__ == "__main__": c = Config() From 79924108c7beae6c513215d102d4fffba511d794 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 10:12:09 -0700 Subject: [PATCH 021/631] Reorder JSON file to emphasize MX policies over address-domain -> MX domain mapping. --- README.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0cc4f5f05..e252681fc 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,11 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "timestamp": 1401093333 "author": "Electronic Frontier Foundation https://eff.org", "expires": 1401414363, // epoch seconds - "address-domains": { - "gmail.com": { - "accept-mx-domains": ["google.com", "gmail.com"] - } - "yahoo.com": { - "accept-mx-domains": ["yahoodns.net"] - } - "eff.org": { - "accept-mx-domains": ["eff.org"] - } - } "mx-domains": { - "eff.org": { + "*.yahoodns.net": { + "require-valid-certificate": true, + } + "*.eff.org": { "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" @@ -73,15 +65,25 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" ] } - "google.com": { + "*.google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", "error-notification": "https://google.com/post/reports/here" }, - "yahoodns.net": { - "require-valid-certificate": true, - } + } + // Since the MX lookup is not secure, we list valid responses to protect + // against DNS spoofing. + "address-domains": { + "yahoo.com": { + "accept-mx-domains": ["*.yahoodns.net"] + } + "gmail.com": { + "accept-mx-domains": ["*.google.com"] + } + "eff.org": { + "accept-mx-domains": ["*.eff.org"] + } } } From 0d43d2988a062d819663b282668dc81bff440be6 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 13:08:01 -0700 Subject: [PATCH 022/631] Update check-starttls.py to generate starttls everywhere config. --- check-starttls.py | 104 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/check-starttls.py b/check-starttls.py index 72f0d3430..042d2e50d 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -6,6 +6,7 @@ import smtplib import socket import subprocess import re +import json import dns.resolver from M2Crypto import X509 @@ -19,9 +20,9 @@ def mkdirp(path): else: raise def extract_names(pem): + """Return a list of DNS subject names from PEM-encoded leaf cert.""" leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) - """Extracts a list of DNS names associated with the leaf cert.""" subj = leaf.get_subject() # Certs have a "subject" identified by a Distingushed Name (DN). # Host certs should also have a Common Name (CN) with a DNS name. @@ -43,16 +44,16 @@ def extract_names(pem): return set(common_names + alt_names) def tls_connect(mx_host, mail_domain): + """Attempt a STARTTLS connection with openssl and save the output.""" if supports_starttls(mx_host): # smtplib doesn't let us access certificate information, # so shell out to openssl. try: output = subprocess.check_output( """openssl s_client \ - -CApath /usr/share/ca-certificates/mozilla/ \ -starttls smtp -connect %s:25 -showcerts /dev/null - """ % mx_host, shell=True) + """ % mx_host, shell = True) except subprocess.CalledProcessError: print "Failed s_client" return @@ -61,14 +62,53 @@ def tls_connect(mx_host, mail_domain): with open(os.path.join(mail_domain, mx_host), "w") as f: f.write(output) - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) - if len(cert) > 0: - names = extract_names(cert[0]) - print "%s certificate -> %s" % (mx_host, ", ".join(names)) - if mx_host in names: - print " MATCH" - else: - print " NOMATCH" +def valid_cert(filename): + """Return true if the certificate is valid. + + Note: CApath must have hashed symlinks to the trust roots. + TODO: Include the -attime flag based on file modification time.""" + + if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: + return False + try: + output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ + -untrusted "%s" \ + "%s" + """ % (filename, filename), shell = True) + return True + except subprocess.CalledProcessError: + return False + +def check_certs(mail_domain): + names = set() + for mx_hostname in os.listdir(mail_domain): + filename = os.path.join(mail_domain, mx_hostname) + if not valid_cert(filename): + return "" + else: + new_names = extract_names_from_openssl_output(filename) + names.update(new_names) + names.add(filename.rstrip(".")) + if len(names) >= 1: + return common_suffix(names) + else: + return "" + +def common_suffix(hosts): + num_components = min(len(h.split(".")) for h in hosts) + longest_suffix = "" + for i in range(1, num_components + 1): + suffixes = set(".".join(h.split(".")[-i:]) for h in hosts) + if len(suffixes) == 1: + longest_suffix = suffixes.pop() + else: + return longest_suffix + return longest_suffix + +def extract_names_from_openssl_output(certificates_file): + openssl_output = open(certificates_file, "r").read() + cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", openssl_output, flags = re.DOTALL) + return extract_names(cert[0]) def supports_starttls(mx_host): try: @@ -84,15 +124,47 @@ def supports_starttls(mx_host): print "No STARTTLS support on %s" % mx_host return False -def check(mail_domain): +def min_tls_version(mail_domain): + protocols = [] + for mx_hostname in os.listdir(mail_domain): + filename = os.path.join(mail_domain, mx_hostname) + contents = open(filename).read() + protocol = re.findall("Protocol : (.*)", contents)[0] + protocols.append(protocol) + return min(protocols) + +def collect(mail_domain): mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: mx_host = str(rdata.exchange).rstrip(".") tls_connect(mx_host, mail_domain) -if len(sys.argv) == 1: - print("Please pass at least one mail domain as an argument") +if __name__ == '__main__': + """Consume a target list of domains and output a configuration file for those domains.""" + if len(sys.argv) == 1: + print("Please pass at least one mail domain as an argument") -for domain in sys.argv[1:]: - check(domain) + config = { + "address-domains": { + }, + "mx-domains": { + } + } + for domain in sys.argv[1:]: + #collect(domain) + if len(os.listdir(domain)) == 0: + continue + suffix = check_certs(domain) + min_version = min_tls_version(domain) + if suffix != "": + suffix_match = "*." + suffix + config["address-domains"][domain] = { + "accept-mx-domains": [suffix_match] + } + config["mx-domains"][suffix_match] = { + "require-tls": True, + "min-tls-version": min_version + } + + print json.dumps(config, indent=2) From bdbc46fc84599dd141b9bb8585f11b2d168aac1a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 13:10:43 -0700 Subject: [PATCH 023/631] Add candidate starttls-everywhere config json --- starttls-everywhere.json | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 starttls-everywhere.json diff --git a/starttls-everywhere.json b/starttls-everywhere.json new file mode 100644 index 000000000..4c91866d5 --- /dev/null +++ b/starttls-everywhere.json @@ -0,0 +1,144 @@ +{ + "mx-domains": { + "*.mx.aol.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.psmtp.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.ukr.net": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.interia.pl": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.gmx.net": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.web.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.marktplaats.nl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.wp.pl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.yahoodns.net": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.t-online.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.rambler.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.t.facebook.com": { + "min-tls-version": "TLSv1", + "require-tls": true + } + }, + "address-domains": { + "wp.pl": { + "accept-mx-domains": [ + "*.wp.pl" + ] + }, + "yahoo.co.uk": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "rocketmail.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "web.de": { + "accept-mx-domains": [ + "*.web.de" + ] + }, + "sbcglobal.net": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "aol.com": { + "accept-mx-domains": [ + "*.mx.aol.com" + ] + }, + "facebook.com": { + "accept-mx-domains": [ + "*.t.facebook.com" + ] + }, + "sompo-japan.co.jp": { + "accept-mx-domains": [ + "*.psmtp.com" + ] + }, + "salesforce.com": { + "accept-mx-domains": [ + "*.psmtp.com" + ] + }, + "rambler.ru": { + "accept-mx-domains": [ + "*.rambler.ru" + ] + }, + "t-online.de": { + "accept-mx-domains": [ + "*.t-online.de" + ] + }, + "gmx.net": { + "accept-mx-domains": [ + "*.gmx.net" + ] + }, + "gmx.de": { + "accept-mx-domains": [ + "*.gmx.net" + ] + }, + "ukr.net": { + "accept-mx-domains": [ + "*.ukr.net" + ] + }, + "rogers.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "ymail.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "marktplaats.nl": { + "accept-mx-domains": [ + "*.marktplaats.nl" + ] + }, + "interia.pl": { + "accept-mx-domains": [ + "*.interia.pl" + ] + } + } +} From d0bcc1305964ad8e756d85c9c50a007f0ffe0933 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 10 Jun 2014 08:08:17 -0700 Subject: [PATCH 024/631] Break ground on an postfix config wrangling engine --- mta-config-generator.py | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 mta-config-generator.py diff --git a/mta-config-generator.py b/mta-config-generator.py new file mode 100644 index 000000000..0444c1e73 --- /dev/null +++ b/mta-config-generator.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +import string + +DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" + +def parse_line(self, line): + "return the and right hand sides of stripped, non-comment postfix config line" + # lines are like: + # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + left, sep, right = line.partition("=") + if not sep: + return None + return (left.strip(), right.strip()) + +#def get_cf_values(lines, var): + +class MTAConfigGenerator: + def __init__(self, stlse_config): + self.c = stlse_config + +class ExistingConfigError(ValueError): pass + +class PostfixConfigGenerator(MTAConfigGenerator): + def __init__(self, stlse_config): + MTAConfigGenerator.__init__(self, stlse_config) + this.postfix_cf_file = this.find_postfix_cf() + this.wrangle_existing_config() + + def ensure_cf_var(self, var, ideal, also_acceptable): + """ + Ensure that existing postfix config @var is in the list of @acceptable + values; if not, set it to the ideal value. """ + + acceptable = [ideal] + also_acceptable + + l = [line for line in cf if line.startswith("stmpd_use_tls")] + if not any(l): + this.additions.append("smtpd_use_tls = yes") + else: + values = [right for left, right in map(parse_line, l)] + if len(set(values)) > 1: + raise ExistingConfigError, "Conflicting existing config values " + `l` + if values[0] != "yes": + + def wrangle_existing_config(self): + "Try to ensure/mutate that the config file is in a sane state." + this.additions = [] + fn = find_postfix_cf() + raw_cf = open(fn).readlines() + cf = map(string.strip, raw_cf) + this.cf = [line for line in cf if line and not line.startswith("#")] + + # Check we're currently accepting inbound STARTTLS sensibly + this.ensure_cf_var("smtpd_use_tls", "yes", []) + # Ideally we use it opportunistically in the outbound direction + this.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + # Maximum verbosity lets us collect failure information + this.ensure_cf_var("smtp_tls_loglevel", "1", []) + # Inject a reference to our per-domain policy map + this.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + + this.maybe_add_config_lines() + + def maybe_add_config_lines(self): + if not this.additions: + return + this.additions[:0]=["","# New config lines added by STARTTLS Everywhere",""] + new_cf_lines = "\n".join(this.additions) + print "Adding to %s:" % fn + print new_cf_lines + if raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" + new_cf = "".join(raw_cf) + sep + new_cf_lines + f = open(fn, "w").write(new_cf) + f.close() + + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return "/etc/postfix/main.cf" From a03db04ff44335e1911e4a79787cdff025bb711f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 10 Jun 2014 08:08:40 -0700 Subject: [PATCH 025/631] WIP implementing deletion of existing cf lines --- mta-config-generator.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mta-config-generator.py b/mta-config-generator.py index 0444c1e73..5d79423ef 100644 --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -4,10 +4,11 @@ import string DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" -def parse_line(self, line): +def parse_line(self, line_data): "return the and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + num,line = line_data left, sep, right = line.partition("=") if not sep: return None @@ -34,22 +35,28 @@ class PostfixConfigGenerator(MTAConfigGenerator): acceptable = [ideal] + also_acceptable - l = [line for line in cf if line.startswith("stmpd_use_tls")] + l = [num,line for num,line in enumerate(cf) if line.startswith(var)] if not any(l): this.additions.append("smtpd_use_tls = yes") else: values = [right for left, right in map(parse_line, l)] if len(set(values)) > 1: + if this.fixup: + this.deletions.append( raise ExistingConfigError, "Conflicting existing config values " + `l` if values[0] != "yes": - def wrangle_existing_config(self): - "Try to ensure/mutate that the config file is in a sane state." + def wrangle_existing_config(self, fixup=false): + """ + Try to ensure/mutate that the config file is in a sane state. + Fixup means we'll delete existing lines if necessary to get there. + """ this.additions = [] fn = find_postfix_cf() raw_cf = open(fn).readlines() - cf = map(string.strip, raw_cf) - this.cf = [line for line in cf if line and not line.startswith("#")] + this.cf = map(string.strip, raw_cf) + #this.cf = [line for line in cf if line and not line.startswith("#")] + this.fixup = fixup # Check we're currently accepting inbound STARTTLS sensibly this.ensure_cf_var("smtpd_use_tls", "yes", []) From 21e841fd13c028e1db9443ae962b53353ae59dde Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 10 Jun 2014 15:08:52 -0400 Subject: [PATCH 026/631] Move synced folders into a common one. Also, create certificates. --- Vagrantfile | 3 +- check-starttls.py | 2 +- vagrant-bootstrap.sh | 6 +- vagrant-shared/certificates/ca.crt | 22 ++++ vagrant-shared/certificates/ca.key | 27 +++++ vagrant-shared/certificates/certificates | 1 + vagrant-shared/certificates/valid.crt | 21 ++++ vagrant-shared/certificates/valid.csr | 18 +++ vagrant-shared/certificates/valid.key | 27 +++++ .../postfix-config-sender-tls_policy | 0 .../postfix-config-sender.cf | 0 .../postfix-config-valid-example-recipient.cf | 0 vm-postfix-config-sender/dynamicmaps.cf | 6 - vm-postfix-config-sender/master.cf | 113 ------------------ vm-postfix-config-valid/dynamicmaps.cf | 6 - vm-postfix-config-valid/master.cf | 113 ------------------ vm-postfix-config-valid/tls_policy | 1 - 17 files changed, 123 insertions(+), 243 deletions(-) create mode 100644 vagrant-shared/certificates/ca.crt create mode 100644 vagrant-shared/certificates/ca.key create mode 120000 vagrant-shared/certificates/certificates create mode 100644 vagrant-shared/certificates/valid.crt create mode 100644 vagrant-shared/certificates/valid.csr create mode 100644 vagrant-shared/certificates/valid.key rename vm-postfix-config-sender/tls_policy => vagrant-shared/postfix-config-sender-tls_policy (100%) rename vm-postfix-config-sender/main.cf => vagrant-shared/postfix-config-sender.cf (100%) rename vm-postfix-config-valid/main.cf => vagrant-shared/postfix-config-valid-example-recipient.cf (100%) delete mode 100644 vm-postfix-config-sender/dynamicmaps.cf delete mode 100644 vm-postfix-config-sender/master.cf delete mode 100644 vm-postfix-config-valid/dynamicmaps.cf delete mode 100644 vm-postfix-config-valid/master.cf delete mode 100644 vm-postfix-config-valid/tls_policy diff --git a/Vagrantfile b/Vagrantfile index c919f16a0..3ce93c917 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -10,13 +10,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" sender.vm.hostname = "sender.example.com" - config.vm.synced_folder "vm-postfix-config-sender", "/etc/postfix" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" valid.vm.hostname = "valid-example-recipient.com" - config.vm.synced_folder "vm-postfix-config-valid", "/etc/postfix" end + config.vm.synced_folder "vagrant-shared", "/vagrant" config.vm.provision :shell, path: "vagrant-bootstrap.sh" config.vm.provider "virtualbox" do |vb| diff --git a/check-starttls.py b/check-starttls.py index 042d2e50d..cd8dfea42 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -152,7 +152,7 @@ if __name__ == '__main__': } } for domain in sys.argv[1:]: - #collect(domain) + collect(domain) if len(os.listdir(domain)) == 0: continue suffix = check_certs(domain) diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 1df09a9a6..13593a0cc 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -23,5 +23,9 @@ if [ "`hostname`" = "sender" ]; then (while sleep 10; do echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com done) & -EOF + ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy fi + +ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf +ln -sf "/vagrant/certificates" /etc/certificates +postfix reload diff --git a/vagrant-shared/certificates/ca.crt b/vagrant-shared/certificates/ca.crt new file mode 100644 index 000000000..dce966d32 --- /dev/null +++ b/vagrant-shared/certificates/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDmzCCAoOgAwIBAgIJAIheS+k2UDobMA0GCSqGSIb3DQEBBQUAMGQxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxDDAKBgNVBAoMA0VGRjEt +MCsGA1UECwwkU1RBUlRUTFMgRXZlcnl3aGVyZSB0ZXN0IGNlcnRpZmljYXRlMB4X +DTE0MDYxMDE4MDg0OVoXDTE5MDYxMDE4MDg1MFowZDELMAkGA1UEBhMCVVMxCzAJ +BgNVBAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRT +VEFSVFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDDYmUQUjaX6L3n9fNks2yQUFhKUBR1NYenx3w2 +DnaZiwpLzI3igJPMQfBdyJGLM1jXZZcpvpgt6yN4OMOLNS2QKBY20gDoQIh0Jmaj +KCoXUbX30H1FfTn+pyU02UpuFsFN3TAk5bQ/BQUYOlMouCowyZ25mnEzzHLeRHKH +Gi2uCH59T53rcgDwjq88pKMVUlndixkOKpeXZkTL++Edg0b5SUpRuzMs6kFmwLoQ +x4xG5lgaAHyu2/9KXhcqielE95s5FNGfi9U2q3nkmpa4dM266DWfkibfvCBP3hiO +Ks+IN+Hyohi2SY7NoDN6hMKcuUNlAK/xD+foA4Ck3R45HII3AgMBAAGjUDBOMB0G +A1UdDgQWBBRPlrBeFMPb27oWESmXykOW1XWLLDAfBgNVHSMEGDAWgBRPlrBeFMPb +27oWESmXykOW1XWLLDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBY +lBQEuRKo927jJFzgvdTKJGIAzxBnC5vg+qKeZkduwxeVEwB13HPP1syvdpAK5dkl +JGuHCF/Yc39HX1OZv7huVRMnyrSKpMt25uqUHirH6Db17HRCS4ZA6rARJfxS6RWu +J31lfXvGRr0hI4yw3XpKGM/c8Gkzji6PsYn4TCPnXjJbRj1GjHFaAuIeyoO0zQjo +OuHnzxs4bIDaV32NHNetuDKSO1GNbenxRiiN1HvQ1vfhzpqerRRaPCHrW4eUcynQ +AAeuC+Ek925t9mH7Ni/kZ0eN7XUwvg3c2lm+LeV+ICP+NWv9r92kfXDZNMhlhNsf +Y8+m1Y9WL3CLj99voNQS +-----END CERTIFICATE----- diff --git a/vagrant-shared/certificates/ca.key b/vagrant-shared/certificates/ca.key new file mode 100644 index 000000000..3ebf4159e --- /dev/null +++ b/vagrant-shared/certificates/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAw2JlEFI2l+i95/XzZLNskFBYSlAUdTWHp8d8Ng52mYsKS8yN +4oCTzEHwXciRizNY12WXKb6YLesjeDjDizUtkCgWNtIA6ECIdCZmoygqF1G199B9 +RX05/qclNNlKbhbBTd0wJOW0PwUFGDpTKLgqMMmduZpxM8xy3kRyhxotrgh+fU+d +63IA8I6vPKSjFVJZ3YsZDiqXl2ZEy/vhHYNG+UlKUbszLOpBZsC6EMeMRuZYGgB8 +rtv/Sl4XKonpRPebORTRn4vVNqt55JqWuHTNuug1n5Im37wgT94YjirPiDfh8qIY +tkmOzaAzeoTCnLlDZQCv8Q/n6AOApN0eORyCNwIDAQABAoIBAQCwNXASDSNBU1zZ +8v3kZtDVQjCuLJSWtIU4cnd6RQb/KN9LRxr7GJyyzREbc4SXduJ7uBphQov6daMS +jJcGWBpUdWK7ZB//Vhv6LJvKL7HuP/oNmhEwd2SzXkj25bTznkANmhsOW794Sm2y +0P8orRcX0u0Vc8z+Ozepby+e2qQx29FXznberRv2rXmMeeQqF+sc2MBu34SlcLnK +KVSe7SmDc+DhWJ3XiPoEpiOTbv4EynndXC85owdse/eN/EahFtrfzqm6jDWVga8A +xA/7RD/Urc2L2IsZOB92xOk/tGs5Df8ZrHavzbSo+i0pzYAKvIxr+G2Krl1Tl/Lh +IXwjWPLZAoGBAOsbdxeZUHH6pk497ICIcmW/NDhAY/mSA6vBWZWLcYF7OU/wMzZ6 +Wlx9v6oz8tBTzQj/Xkpv0ZLIeQqGTuSQzQ7EcrOiTz4PIpM5y/nf/S0oSXwJ7mJw +Sx8w3XHqHEPCT9G83o7EV7xHuRy8/zQzJ9dkFxznA9sRcO7RhE/tsrMNAoGBANS/ +Ql140oyDqyABIlxswyvfJ8Ll4kmb52yNGaCJPSfrAVmSoN15AkbDkpVKFb6hsL6u +xqVPyeVxard4twddrV3GSvTibkGw4fZ8x0FDgDGvCgJ36e4NHb7jQVwwGfNXKMAP +qvYtnE28eAM+zrDHryEhgp4k59zApphr+IU7JQlTAoGAc1h5ODm+rvzTBMX6tyC6 +R1Lkcsicg//wDx8ALY9JM8ZZ2u80oQCsPn5vPzjXYwAKMuTexNRRVJtITzKPmDG2 +eQ1GXP0/tWnFg8eyXDhZRQNj8hgJPYBsSrQ1oMLD9TZq5LKt2gtYJAZoOkI7Tsfe +Px1a/ZIVYTAQYQqnyHMM3i0CgYAQNirWeJiCwJ3PqIZ3yInu0+hxv5bIySqPaQkk +5JBWdF/79WJwvgHgZpLK8YRKrIONZEAa5MObylK5fGdmFktZs/yOQJrqQpJVeBiu +7nfcUVxP59dZnoI/w419euTfWCrwx8DdVYhtnAkBJk4VxoGf4q/TYTiR59RKFSAw +9trRpQKBgDjFQ9mXJUUKoc61P3PIrSYOlnx89c0sznrwm2eeLLAwr/VQ+vNDGRkC +6kXbVLuTxRaFnLSZTlUd0nSXPbJEM5GII48zbGZ7t/ysPtA9390xk1efTO/ryf/j +pn4FBvpgXaneRs4ExyknLgFfJRUEu17fRbyRPZr0bUlTkKg/ZS5M +-----END RSA PRIVATE KEY----- diff --git a/vagrant-shared/certificates/certificates b/vagrant-shared/certificates/certificates new file mode 120000 index 000000000..d162b42af --- /dev/null +++ b/vagrant-shared/certificates/certificates @@ -0,0 +1 @@ +/vagrant/certificates \ No newline at end of file diff --git a/vagrant-shared/certificates/valid.crt b/vagrant-shared/certificates/valid.crt new file mode 100644 index 000000000..f5fe4c213 --- /dev/null +++ b/vagrant-shared/certificates/valid.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlUCAQEwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRTVEFS +VFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwHhcNMTQwNjEwMTgxMTQ3 +WhcNMTkwNjEwMTgxMTQ3WjCBlDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQsw +CQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMTQwMgYDVQQLDCtTVEFSVFRMUyBFdmVy +eXdoZXJlIHRlc3QgY2VydGlmaWNhdGUgKGxlYWYpMScwJQYDVQQDDB5teC52YWxp +ZC1leGFtcGxlLXJlY2lwaWVudC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDZ2NdAwI0LUmXrV7bItKhP8NRzypkYHgApXaoZ6iB6TJnYb1FZyv2Y +wyBdsQQ8zcIgKRyl0rKvNPgHt3riM7nSoB16II6fIQDuR2UnXkhtDOfUd7Ye0wKX +l3A+qoEge42/TfRvzPC6IMa1KH7+dwayprIjLxFUzfMl6GqA/5auLZZtSsV6Pix3 +jXZYFPUHPoBJEyNo/bizcdvSZS/7Kwzfc64l7JLA/OGtRbQpcMAOpRRXCx32nt3N +2L1OXz8Q4It+J20oN8Vfin1a7GbAGdktRbz2bV8bmb0ux1NOnddigPUHRRYMIaKN +W7Nrxp2YQpqmsqBmVEVPA903yRc0ZvuRAgMBAAEwDQYJKoZIhvcNAQEFBQADggEB +AMJ3neFM+to/tFhTiAfUnIrQOKyfk+zYN8gC99HD2SUVbu+Cu1qCNJWxbE3bdxxX +y960yX+Or8E4nG9sWbFbzlGEzz0F8DAPEh4UCtb/5MRkhW158Y9LtbheQAQpZolQ +Sma6lnngCmSDr/OpDW34oM8S7+3VOawYv1e1ruCMqizBwilgcsGq2hY1hWca6LMP +X2FHQ+m4mYBV9wJ9WuKMs9uADz9cfMYuA/wMzNeMZyM82w+nSQGZ+mX7YPu+WBJM +k9OcFrUWxrC/uOQfsyqdjLvFjg4/b49jnusn9mZ/KbtLVmkz5P+5kwSn4kcd5Rlw +LLmEGiXiyEjo8UmEVyumrIQ= +-----END CERTIFICATE----- diff --git a/vagrant-shared/certificates/valid.csr b/vagrant-shared/certificates/valid.csr new file mode 100644 index 000000000..a58aab677 --- /dev/null +++ b/vagrant-shared/certificates/valid.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC2jCCAcICAQAwgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UE +BwwCU0YxDDAKBgNVBAoMA0VGRjE0MDIGA1UECwwrU1RBUlRUTFMgRXZlcnl3aGVy +ZSB0ZXN0IGNlcnRpZmljYXRlIChsZWFmKTEnMCUGA1UEAwwebXgudmFsaWQtZXhh +bXBsZS1yZWNpcGllbnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9RWcr9mMMgXbEE +PM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2HtMCl5dwPqqB +IHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrFej4sd412WBT1 +Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd9p7dzdi9Tl8/ +EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UWDCGijVuza8ad +mEKaprKgZlRFTwPdN8kXNGb7kQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAAhe +pMLNDAaA9fXZ/yqW4ov1pBKYZL1R1RX+YgZqUPoAlrTCiJ0UGIuPcTDGJDMdpZ7x +9gSqwsXrvzXafuHI4GuFjnbY5SIv8zvc+nMXib7IMlyMUcuSBZP8W0sl3ZGWnnpk +legC10c3+I9TfQ7Tl0mpdyf/6yLhM1plxLcIy5bguLJjbBK9JhKNfc84rivqrxUI ++cUqWU13WjMzWdKS6rK5m/Bfleg+jyZ11xYY0QfwNGwuPjfiEBjCs2iqJvdLFRem +FoDq3XBsrH5XohSwWZ6UrZY7wARkmHsYJeYTHulb3MkDPCQKlbU+2SMIpbNk43+/ +fN/ctxWXg3vd/q6eJiE= +-----END CERTIFICATE REQUEST----- diff --git a/vagrant-shared/certificates/valid.key b/vagrant-shared/certificates/valid.key new file mode 100644 index 000000000..b5e8371cd --- /dev/null +++ b/vagrant-shared/certificates/valid.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9R +Wcr9mMMgXbEEPM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2 +HtMCl5dwPqqBIHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrF +ej4sd412WBT1Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd +9p7dzdi9Tl8/EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UW +DCGijVuza8admEKaprKgZlRFTwPdN8kXNGb7kQIDAQABAoIBAHAZgUq0yt+UmxWr +oUdOj33zc5/SFU2vwm2G4U1MiUHlwRT602Xdavn9Dt6nhIK1bruV7EP4VDKMk0WF +SRq1e13DPuflcP65wPzciFTl02cqSPGwWGssMh1HtF7K5n+MlLhoqOwPDaD51MbL +++192lh8Jxar1cNJ52EOZB/VZfhiUZ88JAITWam6clId8KS4cy7SvQ9haptcPlfl +wIeCti2eI4nvC0IR0SCEqWAax6XqypA5k2fjQJklcnBK1R+H1muc00J5lMIz4U8q +qFb5RgxIDMcEeQZWmzfBjU5hO0q/51QyWFFFuBL00MkT4djCHjO/IPYc6DQoAoZR +TbMCWwECgYEA+i3W44tSY96Fdn1gDyYhDor9GPN0XwmRBHpl44TxBLG48I6cGLYI +l0InEh7pu7/0fw+cWylx2OLrl2HeGlWilEOpP2ucJcswcNibATtgSdnu7PjC4eAW +46a5+2hbFk/NONU5VTGClJHCvKzYPWMrAIkRT//WdkIVs23i7bRcOjkCgYEA3upr +cnMhnwVmZrGAIm+DtiDJCc4uVXeqLD2j9csCVNCw/CQAjJ3/5VlQDSjzerjijw/V +uoA+Su8xMoMCg6mAihqdxBMrbwdlh7vje23RaxbeQkIY5JIVAsdDr+UtwofaX3Zx +j3RW5jNeouVlvtExyKZZhyMwuh8d59m4V65/rBkCgYBH+f40EvZOQ0v0jhef5CFo +lLZCgnB9kzwEpM5BihLpfdQuaWkhduW71s102i321UAbejtKwv69HnQXZpHG09Jl +g53i4CvZd77lCHx3+0Q1mxyxUtSGtbkAIAyr9xcVsTni2v2WtBrUcacsLzI7XxeV +HNo9QObLuTGTIM9EAjryiQKBgQCIZEBX56/jn6c3IFX5O+gH8OlxEXFyI+TAavq+ +Mnd7s7EGpXSclTP0fYAofSz0otkklZi9Iyh6Kv4cHOLV8klOtthfFyeVKJ5rvX+D +jv76miRlwBGBEQzABXIZ1oz4IK1xiYQUNSfSdA3sd5WYemEOlxHiSJrQ1qcyrBlJ +tOAzSQKBgQDjLQIZ0rmSAMUCGi7WlDle/pnf6sUEFUPWUqzHOJbJk/CZIIf1IT6o +Evep9eeP6QjiupusY/uSlgjjjj7yxZU5q3VV/0cQl6QhwxG9wqpa8x32wwaM1WBZ +rJgsuSjcEoluAqfX/bqRtumNtQ213DSADzSCpOetlJb+ARANk7YULg== +-----END RSA PRIVATE KEY----- diff --git a/vm-postfix-config-sender/tls_policy b/vagrant-shared/postfix-config-sender-tls_policy similarity index 100% rename from vm-postfix-config-sender/tls_policy rename to vagrant-shared/postfix-config-sender-tls_policy diff --git a/vm-postfix-config-sender/main.cf b/vagrant-shared/postfix-config-sender.cf similarity index 100% rename from vm-postfix-config-sender/main.cf rename to vagrant-shared/postfix-config-sender.cf diff --git a/vm-postfix-config-valid/main.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf similarity index 100% rename from vm-postfix-config-valid/main.cf rename to vagrant-shared/postfix-config-valid-example-recipient.cf diff --git a/vm-postfix-config-sender/dynamicmaps.cf b/vm-postfix-config-sender/dynamicmaps.cf deleted file mode 100644 index 1c48bdcde..000000000 --- a/vm-postfix-config-sender/dynamicmaps.cf +++ /dev/null @@ -1,6 +0,0 @@ -# Postfix dynamic maps configuration file. -# -#type location of .so file open function (mkmap func) -#==== ================================ ============= ============ -tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open -sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config-sender/master.cf b/vm-postfix-config-sender/master.cf deleted file mode 100644 index 3df29d8a9..000000000 --- a/vm-postfix-config-sender/master.cf +++ /dev/null @@ -1,113 +0,0 @@ -# -# Postfix master process configuration file. For details on the format -# of the file, see the master(5) manual page (command: "man 5 master"). -# -# Do not forget to execute "postfix reload" after editing this file. -# -# ========================================================================== -# service type private unpriv chroot wakeup maxproc command + args -# (yes) (yes) (yes) (never) (100) -# ========================================================================== -smtp inet n - - - - smtpd -#smtp inet n - - - 1 postscreen -#smtpd pass - - - - - smtpd -#dnsblog unix - - - - 0 dnsblog -#tlsproxy unix - - - - 0 tlsproxy -#submission inet n - - - - smtpd -# -o syslog_name=postfix/submission -# -o smtpd_tls_security_level=encrypt -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#smtps inet n - - - - smtpd -# -o syslog_name=postfix/smtps -# -o smtpd_tls_wrappermode=yes -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#628 inet n - - - - qmqpd -pickup fifo n - - 60 1 pickup -cleanup unix n - - - 0 cleanup -qmgr fifo n - n 300 1 qmgr -#qmgr fifo n - n 300 1 oqmgr -tlsmgr unix - - - 1000? 1 tlsmgr -rewrite unix - - - - - trivial-rewrite -bounce unix - - - - 0 bounce -defer unix - - - - 0 bounce -trace unix - - - - 0 bounce -verify unix - - - - 1 verify -flush unix n - - 1000? 0 flush -proxymap unix - - n - - proxymap -proxywrite unix - - n - 1 proxymap -smtp unix - - - - - smtp -relay unix - - - - - smtp -# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 -showq unix n - - - - showq -error unix - - - - - error -retry unix - - - - - error -discard unix - - - - - discard -local unix - n n - - local -virtual unix - n n - - virtual -lmtp unix - - - - - lmtp -anvil unix - - - - 1 anvil -scache unix - - - - 1 scache -# -# ==================================================================== -# Interfaces to non-Postfix software. Be sure to examine the manual -# pages of the non-Postfix software to find out what options it wants. -# -# Many of the following services use the Postfix pipe(8) delivery -# agent. See the pipe(8) man page for information about ${recipient} -# and other message envelope options. -# ==================================================================== -# -# maildrop. See the Postfix MAILDROP_README file for details. -# Also specify in main.cf: maildrop_destination_recipient_limit=1 -# -maildrop unix - n n - - pipe - flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} -# -# ==================================================================== -# -# Recent Cyrus versions can use the existing "lmtp" master.cf entry. -# -# Specify in cyrus.conf: -# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 -# -# Specify in main.cf one or more of the following: -# mailbox_transport = lmtp:inet:localhost -# virtual_transport = lmtp:inet:localhost -# -# ==================================================================== -# -# Cyrus 2.1.5 (Amos Gouaux) -# Also specify in main.cf: cyrus_destination_recipient_limit=1 -# -#cyrus unix - n n - - pipe -# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} -# -# ==================================================================== -# Old example of delivery via Cyrus. -# -#old-cyrus unix - n n - - pipe -# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} -# -# ==================================================================== -# -# See the Postfix UUCP_README file for configuration details. -# -uucp unix - n n - - pipe - flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) -# -# Other external delivery methods. -# -ifmail unix - n n - - pipe - flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) -bsmtp unix - n n - - pipe - flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient -scalemail-backend unix - n n - 2 pipe - flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} -mailman unix - n n - - pipe - flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py - ${nexthop} ${user} - diff --git a/vm-postfix-config-valid/dynamicmaps.cf b/vm-postfix-config-valid/dynamicmaps.cf deleted file mode 100644 index 1c48bdcde..000000000 --- a/vm-postfix-config-valid/dynamicmaps.cf +++ /dev/null @@ -1,6 +0,0 @@ -# Postfix dynamic maps configuration file. -# -#type location of .so file open function (mkmap func) -#==== ================================ ============= ============ -tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open -sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config-valid/master.cf b/vm-postfix-config-valid/master.cf deleted file mode 100644 index 3df29d8a9..000000000 --- a/vm-postfix-config-valid/master.cf +++ /dev/null @@ -1,113 +0,0 @@ -# -# Postfix master process configuration file. For details on the format -# of the file, see the master(5) manual page (command: "man 5 master"). -# -# Do not forget to execute "postfix reload" after editing this file. -# -# ========================================================================== -# service type private unpriv chroot wakeup maxproc command + args -# (yes) (yes) (yes) (never) (100) -# ========================================================================== -smtp inet n - - - - smtpd -#smtp inet n - - - 1 postscreen -#smtpd pass - - - - - smtpd -#dnsblog unix - - - - 0 dnsblog -#tlsproxy unix - - - - 0 tlsproxy -#submission inet n - - - - smtpd -# -o syslog_name=postfix/submission -# -o smtpd_tls_security_level=encrypt -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#smtps inet n - - - - smtpd -# -o syslog_name=postfix/smtps -# -o smtpd_tls_wrappermode=yes -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#628 inet n - - - - qmqpd -pickup fifo n - - 60 1 pickup -cleanup unix n - - - 0 cleanup -qmgr fifo n - n 300 1 qmgr -#qmgr fifo n - n 300 1 oqmgr -tlsmgr unix - - - 1000? 1 tlsmgr -rewrite unix - - - - - trivial-rewrite -bounce unix - - - - 0 bounce -defer unix - - - - 0 bounce -trace unix - - - - 0 bounce -verify unix - - - - 1 verify -flush unix n - - 1000? 0 flush -proxymap unix - - n - - proxymap -proxywrite unix - - n - 1 proxymap -smtp unix - - - - - smtp -relay unix - - - - - smtp -# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 -showq unix n - - - - showq -error unix - - - - - error -retry unix - - - - - error -discard unix - - - - - discard -local unix - n n - - local -virtual unix - n n - - virtual -lmtp unix - - - - - lmtp -anvil unix - - - - 1 anvil -scache unix - - - - 1 scache -# -# ==================================================================== -# Interfaces to non-Postfix software. Be sure to examine the manual -# pages of the non-Postfix software to find out what options it wants. -# -# Many of the following services use the Postfix pipe(8) delivery -# agent. See the pipe(8) man page for information about ${recipient} -# and other message envelope options. -# ==================================================================== -# -# maildrop. See the Postfix MAILDROP_README file for details. -# Also specify in main.cf: maildrop_destination_recipient_limit=1 -# -maildrop unix - n n - - pipe - flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} -# -# ==================================================================== -# -# Recent Cyrus versions can use the existing "lmtp" master.cf entry. -# -# Specify in cyrus.conf: -# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 -# -# Specify in main.cf one or more of the following: -# mailbox_transport = lmtp:inet:localhost -# virtual_transport = lmtp:inet:localhost -# -# ==================================================================== -# -# Cyrus 2.1.5 (Amos Gouaux) -# Also specify in main.cf: cyrus_destination_recipient_limit=1 -# -#cyrus unix - n n - - pipe -# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} -# -# ==================================================================== -# Old example of delivery via Cyrus. -# -#old-cyrus unix - n n - - pipe -# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} -# -# ==================================================================== -# -# See the Postfix UUCP_README file for configuration details. -# -uucp unix - n n - - pipe - flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) -# -# Other external delivery methods. -# -ifmail unix - n n - - pipe - flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) -bsmtp unix - n n - - pipe - flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient -scalemail-backend unix - n n - 2 pipe - flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} -mailman unix - n n - - pipe - flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py - ${nexthop} ${user} - diff --git a/vm-postfix-config-valid/tls_policy b/vm-postfix-config-valid/tls_policy deleted file mode 100644 index f8d6a4968..000000000 --- a/vm-postfix-config-valid/tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com encrypt protocols=TLSv1.1 From 46ce09d36d0b15113fd0ea1f61b4db842fb8e948 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 05:01:46 -0700 Subject: [PATCH 027/631] MTA config wrangling seems to work --- mta-config-generator.py | 91 ++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 33 deletions(-) mode change 100644 => 100755 mta-config-generator.py diff --git a/mta-config-generator.py b/mta-config-generator.py old mode 100644 new mode 100755 index 5d79423ef..1346d37dd --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -4,7 +4,7 @@ import string DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" -def parse_line(self, line_data): +def parse_line(line_data): "return the and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache @@ -12,7 +12,7 @@ def parse_line(self, line_data): left, sep, right = line.partition("=") if not sep: return None - return (left.strip(), right.strip()) + return (num, left.strip(), right.strip()) #def get_cf_values(lines, var): @@ -23,10 +23,11 @@ class MTAConfigGenerator: class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, stlse_config): + def __init__(self, stlse_config, fixup=False): + self.fixup = fixup MTAConfigGenerator.__init__(self, stlse_config) - this.postfix_cf_file = this.find_postfix_cf() - this.wrangle_existing_config() + self.postfix_cf_file = self.find_postfix_cf() + self.wrangle_existing_config() def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -35,53 +36,77 @@ class PostfixConfigGenerator(MTAConfigGenerator): acceptable = [ideal] + also_acceptable - l = [num,line for num,line in enumerate(cf) if line.startswith(var)] + l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] if not any(l): - this.additions.append("smtpd_use_tls = yes") + self.additions.append(var + " = " + ideal) else: - values = [right for left, right in map(parse_line, l)] + values = map(parse_line, l) if len(set(values)) > 1: - if this.fixup: - this.deletions.append( - raise ExistingConfigError, "Conflicting existing config values " + `l` - if values[0] != "yes": + if self.fixup: + #print "Scheduling deletions:" + `values` + conflicting_lines = [num for num,_var,val in values] + self.deletions.extend(conflicting_lines) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Conflicting existing config values " + `l` + val = values[0][2] + if val not in acceptable: + #print "Scheduling deletions:" + `values` + if self.fixup: + self.deletions.append(values[0][0]) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - def wrangle_existing_config(self, fixup=false): + def wrangle_existing_config(self): """ Try to ensure/mutate that the config file is in a sane state. Fixup means we'll delete existing lines if necessary to get there. """ - this.additions = [] - fn = find_postfix_cf() - raw_cf = open(fn).readlines() - this.cf = map(string.strip, raw_cf) - #this.cf = [line for line in cf if line and not line.startswith("#")] - this.fixup = fixup + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] # Check we're currently accepting inbound STARTTLS sensibly - this.ensure_cf_var("smtpd_use_tls", "yes", []) + self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction - this.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) # Maximum verbosity lets us collect failure information - this.ensure_cf_var("smtp_tls_loglevel", "1", []) + self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - this.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + self.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) - this.maybe_add_config_lines() + self.maybe_add_config_lines() def maybe_add_config_lines(self): - if not this.additions: + if not self.additions: return - this.additions[:0]=["","# New config lines added by STARTTLS Everywhere",""] - new_cf_lines = "\n".join(this.additions) - print "Adding to %s:" % fn + if self.fixup: + print "Deleting lines:", self.deletions + self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + new_cf_lines = "\n".join(self.additions) + print "Adding to %s:" % self.fn print new_cf_lines - if raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - new_cf = "".join(raw_cf) + sep + new_cf_lines - f = open(fn, "w").write(new_cf) - f.close() + if self.raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" + + self.new_cf = "" + for num, line in enumerate(self.raw_cf): + if self.fixup and num in self.deletions: + self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line + else: + self.new_cf += line + self.new_cf += sep + new_cf_lines + + print self.new_cf + f = open(self.fn, "w").write(self.new_cf) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return "/etc/postfix/main.cf" + +if __name__ == "__main__": + pcgen = PostfixConfigGenerator(None, fixup=True) From 0c4e33281184c489378758d70ccd0cec129c6293 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 11 Jun 2014 11:45:28 -0400 Subject: [PATCH 028/631] Set up test CA and valid signed cert by that CA. Also require valid cert for host 'valid'. --- vagrant-shared/postfix-config-sender-tls_policy | 2 +- vagrant-shared/postfix-config-sender.cf | 1 + vagrant-shared/postfix-config-valid-example-recipient.cf | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vagrant-shared/postfix-config-sender-tls_policy b/vagrant-shared/postfix-config-sender-tls_policy index af948c5e7..f4f1e80c5 100644 --- a/vagrant-shared/postfix-config-sender-tls_policy +++ b/vagrant-shared/postfix-config-sender-tls_policy @@ -1 +1 @@ -#valid-example-recipient.com encrypt protocols=TLSv1.1 +valid-example-recipient.com secure match=valid-example-recipient.com:.valid-example-recipient.com diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf index cf74d122f..6fc9435c6 100644 --- a/vagrant-shared/postfix-config-sender.cf +++ b/vagrant-shared/postfix-config-sender.cf @@ -43,3 +43,4 @@ smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy smtp_tls_loglevel = 1 smtp_tls_security_level = may +smtp_tls_CAfile = /etc/certificates/ca.crt diff --git a/vagrant-shared/postfix-config-valid-example-recipient.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf index a5f7ce575..1d08a20cd 100644 --- a/vagrant-shared/postfix-config-valid-example-recipient.cf +++ b/vagrant-shared/postfix-config-valid-example-recipient.cf @@ -18,8 +18,6 @@ append_dot_mydomain = no readme_directory = no # TLS parameters -smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key smtpd_use_tls=yes smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache @@ -44,3 +42,5 @@ smtpd_tls_received_header = yes #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy +smtpd_tls_cert_file=/etc/certificates/valid.crt +smtpd_tls_key_file=/etc/certificates/valid.key From 6e1bcfdb2a7ebe34511c5bc92265ceebe373b47f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:17:50 -0700 Subject: [PATCH 029/631] WIP implementing domain-wise TLS policies --- mta-config-generator.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/mta-config-generator.py b/mta-config-generator.py index 1346d37dd..1b98e7e96 100755 --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -2,7 +2,9 @@ import string -DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" + +DEFAULT_POLICY_FILE = "/etc/postfix/starttls_everywhere_policy" +POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): "return the and right hand sides of stripped, non-comment postfix config line" @@ -17,17 +19,18 @@ def parse_line(line_data): #def get_cf_values(lines, var): class MTAConfigGenerator: - def __init__(self, stlse_config): - self.c = stlse_config + def __init__(self, policy_config): + self.policy_config = policy_config class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, stlse_config, fixup=False): + def __init__(self, policy_config, fixup=False): self.fixup = fixup - MTAConfigGenerator.__init__(self, stlse_config) + MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() + self.set_domainwise_tls_policies() def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -77,7 +80,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - self.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + self.ensure_cf_var("smtp_tls_policy_maps", POLICY_CF_ENTRY, []) self.maybe_add_config_lines() @@ -87,7 +90,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): if self.fixup: print "Deleting lines:", self.deletions self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + new_cf_lines = "\n".join(self.additions) + "\n" print "Adding to %s:" % self.fn print new_cf_lines if self.raw_cf[-1][-1] == "\n": sep = "" @@ -108,5 +111,18 @@ class PostfixConfigGenerator(MTAConfigGenerator): "Search far and wide for the correct postfix configuration file" return "/etc/postfix/main.cf" + def set_domainwise_tls_policies(self): + self.policy_lines = [] + for domain, policy in self.policy_config.tls_policies: + entry = domain + " encrypt " + if "min-tls-version" in policy: + entry += " " + policy["min-tls-version"] + self.policy_lines.append(entry) + + f = open(DEFAULT_POLICY_FILE, "w") + f.write("\n".join(self.policy_lines)) + if __name__ == "__main__": - pcgen = PostfixConfigGenerator(None, fixup=True) + import config-parser + c = config-parser.Config() + pcgen = PostfixConfigGenerator(c, fixup=True) From eea1b0d8c5cf5367dcb91099ac7331443ad8a631 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:18:56 -0700 Subject: [PATCH 030/631] Switch naming conventions so that modules are importable :) It turns out that python won't import modules with hyphens in their names. It seems that CamelCase is most consistent with our Class naming. However feel free to do something different instead :) --- check-starttls.py => CheckSTARTTLS.py | 0 config-parser.py => ConfigParser.py | 0 mta-config-generator.py => MTAConfigGenerator.py | 0 ...-google-starttls-domains.py => ProcessGoogleSTARTTLSDomains.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename check-starttls.py => CheckSTARTTLS.py (100%) rename config-parser.py => ConfigParser.py (100%) rename mta-config-generator.py => MTAConfigGenerator.py (100%) rename process-google-starttls-domains.py => ProcessGoogleSTARTTLSDomains.py (100%) diff --git a/check-starttls.py b/CheckSTARTTLS.py similarity index 100% rename from check-starttls.py rename to CheckSTARTTLS.py diff --git a/config-parser.py b/ConfigParser.py similarity index 100% rename from config-parser.py rename to ConfigParser.py diff --git a/mta-config-generator.py b/MTAConfigGenerator.py similarity index 100% rename from mta-config-generator.py rename to MTAConfigGenerator.py diff --git a/process-google-starttls-domains.py b/ProcessGoogleSTARTTLSDomains.py similarity index 100% rename from process-google-starttls-domains.py rename to ProcessGoogleSTARTTLSDomains.py From 2540f1f1e8f924d50b629a682986d5973715a968 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:31:41 -0700 Subject: [PATCH 031/631] Writing to the domain-wise policy file actually works now. --- MTAConfigGenerator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 1b98e7e96..27a714361 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -113,16 +113,16 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for domain, policy in self.policy_config.tls_policies: - entry = domain + " encrypt " + for domain, policy in self.policy_config.tls_policies.items(): + entry = domain + " encrypt" if "min-tls-version" in policy: entry += " " + policy["min-tls-version"] self.policy_lines.append(entry) f = open(DEFAULT_POLICY_FILE, "w") - f.write("\n".join(self.policy_lines)) + f.write("\n".join(self.policy_lines) + "\n") if __name__ == "__main__": - import config-parser - c = config-parser.Config() + import ConfigParser + c = ConfigParser.Config() pcgen = PostfixConfigGenerator(c, fixup=True) From 182e9b29e4353a64a910a86edd6adb1d25976882 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:42:17 -0700 Subject: [PATCH 032/631] Trying to standardize JSON terms --- ConfigParser.py | 10 +++++----- MTAConfigGenerator.py | 7 +++++-- config.json | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index dbc244b25..aba4eb23f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -47,7 +47,7 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "tls-policies": + elif atr == "security-policies": self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: @@ -67,18 +67,18 @@ class Config: def check_tls_policy_domains(self, val): if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` + raise TypeError, "security-policies should be a dict" + `val` for domain, policies in val.items(): try: assert type(domain) == unicode d = str(domain) # convert from unicode except: - raise TypeError, "tls-policy domain not a string" + `domain` + raise TypeError, "security-policy domain not a string" + `domain` if not d.startswith("*."): - raise ValueError, "tls-policy domains must start with *.; try *."+d + raise ValueError, "security-policy domains must start with *.; try *."+d d = d.partition("*.")[2] if not looks_like_a_domain(d): - raise ValueError, "tls-policy for something that a domain? " + d + raise ValueError, "security-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 27a714361..bee518e05 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -105,7 +105,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += sep + new_cf_lines print self.new_cf - f = open(self.fn, "w").write(self.new_cf) + f = open(self.fn, "w") + f.write(self.new_cf) + f.close() def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -121,8 +123,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): f = open(DEFAULT_POLICY_FILE, "w") f.write("\n".join(self.policy_lines) + "\n") + f.close() if __name__ == "__main__": import ConfigParser - c = ConfigParser.Config() + c = ConfigParser.Config("starttls-everywhere.json") pcgen = PostfixConfigGenerator(c, fixup=True) diff --git a/config.json b/config.json index 1a9034545..8d38696ff 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", - "tls-policies": { + "security-policies": { "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } From 3712a4539907a05a98e289cfc48a6c62adccd686 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:48:43 -0700 Subject: [PATCH 033/631] Further (and different, and better) standardisation --- ConfigParser.py | 10 +++++----- README.md | 2 +- starttls-everywhere.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index aba4eb23f..dbc244b25 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -47,7 +47,7 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "security-policies": + elif atr == "tls-policies": self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: @@ -67,18 +67,18 @@ class Config: def check_tls_policy_domains(self, val): if type(val) != dict: - raise TypeError, "security-policies should be a dict" + `val` + raise TypeError, "tls-policies should be a dict" + `val` for domain, policies in val.items(): try: assert type(domain) == unicode d = str(domain) # convert from unicode except: - raise TypeError, "security-policy domain not a string" + `domain` + raise TypeError, "tls-policy domain not a string" + `domain` if not d.startswith("*."): - raise ValueError, "security-policy domains must start with *.; try *."+d + raise ValueError, "tls-policy domains must start with *.; try *."+d d = d.partition("*.")[2] if not looks_like_a_domain(d): - raise ValueError, "security-policy for something that a domain? " + d + raise ValueError, "tls-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/README.md b/README.md index 79e0ed30d..2f0a410a1 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co // "timestamp": 1401414363, : also acceptable "author": "Electronic Frontier Foundation https://eff.org", "expires": "2014-06-06T14:30:16+00:00", - "security-policies": { + "tls-policies": { // These match on the MX domain. "*.yahoodns.net": { "require-valid-certificate": true, diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 4c91866d5..d0e656186 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,5 @@ { - "mx-domains": { + "tls-policies": { "*.mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true @@ -49,7 +49,7 @@ "require-tls": true } }, - "address-domains": { + "acceptable-mxs": { "wp.pl": { "accept-mx-domains": [ "*.wp.pl" From 34cba3accfe2da92846cdf6a427724f60999a96a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:51:56 -0700 Subject: [PATCH 034/631] Now successfully parsing the larger policy set --- ConfigParser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfigParser.py b/ConfigParser.py index dbc244b25..8bc7e708f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -55,7 +55,7 @@ class Config: self.tls_policies[domain] = {} # being here enforces TLS at all for policy, value in policies.items(): if policy == "min-tls-version": - reasonable = ["TLS", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) From a2ee328bc0dc3e384f9d1ecf6567aa16afbeb7ef Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 10:32:52 -0700 Subject: [PATCH 035/631] Paramaterise "/etc/postfix" --- MTAConfigGenerator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index bee518e05..3b69ba37a 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,9 +1,10 @@ #!/usr/bin/env python import string +import os.path - -DEFAULT_POLICY_FILE = "/etc/postfix/starttls_everywhere_policy" +POSTFIX_DIR = "/etc/postfix" +DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): @@ -111,7 +112,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return "/etc/postfix/main.cf" + return os.path.join(POSTFIX_DIR,"main.cf") def set_domainwise_tls_policies(self): self.policy_lines = [] From e99abfacfd40c4f7c1f404c7225d5d3ad4792ef8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:22:30 -0700 Subject: [PATCH 036/631] Include the demo example with the real stuff, temporarily --- starttls-everywhere.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d0e656186..4b47ea0c3 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,9 @@ { "tls-policies": { + "*.valid-example-recipient.com": { + "min-tls-version": "TLSv1.1", + "force-tls" : true + }, "*.mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true @@ -50,6 +54,9 @@ } }, "acceptable-mxs": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "*.valid-example-recipient.com" ] + }, "wp.pl": { "accept-mx-domains": [ "*.wp.pl" From 3d7b53daf19c4f47e34e1fb74b57367462968142 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:24:05 -0700 Subject: [PATCH 037/631] Tune verbosity; reload postfix conf if we can --- MTAConfigGenerator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3b69ba37a..d78fa55e5 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,14 +1,16 @@ #!/usr/bin/env python import string -import os.path +import os, os.path POSTFIX_DIR = "/etc/postfix" DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): - "return the and right hand sides of stripped, non-comment postfix config line" + """ + return the (line number, left hand side, right hand side) + of a stripped postfix config line""" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache num,line = line_data @@ -32,6 +34,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -105,7 +108,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += line self.new_cf += sep + new_cf_lines - print self.new_cf + #print self.new_cf f = open(self.fn, "w") f.write(self.new_cf) f.close() From 02abaf57bd0cd67cabb8e1cdaefe1dabbb269776 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:24:51 -0700 Subject: [PATCH 038/631] Remove stray conf entry from README --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 2f0a410a1..7a08c3ecf 100644 --- a/README.md +++ b/README.md @@ -86,9 +86,6 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "eff.org": { "accept-mx-domains": ["*.eff.org"] } - "*.yahoodns.net": { - "require-valid-certificate": true, - } } } From 9ce047980a991129de1bfc6b56a8bd133f798942 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:40:17 -0400 Subject: [PATCH 039/631] Add .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..94c6f7089 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.* +*.orig From 9cd71642fb13688565a2b245eac06089eaad8dcb Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:50:39 -0400 Subject: [PATCH 040/631] Fix italicization boundaries --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2f0a410a1..6717f8b72 100644 --- a/README.md +++ b/README.md @@ -109,19 +109,19 @@ Config-generator should attempt to fetch the configuration file daily and transf The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. -Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. +Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains_ list must correspond to an exactly matching field in the _mx-domains_ config section. The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. **mx-domains** -The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain _can be a subdomain of any other _mx-domain _in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. +The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain_can be a subdomain of any other _mx-domain_in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ -_Require-valid-certificate _defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: +_Require-valid-certificate_defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: 1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. 2. The certificate is unexpired. From 499f6c2fad62fe4d793cfee727576f399012a1b2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:53:02 -0400 Subject: [PATCH 041/631] Add comment to ProcessgoogleSTARTTLSDomains.py --- ProcessGoogleSTARTTLSDomains.py | 11 ++++++++++- config.json | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ProcessGoogleSTARTTLSDomains.py b/ProcessGoogleSTARTTLSDomains.py index 0a0201875..abb2b3495 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/ProcessGoogleSTARTTLSDomains.py @@ -1,4 +1,13 @@ #!/usr/bin/python +""" +Process Google's TLS delivery data from +https://www.google.com/transparencyreport/saferemail/data/?hl=en +to look for outbound domains that can negotiate an encrypted +connection >99% of the time. + +Usage: + ./ProcessGoogleSTARTTLSDomains.py google-starttls-domains.csv +""" import csv import codecs import sys @@ -14,5 +23,5 @@ for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in pass for address_suffix, fraction_encrypted in d.iteritems(): - if min(fraction_encrypted) >= 0.50: + if min(fraction_encrypted) >= 0.99: print min(fraction_encrypted), address_suffix diff --git a/config.json b/config.json index 8d38696ff..1a9034545 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", - "security-policies": { + "tls-policies": { "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } From 43d457aa771e663a7c15886fbc81db081947d9cf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 13:18:20 -0400 Subject: [PATCH 042/631] Typo cleanup in MTAConfigGenerator --- ConfigParser.py | 1 - MTAConfigGenerator.py | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 8bc7e708f..206132293 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -36,7 +36,6 @@ class Config: f = open(cfg_file_name) self.cfg = json.loads(f.read()) for atr, val in self.cfg.items(): - #print atr,val # Verify each attribute of the structure if atr.startswith("comment"): continue diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3b69ba37a..d5ec334ec 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -3,12 +3,12 @@ import string import os.path -POSTFIX_DIR = "/etc/postfix" +POSTFIX_DIR = "postfix-copy" DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): - "return the and right hand sides of stripped, non-comment postfix config line" + "return the left and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache num,line = line_data @@ -17,8 +17,6 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) -#def get_cf_values(lines, var): - class MTAConfigGenerator: def __init__(self, policy_config): self.policy_config = policy_config @@ -36,8 +34,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. """ - + values; if not, set it to the ideal value. + """ acceptable = [ideal] + also_acceptable l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] From 3cf61a54b7ce08389a85f5282fd86ca958b121b7 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 13 Jun 2014 13:57:05 -0400 Subject: [PATCH 043/631] Add alternatives section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6717f8b72..6e3d4212b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. +## Alternatives + +Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of a DANE solution easier. + ## Detailed design Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: From 9da4f93ae587156d4f9be5829a32a4201e5f1588 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 16 Jun 2014 02:46:46 -0700 Subject: [PATCH 044/631] Changes for live demo --- Vagrantfile | 1 + starttls-everywhere.json | 1 - vagrant-bootstrap.sh | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 3ce93c917..b7153a7b8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,6 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| valid.vm.hostname = "valid-example-recipient.com" end config.vm.synced_folder "vagrant-shared", "/vagrant" + config.vm.synced_folder "vagrant-shared/starttls-everywhere", "/vagrant/starttls-everywhere" config.vm.provision :shell, path: "vagrant-bootstrap.sh" config.vm.provider "virtualbox" do |vb| diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 4b47ea0c3..df3ff41ab 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,7 +1,6 @@ { "tls-policies": { "*.valid-example-recipient.com": { - "min-tls-version": "TLSv1.1", "force-tls" : true }, "*.mx.aol.com": { diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 13593a0cc..82622edea 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -23,9 +23,9 @@ if [ "`hostname`" = "sender" ]; then (while sleep 10; do echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com done) & - ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy + #ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy fi -ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf -ln -sf "/vagrant/certificates" /etc/certificates +#ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf +#ln -sf "/vagrant/certificates" /etc/certificates postfix reload From 51f90ffafb3e74406e24eb63e7f359ed0483b461 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 16 Jun 2014 18:26:56 +0000 Subject: [PATCH 045/631] Write policies based on address domain, not stripped mx-domain --- ConfigParser.py | 9 ++------- MTAConfigGenerator.py | 13 +++++++++---- starttls-everywhere.json | 5 ----- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 206132293..072c7e8ff 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -59,10 +59,10 @@ class Config: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) elif atr == "acceptable-mxs": + self.acceptable_mxs = val pass else: - sys.stderr.write("Uknown attribute: " + `atr` + "\n") - print self.tls_policies + sys.stderr.write("Unknown attribute: " + `atr` + "\n") def check_tls_policy_domains(self, val): if type(val) != dict: @@ -73,11 +73,6 @@ class Config: d = str(domain) # convert from unicode except: raise TypeError, "tls-policy domain not a string" + `domain` - if not d.startswith("*."): - raise ValueError, "tls-policy domains must start with *.; try *."+d - d = d.partition("*.")[2] - if not looks_like_a_domain(d): - raise ValueError, "tls-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index d5ec334ec..859095acc 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -114,10 +114,15 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for domain, policy in self.policy_config.tls_policies.items(): - entry = domain + " encrypt" - if "min-tls-version" in policy: - entry += " " + policy["min-tls-version"] + for address_domain, properties in self.policy_config.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + mx_policy = self.policy_config.tls_policies[mx_domain] + entry = address_domain + " encrypt" + if "min-tls-version" in mx_policy: + entry += " " + mx_policy["min-tls-version"] self.policy_lines.append(entry) f = open(DEFAULT_POLICY_FILE, "w") diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d0e656186..a98a2293f 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -50,11 +50,6 @@ } }, "acceptable-mxs": { - "wp.pl": { - "accept-mx-domains": [ - "*.wp.pl" - ] - }, "yahoo.co.uk": { "accept-mx-domains": [ "*.yahoodns.net" From 2eba47a716956bdcfc99c466b617ecc2e5b9c32a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 16 Jun 2014 14:23:33 -0700 Subject: [PATCH 046/631] Verify more of the policy language --- ConfigParser.py | 28 ++++++++++++++++++++++++---- MTAConfigGenerator.py | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 206132293..541ff3f30 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -35,6 +35,8 @@ class Config: def __init__(self, cfg_file_name = "config.json"): f = open(cfg_file_name) self.cfg = json.loads(f.read()) + self.tls_policies = {} + self.mx_map = {} for atr, val in self.cfg.items(): # Verify each attribute of the structure if atr.startswith("comment"): @@ -47,21 +49,39 @@ class Config: elif atr == "expires": self.expires = parse_timestamp(val) elif atr == "tls-policies": - self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, value in policies.items(): - if policy == "min-tls-version": + for policy, v in policies.items(): + value = lower(str(v)) + if policy == "require-tls": + if value in ("true", "1", "yes"): + self.tls_policies[domain]["required"] = True + elif value in ("false", "0", "no"): + self.tls_policies[domain]["required"] = False + else: + raise ValueError, "Unknown require-tls value " + `value` + elif policy == "min-tls-version": reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) + elif policy == "enforce-mode": + if value == "enforce": + self.tls_policies[domain]["enforce"] = True + elif value == "log-only": + self.tls_policies[domain]["enforce"] = False + else: + raise ValueError, "Not a known enoforcement policy " + `value` elif atr == "acceptable-mxs": - pass + for domain, mxball in acceptable_mx: + pass else: sys.stderr.write("Uknown attribute: " + `atr` + "\n") + # XXX is it ever permissible to have a domain with an acceptable-mx + # that does not point to a TLS security policy? If not, check/warn/fail + # here print self.tls_policies def check_tls_policy_domains(self, val): diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 875624e57..1d322c8e9 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -78,7 +78,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Check we're currently accepting inbound STARTTLS sensibly self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map From 67ee3b048876d1e27cdc21d29139afc1bb812b68 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 18 Jun 2014 12:32:17 -0400 Subject: [PATCH 047/631] Config format change - don't use * as it's misleading. --- config.json | 4 +-- starttls-everywhere.json | 58 ++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config.json b/config.json index 1a9034545..05fc237bf 100644 --- a/config.json +++ b/config.json @@ -4,13 +4,13 @@ "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", "tls-policies": { - "*.valid-example-recipient.com": { + ".valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } }, "acceptable-mxs": { "valid-example-recipient.com": { - "accept-mx-domains": [ "*.valid-example-recipient.com" ] + "accept-mx-domains": [ ".valid-example-recipient.com" ] } } diff --git a/starttls-everywhere.json b/starttls-everywhere.json index a98a2293f..d00859d8c 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,50 +1,50 @@ { "tls-policies": { - "*.mx.aol.com": { + ".mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.psmtp.com": { + ".psmtp.com": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.ukr.net": { + ".ukr.net": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.interia.pl": { + ".interia.pl": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.gmx.net": { + ".gmx.net": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.web.de": { + ".web.de": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.marktplaats.nl": { + ".marktplaats.nl": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.wp.pl": { + ".wp.pl": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.yahoodns.net": { + ".yahoodns.net": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.t-online.de": { + ".t-online.de": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.rambler.ru": { + ".rambler.ru": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.t.facebook.com": { + ".t.facebook.com": { "min-tls-version": "TLSv1", "require-tls": true } @@ -52,87 +52,87 @@ "acceptable-mxs": { "yahoo.co.uk": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "rocketmail.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "web.de": { "accept-mx-domains": [ - "*.web.de" + ".web.de" ] }, "sbcglobal.net": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "aol.com": { "accept-mx-domains": [ - "*.mx.aol.com" + ".mx.aol.com" ] }, "facebook.com": { "accept-mx-domains": [ - "*.t.facebook.com" + ".t.facebook.com" ] }, "sompo-japan.co.jp": { "accept-mx-domains": [ - "*.psmtp.com" + ".psmtp.com" ] }, "salesforce.com": { "accept-mx-domains": [ - "*.psmtp.com" + ".psmtp.com" ] }, "rambler.ru": { "accept-mx-domains": [ - "*.rambler.ru" + ".rambler.ru" ] }, "t-online.de": { "accept-mx-domains": [ - "*.t-online.de" + ".t-online.de" ] }, "gmx.net": { "accept-mx-domains": [ - "*.gmx.net" + ".gmx.net" ] }, "gmx.de": { "accept-mx-domains": [ - "*.gmx.net" + ".gmx.net" ] }, "ukr.net": { "accept-mx-domains": [ - "*.ukr.net" + ".ukr.net" ] }, "rogers.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "ymail.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "marktplaats.nl": { "accept-mx-domains": [ - "*.marktplaats.nl" + ".marktplaats.nl" ] }, "interia.pl": { "accept-mx-domains": [ - "*.interia.pl" + ".interia.pl" ] } } From 3df343495e2ac81d5ceb501b43240a68cbbca54c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 18 Jun 2014 10:58:53 -0700 Subject: [PATCH 048/631] Various fixups --- ConfigParser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 85b704d93..4fb69b6b2 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -54,7 +54,7 @@ class Config: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all for policy, v in policies.items(): - value = lower(str(v)) + value = str(v).lower() if policy == "require-tls": if value in ("true", "1", "yes"): self.tls_policies[domain]["required"] = True @@ -64,6 +64,7 @@ class Config: raise ValueError, "Unknown require-tls value " + `value` elif policy == "min-tls-version": reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + reasonable = map(string.lower, reasonable) if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) @@ -76,7 +77,7 @@ class Config: raise ValueError, "Not a known enoforcement policy " + `value` elif atr == "acceptable-mxs": self.acceptable_mxs = val - for domain, mxball in selg.acceptable_mxs: + for domain, mxball in self.acceptable_mxs.items(): pass else: sys.stderr.write("Unknown attribute: " + `atr` + "\n") From 51980e212f7ea46fc584ebc8e7938f337d2e666c Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 18 Jun 2014 17:50:41 -0400 Subject: [PATCH 049/631] First pass at logs analysis --- ConfigParser.py | 19 +++++++++++++++++-- PostfixLogSummary.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 PostfixLogSummary.py diff --git a/ConfigParser.py b/ConfigParser.py index 072c7e8ff..69779548f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -4,7 +4,7 @@ import sys import json from datetime import datetime import string - +import collections def parse_timestamp(ts): try: @@ -48,7 +48,7 @@ class Config: self.expires = parse_timestamp(val) elif atr == "tls-policies": self.tls_policies = {} - for domain,policies in self.check_tls_policy_domains(val): + for domain, policies in self.check_tls_policy_domains(val): if type(policies) != dict: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all @@ -60,10 +60,25 @@ class Config: self.tls_policies[domain]["min-tls-version"] = str(value) elif atr == "acceptable-mxs": self.acceptable_mxs = val + self.mx_domain_to_address_domains = collections.defaultdict(set) + for address_domain, properties in self.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + self.mx_domain_to_address_domains[mx_domain].add(address_domain) pass else: sys.stderr.write("Unknown attribute: " + `atr` + "\n") + def get_address_domains(self, mx_hostname): + for mx_domain, address_domains in self.mx_domain_to_address_domains.items(): + # TODO: write this better + if (mx_hostname.find(mx_domain) > 0 and + mx_hostname.find(mx_domain) == len(mx_hostname) - len(mx_domain)): + return address_domains + return None + def check_tls_policy_domains(self, val): if type(val) != dict: raise TypeError, "tls-policies should be a dict" + `val` diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py new file mode 100755 index 000000000..e13c230aa --- /dev/null +++ b/PostfixLogSummary.py @@ -0,0 +1,33 @@ +#!/usr/bin/python2.7 +import re +import sys +import collections + +import ConfigParser + +def get_counts(input, config): + counts = collections.defaultdict(lambda: collections.defaultdict(int)) + r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + for line in sys.stdin: + result = r.search(line) + if result: + validation = result.group(1) + mx_hostname = result.group(2) + address_domains = config.get_address_domains(mx_hostname) + if address_domains: + for d in address_domains: + counts[d][validation] += 1 + counts[d]["all"] += 1 + return counts + +def print_summary(counts): + for mx_hostname, validations in counts.items(): + for validation, validation_count in validations.items(): + if validation == "all": + continue + print mx_hostname, validation, validation_count / validations["all"] + +if __name__ == "__main__": + config = ConfigParser.Config("starttls-everywhere.json") + counts = get_counts(sys.stdin, config) + print_summary(counts) From 3de8c2a65152a9985b04f11a64c6568c93a70dbf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 17:08:30 -0400 Subject: [PATCH 050/631] Do a better job finding address domain from mx doman. --- ConfigParser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 69779548f..eff4d3557 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -72,11 +72,11 @@ class Config: sys.stderr.write("Unknown attribute: " + `atr` + "\n") def get_address_domains(self, mx_hostname): - for mx_domain, address_domains in self.mx_domain_to_address_domains.items(): - # TODO: write this better - if (mx_hostname.find(mx_domain) > 0 and - mx_hostname.find(mx_domain) == len(mx_hostname) - len(mx_domain)): - return address_domains + labels = mx_hostname.split(".") + for n in range(1, len(labels)): + parent = "." + ".".join(labels[n:]) + if parent in self.mx_domain_to_address_domains: + return self.mx_domain_to_address_domains[parent] return None def check_tls_policy_domains(self, val): From 1c3c69aaad9477988e48f1a3c5080f1d1b201e2d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 17:08:49 -0400 Subject: [PATCH 051/631] Allow parameters to MTAConfigGenerator. --- MTAConfigGenerator.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 859095acc..15bc017ab 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,16 +1,17 @@ #!/usr/bin/env python +import sys import string import os.path -POSTFIX_DIR = "postfix-copy" -DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") -POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE - def parse_line(line_data): - "return the left and right hand sides of stripped, non-comment postfix config line" - # lines are like: - # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + Return the left and right hand sides of stripped, non-comment postfix + config line. + + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ num,line = line_data left, sep, right = line.partition("=") if not sep: @@ -24,12 +25,15 @@ class MTAConfigGenerator: class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, fixup=False): + def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + print "Configuration complete. Now run `sudo service postfix reload'." def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -59,7 +63,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.additions.append(var + " = " + ideal) else: raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - + def wrangle_existing_config(self): """ Try to ensure/mutate that the config file is in a sane state. @@ -79,7 +83,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - self.ensure_cf_var("smtp_tls_policy_maps", POLICY_CF_ENTRY, []) + policy_cf_entry = "texthash:" + self.policy_file + + self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.maybe_add_config_lines() @@ -110,7 +116,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return os.path.join(POSTFIX_DIR,"main.cf") + return os.path.join(self.postfix_dir, "main.cf") def set_domainwise_tls_policies(self): self.policy_lines = [] @@ -122,14 +128,18 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.tls_policies[mx_domain] entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: - entry += " " + mx_policy["min-tls-version"] + entry += " protocols=" + mx_policy["min-tls-version"] self.policy_lines.append(entry) - f = open(DEFAULT_POLICY_FILE, "w") + f = open(self.policy_file, "w") f.write("\n".join(self.policy_lines) + "\n") f.close() if __name__ == "__main__": import ConfigParser - c = ConfigParser.Config("starttls-everywhere.json") - pcgen = PostfixConfigGenerator(c, fixup=True) + if len(sys.argv) != 3: + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" + sys.exit(1) + c = ConfigParser.Config(sys.argv[1]) + postfix_dir = sys.argv[2] + pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) From a85fad98c0be1c5494090486c476608c4951032f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:22:08 -0400 Subject: [PATCH 052/631] Restore default config --- vagrant-shared/postfix-config-sender.cf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf index 6fc9435c6..b9f265058 100644 --- a/vagrant-shared/postfix-config-sender.cf +++ b/vagrant-shared/postfix-config-sender.cf @@ -37,10 +37,3 @@ mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all - -#STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy - -smtp_tls_loglevel = 1 -smtp_tls_security_level = may -smtp_tls_CAfile = /etc/certificates/ca.crt From 127d49e837d2fa8713bf1932ccecad11607b58e3 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:22:32 -0400 Subject: [PATCH 053/631] Manually add a couple known-good domains. These were skipped because in the Google data they are represented as, e.g. 'gmail.{..}'. --- golden-domains.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/golden-domains.txt b/golden-domains.txt index 9a0c95948..d340f23e4 100644 --- a/golden-domains.txt +++ b/golden-domains.txt @@ -20,3 +20,5 @@ facebook.com craigslist.org bigpond.com aol.com +gmail.com +yahoo.com From dd4f9d35ae8fd21a8593cd232177f6550b2a63e8 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:23:13 -0400 Subject: [PATCH 054/631] Improve checker and starttls-everywhere.json. Now we alphabetize keys on output for more useful diffs. --- CheckSTARTTLS.py | 23 ++-- starttls-everywhere.json | 225 ++++++++++++++++++--------------------- 2 files changed, 122 insertions(+), 126 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index cd8dfea42..205a4a779 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -10,6 +10,9 @@ import json import dns.resolver from M2Crypto import X509 +from publicsuffix import PublicSuffixList + +public_suffix_list = PublicSuffixList() def mkdirp(path): try: @@ -71,6 +74,8 @@ def valid_cert(filename): if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: return False try: + # The file contains both the leaf cert and any intermediates, so we pass it + # as both the cert to validate and as the "untrusted" chain. output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ -untrusted "%s" \ "%s" @@ -87,10 +92,11 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) + new_names = map(lambda n: public_suffix_list.get_public_suffix(n), new_names) names.update(new_names) - names.add(filename.rstrip(".")) if len(names) >= 1: - return common_suffix(names) + # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. + return names.pop() else: return "" @@ -134,6 +140,7 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): + print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: @@ -143,7 +150,7 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" if len(sys.argv) == 1: - print("Please pass at least one mail domain as an argument") + print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = { "address-domains": { @@ -151,14 +158,16 @@ if __name__ == '__main__': "mx-domains": { } } - for domain in sys.argv[1:]: - collect(domain) + for domain in open(sys.argv[1]).readlines(): + domain = domain.strip() + if not os.path.exists(domain): + collect(domain) if len(os.listdir(domain)) == 0: continue suffix = check_certs(domain) min_version = min_tls_version(domain) if suffix != "": - suffix_match = "*." + suffix + suffix_match = "." + suffix config["address-domains"][domain] = { "accept-mx-domains": [suffix_match] } @@ -167,4 +176,4 @@ if __name__ == '__main__': "min-tls-version": min_version } - print json.dumps(config, indent=2) + print json.dumps(config, indent=2, sort_keys=True) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d00859d8c..5dd487f9a 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,128 +1,18 @@ { - "tls-policies": { - ".mx.aol.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".psmtp.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".ukr.net": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".interia.pl": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".gmx.net": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".web.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".marktplaats.nl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".wp.pl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yahoodns.net": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".t-online.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".rambler.ru": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".t.facebook.com": { - "min-tls-version": "TLSv1", - "require-tls": true - } - }, - "acceptable-mxs": { - "yahoo.co.uk": { + "address-domains": { + "craigslist.org": { "accept-mx-domains": [ - ".yahoodns.net" + ".craigslist.org" ] }, - "rocketmail.com": { + "gmail.com": { "accept-mx-domains": [ - ".yahoodns.net" + ".google.com" ] }, - "web.de": { + "interia.pl": { "accept-mx-domains": [ - ".web.de" - ] - }, - "sbcglobal.net": { - "accept-mx-domains": [ - ".yahoodns.net" - ] - }, - "aol.com": { - "accept-mx-domains": [ - ".mx.aol.com" - ] - }, - "facebook.com": { - "accept-mx-domains": [ - ".t.facebook.com" - ] - }, - "sompo-japan.co.jp": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "salesforce.com": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "rambler.ru": { - "accept-mx-domains": [ - ".rambler.ru" - ] - }, - "t-online.de": { - "accept-mx-domains": [ - ".t-online.de" - ] - }, - "gmx.net": { - "accept-mx-domains": [ - ".gmx.net" - ] - }, - "gmx.de": { - "accept-mx-domains": [ - ".gmx.net" - ] - }, - "ukr.net": { - "accept-mx-domains": [ - ".ukr.net" - ] - }, - "rogers.com": { - "accept-mx-domains": [ - ".yahoodns.net" - ] - }, - "ymail.com": { - "accept-mx-domains": [ - ".yahoodns.net" + ".interia.pl" ] }, "marktplaats.nl": { @@ -130,10 +20,107 @@ ".marktplaats.nl" ] }, - "interia.pl": { + "rambler.ru": { "accept-mx-domains": [ - ".interia.pl" + ".rambler.ru" ] + }, + "rocketmail.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "rogers.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "salesforce.com": { + "accept-mx-domains": [ + ".psmtp.com" + ] + }, + "sbcglobal.net": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "sompo-japan.co.jp": { + "accept-mx-domains": [ + ".psmtp.com" + ] + }, + "t-online.de": { + "accept-mx-domains": [ + ".t-online.de" + ] + }, + "wp.pl": { + "accept-mx-domains": [ + ".wp.pl" + ] + }, + "yahoo.co.uk": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "yahoo.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "yandex.ru": { + "accept-mx-domains": [ + ".yandex.ru" + ] + }, + "ymail.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + } + }, + "mx-domains": { + ".craigslist.org": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".google.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".interia.pl": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".marktplaats.nl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".psmtp.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".rambler.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".t-online.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".wp.pl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".yahoo.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".yandex.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true } } } From 6a40e1964b98ba19709055600278807718bac899 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 15:05:10 -0400 Subject: [PATCH 055/631] Notice if there are no "Trusted" entries. --- PostfixLogSummary.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index e13c230aa..daec93db0 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -6,18 +6,24 @@ import collections import ConfigParser def get_counts(input, config): + seen_trusted = False + counts = collections.defaultdict(lambda: collections.defaultdict(int)) r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) if result: validation = result.group(1) - mx_hostname = result.group(2) + mx_hostname = result.group(2).lower() + if validation == "Trusted" or validation == "Verified": + seen_trusted = True address_domains = config.get_address_domains(mx_hostname) if address_domains: for d in address_domains: counts[d][validation] += 1 counts[d]["all"] += 1 + if not seen_trusted: + print "Didn't see any trusted connections. Need to install some certs?" return counts def print_summary(counts): @@ -25,7 +31,7 @@ def print_summary(counts): for validation, validation_count in validations.items(): if validation == "all": continue - print mx_hostname, validation, validation_count / validations["all"] + print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": config = ConfigParser.Config("starttls-everywhere.json") From bdd4d01dc73f6c91af07531304584348dbf0d22d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 16:57:51 -0400 Subject: [PATCH 056/631] Update and sort golden-domains.txt, commit google-starttls-domains.csv --- golden-domains.txt | 50 +- google-starttls-domains.csv | 6734 +++++++++++++++++++++++++++++++++++ 2 files changed, 6763 insertions(+), 21 deletions(-) create mode 100644 google-starttls-domains.csv diff --git a/golden-domains.txt b/golden-domains.txt index d340f23e4..bca6d2f89 100644 --- a/golden-domains.txt +++ b/golden-domains.txt @@ -1,24 +1,32 @@ -ymail.com -yandex.ru -yahoo.co.uk -wp.pl -web.de -vtext.com -ukr.net -t-online.de -sompo-japan.co.jp -sbcglobal.net -salesforce.com -rogers.com -rocketmail.com -rambler.ru -marktplaats.nl -interia.pl -gmx.net -gmx.de -facebook.com -craigslist.org -bigpond.com +163.com aol.com +bigpond.com +comcast.net +craigslist.org +facebook.com gmail.com +gmx.de +hotmail.com +icloud.com +live.com +mac.com +me.com +msn.com +naver.com +outlook.com +qq.com +rocketmail.com +rogers.com +salesforce.com +sbcglobal.net +shaw.ca +sympatico.ca +t-online.de +ukr.net +vtext.com +web.de +wp.pl yahoo.com +yahoogroups.com +yandex.ru +ymail.com diff --git a/google-starttls-domains.csv b/google-starttls-domains.csv new file mode 100644 index 000000000..5d6a00f17 --- /dev/null +++ b/google-starttls-domains.csv @@ -0,0 +1,6734 @@ +Address Suffix,Hostname Suffix,Direction,UN M.49 Region Code,Region Name,Fraction Encrypted +0101.co.jp,0101.co.jp,inbound,001,World,0 +04auto.biz,01auto.biz,inbound,001,World,0 +0bz.biz,hmts.jp,inbound,001,World,0 +1-day.co.nz,1-day.co.nz,inbound,001,World,0 +104.com.tw,104.com.tw,inbound,001,World,0.091112 +1105info.com,1105info.com,inbound,001,World,0 +1111.com.tw,1111.com.tw,inbound,001,World,0 +123.com.tw,123.com.tw,inbound,001,World,0 +12manage.com,netarrest.com,inbound,001,World,0.99998 +160by2.us,160by2.us,inbound,001,World,0 +160by2inbox.com,160by2inbox.com,inbound,001,World,0 +160by2invite.com,160by2invite.com,inbound,001,World,0 +160by2mail.com,160by2mail.com,inbound,001,World,0 +163.com,163.com,inbound,001,World,0.711306 +163.com,netease.com,outbound,001,World,1 +17life.com.tw,17life.com.tw,inbound,001,World,0 +1800flowersinc.com,1800flowersinc.com,inbound,001,World,0 +1800petmeds.com,1800petmeds.com,inbound,001,World,0.00599 +1lejend.com,asumeru.com,inbound,001,World,0 +1lejend.com,asumeru001.com,inbound,001,World,0 +1sale.com,1sale.com,inbound,001,World,0 +1v1y.com,euromsg.net,inbound,001,World,0 +2touchbase.com,infimail.com,inbound,001,World,0 +33go.com.tw,33go.com.tw,inbound,001,World,0 +3suisses.be,3suisses.be,inbound,001,World,0 +3suisses.fr,3suisses.fr,inbound,001,World,0 +4shared.com,4shared.com,inbound,001,World,0.999946 +4wheelparts.com,4wheelparts.com,inbound,001,World,0 +518.com.tw,518.com.tw,inbound,001,World,0 +6pm.com,6pm.com,inbound,001,World,1 +6pm.com,zappos.com,inbound,001,World,0.782581 +7net.com.tw,7net.com.tw,inbound,001,World,0 +99acres.com,99acres.com,inbound,001,World,0 +9dot9digital.in,emce2.in,inbound,001,World,0 +a8.net,a8.net,inbound,001,World,0 +aaa.com,nextjump.com,inbound,001,World,0 +aaas-science.org,aaas-science.org,inbound,001,World,0 +aafes.com,aafes.com,inbound,001,World,0 +aanotifier.nl,aanotifier.nl,inbound,001,World,0 +aarp.org,aarp.org,inbound,001,World,0.006249 +ab0.jp,altovision.co.jp,inbound,001,World,0 +abercrombie-email.com,abercrombie-email.com,inbound,001,World,0 +abercrombiekids-email.com,abercrombie-email.com,inbound,001,World,0 +about.com,about.com,inbound,001,World,2.4e-05 +about.com,sailthru.com,inbound,001,World,0 +academy-enews.com,academy-enews.com,inbound,001,World,0 +accenture.com,outlook.com,inbound,001,World,1 +accountonline.com,accountonline.com,inbound,001,World,0.348991 +acehelpfulemails.com,teradatadmc.com,inbound,001,World,0 +acemserv.com,acemserv.com,inbound,001,World,0 +activesafelist.com,zoothost.com,inbound,001,World,0 +activetrail.com,atmailsvr.net,inbound,001,World,0 +activetrail.com,mymarketing.co.il,inbound,001,World,0 +actorsaccess.com,nonfatmedia.com,inbound,001,World,0 +adchiever.com,kinder-rash-marketing.com,inbound,001,World,0 +adidas.com,neolane.net,inbound,001,World,0 +adidasusnews.com,adidasusnews.com,inbound,001,World,0 +adityabirla.com,adityabirla.com,inbound,001,World,0.006468 +adjockeys.com,thomas-j-brown.com,inbound,001,World,0 +admail.hu,sanomaonline.hu,inbound,001,World,0 +admastersafelist.com,zoothost.com,inbound,001,World,0 +adminforfree.com,adminforfree.com,inbound,001,World,1 +adminforfree.net,adminforfree.com,inbound,001,World,1 +administrativejobinsider.com,administrativejobinsider.com,inbound,001,World,0 +adobe.com,obsmtp.com,inbound,001,World,0.999986 +adobesystems.com,adobesystems.com,inbound,001,World,0 +adorama.com,adorama.com,inbound,001,World,0 +adoreme.com,exacttarget.com,inbound,001,World,0 +adp.com,adp.com,inbound,001,World,1 +adpirate.net,thomas-j-brown.com,inbound,001,World,0 +adsender.us,adsender.us,inbound,001,World,0 +adsolutionline.com,adsolutionline.com,inbound,001,World,0 +adtpulse.com,adtpulse.com,inbound,001,World,0 +adultfriendfinder.com,friendfinder.com,inbound,001,World,0 +advanceauto.com,bigfootinteractive.com,inbound,001,World,0 +advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,001,World,0 +adverts.ie,adverts.ie,inbound,001,World,1 +advfn.com,advfn.com,inbound,001,World,0.635382 +ae.com,ae.com,inbound,001,World,0 +aexp.com,aexp.com,inbound,001,World,1 +af.mil,af.mil,inbound,001,World,0.996166 +affairalert.com,iverificationsystems.com,inbound,001,World,0 +agnitas.de,agnitas.de,inbound,001,World,0.999277 +agoda-emails.com,agoda-emails.com,inbound,001,World,0 +agora.co.il,1host.co.il,inbound,001,World,0 +agorafinancial.com,agorafinancial.com,inbound,001,World,0 +agrupemonos.cl,agrupemonos.cl,inbound,001,World,1 +airbnb.com,airbnb.com,inbound,001,World,0.867975 +airbrake.io,mailgun.net,inbound,001,World,1 +airfarewatchdog.com,smartertravelmedia.com,inbound,001,World,0.011961 +airliquide.com,airliquide.com,inbound,001,World,1 +airmiles.ca,bigfootinteractive.com,inbound,001,World,0 +airtel.com,airtel.in,inbound,001,World,0.064377 +akcijatau.lt,akcijatau.lt,inbound,001,World,0 +alarm.com,alarm.com,inbound,001,World,0 +alarmnet.com,alarmnet.com,inbound,001,World,0 +alaskaair.com,alaskaair.com,inbound,001,World,0 +albertsonsemail.com,email4-mywebgrocer.com,inbound,001,World,0 +alerteimmo.com,alerteimmo.com,inbound,001,World,0 +alertid.com,alertid.com,inbound,001,World,0 +alertsindia.in,alertsindia.in,inbound,001,World,1 +alibaba.com,alibaba.com,inbound,001,World,0 +alice.it,alice.it,inbound,001,World,0 +alice.it,aliceposta.it,outbound,001,World,0 +aliexpress.com,alibaba.com,inbound,001,World,0 +alinea.fr,bp06.net,inbound,001,World,0 +alipay.com,alipay.com,inbound,001,World,1 +allegro.pl,allegro.pl,inbound,001,World,0 +allegrogroup.ua,allegrogroup.ua,inbound,001,World,0 +allegroup.hu,allegroup.hu,inbound,001,World,0 +allheart.com,allheart.com,inbound,001,World,0 +alljob.co.il,alljob.co.il,inbound,001,World,0 +allmodern.com,allmodern.com,inbound,001,World,0 +allout.org,allout.org,inbound,001,World,1 +allrecipes.com,allrecipes.com,inbound,001,World,0 +allsaints.com,allsaints.com,inbound,001,World,0 +allstate.com,rsys1.com,inbound,001,World,0 +alm.com,sailthru.com,inbound,001,World,0 +alumniclass.com,alumniclass.com,inbound,001,World,0 +alumniconnections.com,alumniconnections.com,inbound,001,World,0 +alza.cz,alza.cz,inbound,001,World,0.039449 +alza.sk,alza.cz,inbound,001,World,0.027725 +ama-assn.org,elabs10.com,inbound,001,World,0 +amadeus.com,amadeus.net,inbound,001,World,0 +amazon.{...},amazon.{...},inbound,001,World,0.020886 +amazon.{...},amazonses.com,inbound,001,World,0.999971 +amazon.{...},postini.com,inbound,001,World,0.7325 +amazon.{...},yahoo.{...},inbound,001,World,0.995995 +amazonses.com,amazonses.com,inbound,001,World,0.997919 +amazonses.com,postini.com,inbound,001,World,0.843221 +amctheatres.com,amctheatres.com,inbound,001,World,0 +americanas.com,americanas.com,inbound,001,World,0 +americanbar.org,abanet.org,inbound,001,World,0.00574 +americanexpress.com,americanexpress.com,inbound,001,World,0.000688 +americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,001,World,0 +amubm.com,amubm.com,inbound,001,World,1 +amwayemail.com,mailrouter.net,inbound,001,World,1 +ana.co.jp,ana.co.jp,inbound,001,World,0.534416 +ancestry.com,ancestry.com,inbound,001,World,0 +andrewchristian.com,emv8.com,inbound,001,World,0 +angelbroking.in,infimail.com,inbound,001,World,0 +anghami.com,mailgun.net,inbound,001,World,1 +angieslist.com,angieslist.com,inbound,001,World,0.014064 +anntaylor.com,anntaylor.com,inbound,001,World,0 +anpasia.com,anpasia.com,inbound,001,World,0 +anpdm.com,anpdm.com,inbound,001,World,6e-06 +anthropologie.com,freepeople.com,inbound,001,World,0 +aol.com,aol.com,inbound,001,World,0.999529 +aol.com,aol.com,outbound,001,World,0.999992 +aol.com,sailthru.com,inbound,001,World,0 +aol.net,aol.com,inbound,001,World,1 +apache.org,apache.org,inbound,001,World,0 +apnacomplex.com,apnacomplex.com,inbound,001,World,0.999981 +apple.com,apple.com,inbound,001,World,0.974422 +apply-4-jobs.com,apply-4-jobs.com,inbound,001,World,0 +aprovaconcursos.com.br,eadunicid.com.br,inbound,001,World,0 +aptmail.in,mailurja.com,inbound,001,World,0 +ara.cat,ara.cat,inbound,001,World,0.997709 +arcamax.com,arcamax.com,inbound,001,World,1.4e-05 +argos.co.uk,argos.co.uk,inbound,001,World,1 +argos.co.uk,exacttarget.com,inbound,001,World,1 +aritzia.com,aritzia.com,inbound,001,World,0 +armaniexchange.com,bronto.com,inbound,001,World,0 +artists-hub.com,artists-hub.com,inbound,001,World,0 +artscow.com,dyxnet.com,inbound,001,World,0.00062 +aruba.it,aruba.it,inbound,001,World,0.055666 +asadventure.com,asadventure.com,inbound,001,World,0 +asana.com,asana.com,inbound,001,World,1 +asda.com,ec-cluster.com,inbound,001,World,0 +ashampoo.com,ashampoo.com,inbound,001,World,1 +ashleymadison.com,ashleymadison.com,inbound,001,World,1 +ask.fm,ask.fm,inbound,001,World,1e-05 +askmen.com,askmen.com,inbound,001,World,0 +asos.com,asos.com,inbound,001,World,0 +assembla.com,assembla.com,inbound,001,World,1 +astrocenter.com,center.com,inbound,001,World,0 +astrology.com,astrology.com,inbound,001,World,0 +astrology.com,hsnlmailsvc.com,inbound,001,World,0 +astrology.com,webstakes.com,inbound,001,World,0 +astrology.com,wsafmailsvc.com,inbound,001,World,0 +asus.com,asus.com,inbound,001,World,0 +aswatson.com,emarsys.net,inbound,001,World,0 +athleta.com,athleta.com,inbound,001,World,0 +atlassian.net,uc-inf.net,inbound,001,World,1 +atrapalo.cl,atrapalo.com,inbound,001,World,0 +atrapalo.com,atrapalo.com,inbound,001,World,0 +att-mail.com,att-mail.com,inbound,001,World,2.2e-05 +att-mail.com,att.com,inbound,001,World,0.999966 +att.net,att.net,outbound,001,World,0.204629 +att.net,mycingular.net,inbound,001,World,0.00021 +att.net,yahoo.{...},inbound,001,World,0.99997 +auctionzip-email.com,email-auctionholdings.com,inbound,001,World,0 +auinmeio.com.br,fnac.com.br,inbound,001,World,0 +australiagsm.net,australiagsm.net,inbound,001,World,0 +authorize.net,authorize.net,inbound,001,World,0 +authorize.net,visa.com,inbound,001,World,0.993558 +autoloop.us,loop28.com,inbound,001,World,0 +autoreply.com,autoreply.com,inbound,001,World,0 +avaaz.org,avaaz.org,inbound,001,World,0 +avalanchesafelist.com,zoothost.com,inbound,001,World,0 +avast.com,avast.com,inbound,001,World,0.007867 +aveda.com,esteelauder.com,inbound,001,World,0 +avenue.com,avenue.com,inbound,001,World,0 +avg.com,avg.com,inbound,001,World,0.015703 +avira.com,avira.com,inbound,001,World,0.042528 +avito.ru,avito.ru,inbound,001,World,0.002677 +avomail.com,avomail.com,inbound,001,World,0 +avon.com,email-avonglobal.com,inbound,001,World,0 +avon.com,postdirect.com,inbound,001,World,0 +aweber.com,aweber.com,inbound,001,World,3e-06 +ayi.com,ayi.com,inbound,001,World,0 +b2b-mail.net,b2b-mail.net,inbound,001,World,0 +b2b-mail.net,contact-list.net,inbound,001,World,0 +babycenter.com,rsys3.com,inbound,001,World,0 +babyoye.com,babyoye.com,inbound,001,World,0.011564 +backcountry.com,backcountry.com,inbound,001,World,0 +backlog.jp,backlog.jp,inbound,001,World,0 +badoo.com,monopost.com,inbound,001,World,1 +bagitgetitmailer.in,emce2.in,inbound,001,World,0 +baligam.co.il,baligam.co.il,inbound,001,World,1 +balsamik.fr,balsamik.fr,inbound,001,World,0 +banamex.com,citi.com,inbound,001,World,0.999958 +banamex.com,ibrands.es,inbound,001,World,0 +bananarepublic.com,bananarepublic.com,inbound,001,World,3.48869418874254e-07 +bancoahorrofamsa.com,avantel.net.mx,inbound,001,World,1 +bancochile.cl,bancochile.cl,inbound,001,World,0.999504 +bancofalabella.com,bancofalabella.com,inbound,001,World,0 +bancomer.com,postini.com,inbound,001,World,5.9e-05 +bancomercorreo.com,bancomercorreo.com,inbound,001,World,0 +bandsintown.com,bandsintown.com,inbound,001,World,1 +banesco.com,banesco.com,inbound,001,World,0 +bankofamerica.com,bankofamerica.com,inbound,001,World,0.97133 +banorte.com,gfnorte.com.mx,inbound,001,World,0.994999 +barclaycard.co.uk,barclays.co.uk,inbound,001,World,0 +barclaycardus.com,bigfootinteractive.com,inbound,001,World,0 +barenecessities.com,barenecessities.com,inbound,001,World,0 +barleyment.ca,barleyment.ca,inbound,001,World,0 +barneys.com,barneys.com,inbound,001,World,0 +baseballsavings.com,baseballsavings.com,inbound,001,World,0 +basecamp.com,basecamp.com,inbound,001,World,1 +basecamphq.com,basecamphq.com,inbound,001,World,1 +baskinrobbins.com,baskinrobbins.com,inbound,001,World,0 +basspronews.com,basspronews.com,inbound,001,World,0 +bathandbodyworks.com,bathandbodyworks.com,inbound,001,World,0 +baublebar.com,baublebar.com,inbound,001,World,0.011219 +baycrews.co.jp,webcas.net,inbound,001,World,0 +bayt.com,bayt.com,inbound,001,World,2e-06 +bazarchic-invitations.com,bazarchic-emstech.com,inbound,001,World,0 +bbvacompass.com,postini.com,inbound,001,World,0.996402 +bcbg.com,bcbg.com,inbound,001,World,0 +bci.cl,bci.cl,inbound,001,World,0.999963 +bcp.com.pe,bcp.com.pe,inbound,001,World,1 +be2.com,nmp1.net,inbound,001,World,0 +beamtele.com,beamtele.com,inbound,001,World,0 +beanfun.com,beanfun.com,inbound,001,World,1 +beatport-email.com,beatport-email.com,inbound,001,World,0 +beautylish.com,beautylish.com,inbound,001,World,1 +bebe.com,ed10.com,inbound,001,World,0 +befrugal.com,befrugal.com,inbound,001,World,0.050462 +belkemail.com,belkemail.com,inbound,001,World,0 +bellsouth.net,att.net,outbound,001,World,0 +bellsouth.net,yahoo.{...},inbound,001,World,0.999967 +belluna.net,belluna.net,inbound,001,World,0 +benihana-news.com,benihana-news.com,inbound,001,World,0 +bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,001,World,0 +bespokeoffers.co.uk,chtah.net,inbound,001,World,0 +bestbuy.ca,bestbuy.ca,inbound,001,World,0 +bestbuy.com,bestbuy.com,inbound,001,World,0.003289 +bestdealsforyou.in,elabs5.com,inbound,001,World,0 +beta.lt,mailersend3.com,inbound,001,World,0 +betrend.com,betrend.com,inbound,001,World,0 +bevmo.com,bevmo.com,inbound,001,World,0.000967 +beyondtherack.com,beyondtherack.com,inbound,001,World,0 +bharatmatrimony.com,bharatmatrimony.com,inbound,001,World,1 +bhcosmetics.com,bronto.com,inbound,001,World,0 +bhg.com,meredith.com,inbound,001,World,0 +bigfishgames.com,bigfishgames.com,inbound,001,World,0 +biglion.ru,biglion.ru,inbound,001,World,0.999775 +biglist.com,biglist.com,inbound,001,World,0 +biglots.com,biglots.com,inbound,001,World,0.00029 +bigmailsender.com,bigmailsender.com,inbound,001,World,0 +bigpond.com,bigpond.com,inbound,001,World,0 +bigpond.com,bigpond.com,outbound,001,World,1 +bigtent.com,carezen.net,inbound,001,World,0 +bioagri.com.br,postini.com,inbound,001,World,0.991765 +biomedcentral.com,emv5.com,inbound,001,World,0 +bionexo.com,bionexo.com.br,inbound,001,World,0.999594 +birthdayalarm.com,monkeyinferno.net,inbound,001,World,0 +bitbucket.org,bitbucket.org,inbound,001,World,0 +bitlysupport.com,mailgun.info,inbound,001,World,1 +bitlysupport.com,mailgun.us,inbound,001,World,1 +bitslane.email,bitslane.email,inbound,001,World,0 +bitstatement.org,bitstatement.org,inbound,001,World,1 +bizjournals.com,bizjournals.com,inbound,001,World,0 +bizmailtoday.com,bizmailtoday.com,inbound,001,World,0 +bjs.com,bjs.com,inbound,001,World,0 +bjsrestaurants.com,bjsrestaurants.com,inbound,001,World,0 +bk.ru,mail.ru,inbound,001,World,0.992498 +blablacar.com,blablacar.com,inbound,001,World,1 +blackberry.com,blackberry.com,inbound,001,World,0 +blackboard.com,blackboard.com,inbound,001,World,0.998206 +blackboard.com,notification.com,inbound,001,World,0 +blackpeoplemeet.com,blackpeoplemeet.com,inbound,001,World,0 +blayn.jp,bserver.jp,inbound,001,World,0 +blinkboxmusic.com,mediagraft.com,inbound,001,World,1 +blissworld.com,lstrk.net,inbound,001,World,1 +blizzard.com,battle.net,inbound,001,World,0.11977 +bloglovin.com,bloglovin.com,inbound,001,World,0.000154 +blogtrottr.com,blogtrottr.com,inbound,001,World,0 +bloomberg.com,bloomberg.com,inbound,001,World,0.00501 +bloomberg.net,bloomberg.net,inbound,001,World,1 +bloomingdales.com,bloomingdales.com,inbound,001,World,0 +bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,001,World,0 +blue-compass.com,blue-compass.com,inbound,001,World,0 +bluediamondhost3.com,web-hosting.com,inbound,001,World,1 +bluehornet.com,bluehornet.com,inbound,001,World,0 +bluehost.com,bluehost.com,inbound,001,World,0.000943 +bluehost.com,hostmonster.com,inbound,001,World,0 +bluehost.com,unifiedlayer.com,inbound,001,World,1.6e-05 +bluenile.com,bluenile.com,inbound,001,World,0 +blueshellgames.com,blueshellgames.com,inbound,001,World,0 +bluestatedigital.com,bluestatedigital.com,inbound,001,World,0 +bluestonemx.com,bluestonemx.com,inbound,001,World,1 +bm05.net,bm05.net,inbound,001,World,0 +bm23.com,bronto.com,inbound,001,World,0 +bm324.com,bronto.com,inbound,001,World,0 +bmdeda99.com,bmdeda99.com,inbound,001,World,0 +bme.jp,bserver.jp,inbound,001,World,0 +bmnt.jp,bmnt.jp,inbound,001,World,0 +bmsend.com,bmsend.com,inbound,001,World,0 +bn.com,bn.com,inbound,001,World,0 +bncollegemail.com,bncollegemail.com,inbound,001,World,0 +bnetmail.com,bnetmail.com,inbound,001,World,0 +bol.com.br,bol.com.br,inbound,001,World,0 +bol.com.br,bol.com.br,outbound,001,World,0 +boletinrenuevo.com,boletinrenuevo.com,inbound,001,World,0 +bolsfr.fr,colt.net,inbound,001,World,0 +bomnegocio.com,bomnegocio.com,inbound,001,World,0.687113 +bonobos.com,bronto.com,inbound,001,World,0 +bonuszbrigad.hu,bonuszbrigad.hu,inbound,001,World,0 +boohooemail.com,smartfocusdigital.net,inbound,001,World,0 +bookbub.com,bookbub.com,inbound,001,World,1 +booking.com,booking.com,inbound,001,World,1 +bookingbuddy.com,smartertravelmedia.com,inbound,001,World,0.000486 +bookmyshow.com,eccluster.com,inbound,001,World,0 +bookoffonline.co.jp,bookoffonline.co.jp,inbound,001,World,0 +boomtownroi.com,boomtownroi.com,inbound,001,World,0 +boots.com,boots.com,inbound,001,World,0 +boscovs.com,boscovs.com,inbound,001,World,0 +bostonproper.com,bostonproper.com,inbound,001,World,0 +bouncemanager.it,musvc.com,inbound,001,World,0.362026 +boutiquesecret.com,chtah.net,inbound,001,World,0 +box.com,box.com,inbound,001,World,0.955607 +br.com,cmailsys.com,inbound,001,World,0 +bradfordexchange.com,bradfordexchange.com,inbound,001,World,0 +bradsdeals.com,bradsdeals.com,inbound,001,World,0 +brandalley.com,brandalley.com,inbound,001,World,0 +brands4friends.de,emv5.com,inbound,001,World,0 +brands4friends.jp,webcas.net,inbound,001,World,0 +brandsfever.com,mailgun.net,inbound,001,World,1 +brandsvillage.net,brandsvillage.net,inbound,001,World,0 +brassring.com,brassring.com,inbound,001,World,0.999989 +briantracyintl.com,briantracyintl.com,inbound,001,World,0 +brierleycrm.com,brierleycrm.com,inbound,001,World,0 +brijj.com,brijj.com,inbound,001,World,0 +brincltd.com,brincltd.com,inbound,001,World,0 +bronto.com,bronto.com,inbound,001,World,0 +brooksbrothers.com,brooksbrothers.com,inbound,001,World,0 +bsf01.com,bsftransmit33.com,inbound,001,World,0 +bt.com,bt.com,inbound,001,World,0.409404 +btinternet.com,cpcloud.co.uk,inbound,001,World,0 +btinternet.com,cpcloud.co.uk,outbound,001,World,0 +btinternet.com,yahoo.{...},inbound,001,World,0.99998 +budgettravel.com,email-budgettravel.com,inbound,001,World,0 +buffalo.edu,buffalo.edu,inbound,001,World,0.001059 +bumeran.com,bumeran.com,inbound,001,World,0 +burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,001,World,0 +burton.co.uk,burton.co.uk,inbound,001,World,0 +buscojobs.com,amazonaws.com,inbound,001,World,0 +buy123.com.tw,buy123.com.tw,inbound,001,World,1 +buyinvite.com.au,buyinvite.com.au,inbound,001,World,0 +buyma.com,buyma.com,inbound,001,World,0 +bv.com.br,bv.com.br,inbound,001,World,0 +bweeble.com,adlabsinc.com,inbound,001,World,0 +byway.it,byway.it,inbound,001,World,0 +bzm.mobi,nmsrv.com,inbound,001,World,1 +c21stores.com,c21stores.com,inbound,001,World,0 +ca.gov,ca.gov,inbound,001,World,0.694242 +cabelas.com,cabelas.com,inbound,001,World,0 +cabestan.com,cab07.net,inbound,001,World,0 +cadremploi.fr,cadremploi.fr,inbound,001,World,0 +cafepress.com,cafepress.com,inbound,001,World,0.000778 +caixa.gov.br,caixa.gov.br,inbound,001,World,0 +californiajobdepartment.com,californiajobdepartment.com,inbound,001,World,0 +californiapsychicsemail.com,californiapsychicsemail.com,inbound,001,World,0 +callcommand.com,callcommand.com,inbound,001,World,0 +calottery.com,calottery.com,inbound,001,World,0.999517 +cam2life.com,hinet.net,inbound,001,World,0 +camel.com,rjrsignup.com,inbound,001,World,0 +camsonline.com,camsonline.com,inbound,001,World,0.136261 +canadiantire.ca,canadiantire.ca,inbound,001,World,0 +canadianvisaexpert.net,canadianvisaexpert.net,inbound,001,World,0 +canalplus.es,canalplus.es,inbound,001,World,0 +cancer.org,delivery.net,inbound,001,World,0 +capillary.co.in,capillary.co.in,inbound,001,World,1 +capitalone.com,bigfootinteractive.com,inbound,001,World,0 +capitalone360.com,ingdirect.com,inbound,001,World,0 +capitaloneemail.com,capitaloneemail.com,inbound,001,World,0 +cardsys.at,cardsys.at,inbound,001,World,1 +care.com,care.com,inbound,001,World,0 +care2.com,care2.com,inbound,001,World,0 +career-hub.net,career-hub.net,inbound,001,World,1 +careerage.com,careerage.com,inbound,001,World,0 +careerbuilder-email.com,careerbuilder-email.com,inbound,001,World,0 +careerbuilder.com,careerbuilder.com,inbound,001,World,0.000449 +careerflash.net,careerflash.net,inbound,001,World,1 +careers24.com,careers24.com,inbound,001,World,0 +careesma.in,careesma.in,inbound,001,World,0 +carmamail.com,carmamail.com,inbound,001,World,0 +carnivalfunmail.com,carnivalfunmail.com,inbound,001,World,0 +carolsdaughter.com,carolsdaughter.com,inbound,001,World,0 +carrefour.fr,carrefour.fr,inbound,001,World,0 +carters.com,carters.com,inbound,001,World,0.003931 +cartrade.com,cartrade.com,inbound,001,World,0.006389 +carwale.com,carwale.com,inbound,001,World,0 +casasbahia.com.br,casasbahia.com.br,inbound,001,World,0 +case.edu,cwru.edu,inbound,001,World,0.999942 +caseyresearch.com,caseyresearch.com,inbound,001,World,1 +castingnetworks.com,castingnetworks.com,inbound,001,World,0.000102 +catchoftheday.com.au,inxserver.de,inbound,001,World,1 +catchyfreebies.net,mmsend53.com,inbound,001,World,0 +catererglobal.com,madgexjb.com,inbound,001,World,0 +caterermail.com,totaljobsmail.co.uk,inbound,001,World,0 +cathkidston.com,cathkidston.co.uk,inbound,001,World,0 +causes.com,causes.com,inbound,001,World,1 +cb2.com,cb2.com,inbound,001,World,0 +cbsig.net,cbsig.net,inbound,001,World,1 +ccavenue.com,avenues.info,inbound,001,World,1 +ccbchurch.com,ccbchurch.com,inbound,001,World,1 +cccampaigns.com,emv5.com,inbound,001,World,0 +cccampaigns.com,emv8.com,inbound,001,World,0 +cccampaigns.net,01net.com,inbound,001,World,0 +cccampaigns.net,cccampaigns.net,inbound,001,World,0 +cccampaigns.net,emv4.net,inbound,001,World,0 +cccampaigns.net,emv9.net,inbound,001,World,0 +ccialerts.com,ccialerts.com,inbound,001,World,0 +ccmbg.com,benchmark.fr,inbound,001,World,0 +ccs.com,footlocker.com,inbound,001,World,2.5e-05 +cdongroup.com,cdongroup.com,inbound,001,World,0.000147 +cecentertainment.com,cecentertainment.com,inbound,001,World,0 +celebritycruises.com,celebritycruises.com,inbound,001,World,0 +cenlat.com,cenlat.com,inbound,001,World,0.064459 +centaur.co.uk,centaur.co.uk,inbound,001,World,0 +centauro.com.br,centauro.com.br,inbound,001,World,0 +centerparcs.co.uk,ec-cluster.com,inbound,001,World,0 +cerberusapp.com,cerberusapp.com,inbound,001,World,1 +cfmailer.com,elabs11.com,inbound,001,World,0 +cfmvmail.com,cfmvmail.com,inbound,001,World,0 +chabad.org,chabad.org,inbound,001,World,0 +champssports.com,footlocker.com,inbound,001,World,0.000131 +chance.com,data-hotel.net,inbound,001,World,0 +change.org,change.org,inbound,001,World,1 +channel4.com,channel4.com,inbound,001,World,0.000613 +charlestyrwhitt.com,charlestyrwhitt.com,inbound,001,World,0 +charter.net,charter.net,inbound,001,World,0 +charter.net,charter.net,outbound,001,World,0 +chase.com,bigfootinteractive.com,inbound,001,World,0 +chase.com,jpmchase.com,inbound,001,World,0.999999 +chatcitynotifications.com,chatcitynotifications.com,inbound,001,World,0 +chaturbate.com,chaturbate.com,inbound,001,World,1 +cheapairmailer.com,cheapairmailer.com,inbound,001,World,0 +cheaperthandirt.com,cheaperthandirt.com,inbound,001,World,2.2e-05 +cheapflights.co.uk,cheapflights.co.uk,inbound,001,World,0 +cheapflights.com,cheapflights.com,inbound,001,World,0 +check.me,check.me,inbound,001,World,0 +cheekylovers.com,ropot.net,inbound,001,World,0 +chefscatalog.com,chefscatalog.com,inbound,001,World,0 +chelseafc.com,chelseafc.com,inbound,001,World,0.003566 +chemistdirect.co.uk,ec-cluster.com,inbound,001,World,0 +chemistry.com,chemistry.com,inbound,001,World,0 +chess.com,chess.com,inbound,001,World,1 +chiangcn.com,chiangcn.com,inbound,001,World,0 +chicagotribune.com,latimes.com,inbound,001,World,0 +chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,001,World,0 +chicos.com,chicos.com,inbound,001,World,0.000156 +childrensplace.com,childrensplace.com,inbound,001,World,0 +chinatrust.com.tw,chinatrust.com.tw,inbound,001,World,0.001272 +chopra.com,chopra.com,inbound,001,World,1 +christianbook.com,christianbook.com,inbound,001,World,1 +christianmingle.com,christianmingle.com,inbound,001,World,0 +christianmingle.com,postdirect.com,inbound,001,World,0 +chtah.com,chtah.net,inbound,001,World,0 +chtah.net,chtah.net,inbound,001,World,0 +cincghq.com,searchhomesingta.com,inbound,001,World,1 +cinesa.es,cccampaigns.com,inbound,001,World,0 +cipherzone.com,infimail.com,inbound,001,World,0 +cir.ca,cir.ca,inbound,001,World,1 +circleofmomsmail.com,circleofmomsmail.com,inbound,001,World,0 +citi.com,citi.com,inbound,001,World,0.999941 +citibank.com,bigfootinteractive.com,inbound,001,World,0 +citibank.com,citi.com,inbound,001,World,0.999997 +citicorp.com,citi.com,inbound,001,World,0.999999 +citruslane.com,citruslane.com,inbound,001,World,5e-06 +citybrands.hu,webinform.hu,inbound,001,World,1 +cityheaven.net,cityheaven.net,inbound,001,World,0 +ck.com,ck.com,inbound,001,World,0 +clarisonic.com,clarisonic.com,inbound,001,World,0 +clarks.com,clarks.com,inbound,001,World,0 +classmates.com,classmates.com,inbound,001,World,0 +clickdimensions.com,clickdimensions.com,inbound,001,World,0 +clickexperts.net,clickexperts.net,inbound,001,World,0 +clickmailer.jp,clickmailer.jp,inbound,001,World,9e-05 +clickon.com.ar,clickon.com.ar,inbound,001,World,0 +clickon.com.br,clickon.com.br,inbound,001,World,0 +clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,001,World,0 +clicplan.com,dmdelivery.com,inbound,001,World,0 +climber.com,climber.com,inbound,001,World,0 +clinique.com,esteelauder.com,inbound,001,World,0 +clubcupon.com.ar,clubcupon.com.ar,inbound,001,World,0 +cmail1.com,createsend.com,inbound,001,World,0 +cmail2.com,createsend.com,inbound,001,World,0 +cmm01.com,coremotivesmarketing.com,inbound,001,World,0 +cmrfalabella.com,cmrfalabella.com,inbound,001,World,0 +coach.com,delivery.net,inbound,001,World,0 +cobone.com,emarsys.net,inbound,001,World,0 +cocacola.co.jp,cocacola.co.jp,inbound,001,World,0 +codebreak.info,codebreak.info,inbound,001,World,1 +codeproject.com,codeproject.com,inbound,001,World,0 +coldwatercreek.com,coldwatercreek.com,inbound,001,World,0 +collectionsetc.com,collectionsetc.com,inbound,001,World,0 +columbia.edu,columbia.edu,inbound,001,World,0.762355 +combzmail.jp,combzmail.jp,inbound,001,World,0 +comcast.net,comcast.net,inbound,001,World,0.888399 +comcast.net,comcast.net,outbound,001,World,0.999999 +comenity.net,alldata.net,inbound,001,World,1 +comenity.net,bigfootinteractive.com,inbound,001,World,0 +commonfloor.com,commonfloor.com,inbound,001,World,1 +communicatoremail.com,communicatoremail.com,inbound,001,World,0 +communitymatrimony.com,communitymatrimony.com,inbound,001,World,1 +compute.internal,amazonaws.com,inbound,001,World,0.858276 +computerworld.com,computerworld.com,inbound,001,World,0 +comunicacaodemkt.com,locaweb.com.br,inbound,001,World,0 +confirmedoptin.com,confirmedoptin.com,inbound,001,World,0 +confirmsignup.com,mmsend53.com,inbound,001,World,0 +conrepmail.com,conrepmail.com,inbound,001,World,0 +constantcontact.com,confirmedcc.com,inbound,001,World,0 +constantcontact.com,constantcontact.com,inbound,001,World,5.3e-05 +constantcontact.com,postini.com,inbound,001,World,0.078144 +constantcontact.com,yahoo.{...},inbound,001,World,0.999724 +contact-darty.com,mm-send.com,inbound,001,World,0 +contactlab.it,contactlab.it,inbound,001,World,0 +containerstore.com,containerstore.com,inbound,001,World,0 +continente.pt,1-hostingservice.com,inbound,001,World,0 +converse.com,converse.com,inbound,001,World,1 +convio.net,convio.net,inbound,001,World,0 +cookingchanneltv.com,cookingchanneltv.com,inbound,001,World,0 +cookpad.com,cookpad.com,inbound,001,World,0 +copernica.nl,picsrv.net,inbound,001,World,0.011507 +copernica.nl,vicinity.nl,inbound,001,World,0.011753 +coppel.com,coppel.com,inbound,001,World,0 +coremotivesmarketing.com,coremotivesmarketing.com,inbound,001,World,0 +cornell.edu,cornell.edu,inbound,001,World,0.195104 +corporateperks.com,nextjump.com,inbound,001,World,0 +correosocc.com,correosocc.com,inbound,001,World,1 +costco.co.uk,costco.com,inbound,001,World,0 +costco.com,costco.com,inbound,001,World,7e-06 +costcophotocenter.com,wc09.net,inbound,001,World,0 +costcoservices.com,costco.com,inbound,001,World,0 +cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,001,World,0 +couchsurfing.org,couchsurfing.com,inbound,001,World,0 +countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,001,World,0 +couponamama.com,couponamama.com,inbound,001,World,1 +coupondunia.in,coupondunia.in,inbound,001,World,1 +cox.com,cox.com,inbound,001,World,0.001665 +cox.net,cox.net,inbound,001,World,0.009187 +cox.net,cox.net,outbound,001,World,0 +coyotelogistics.com,postini.com,inbound,001,World,0 +cp20.com,cp20.com,inbound,001,World,0 +cpbnc.com,cpbnc.com,inbound,001,World,0 +cpbnc.com,fye.com,inbound,001,World,0 +cpc.gov.in,cpc.gov.in,inbound,001,World,0 +cpm.co.ma,cpm.co.ma,inbound,001,World,0 +crabtree-evelyn.com,crabtree-evelyn.com,inbound,001,World,0.000566 +crackle.com,crackle.com,inbound,001,World,0 +craigslist.org,craigslist.org,inbound,001,World,0 +craigslist.org,craigslist.org,outbound,001,World,1 +crainnewsalerts.com,crainnewsalerts.com,inbound,001,World,0 +crashlytics.com,crashlytics.com,inbound,001,World,1 +crashlytics.com,sendgrid.net,inbound,001,World,1 +crateandbarrel.com,crateandbarrel.com,inbound,001,World,0 +cratusservices.in,ramcorp.in,inbound,001,World,0 +creationsrewards.net,creationsrewards.net,inbound,001,World,0 +creditkarma.com,creditkarma.com,inbound,001,World,1 +credoaction.com,credoaction.com,inbound,001,World,1 +cricinfo.com,cricinfo.com,inbound,001,World,0 +cricut.com,elabs12.com,inbound,001,World,0 +criticalimpactinc.com,criticalimpactinc.com,inbound,001,World,0 +critsend.com,critsend.com,inbound,001,World,0 +crmstyle.com,crmstyle.com,inbound,001,World,0 +crocos.jp,crocos.jp,inbound,001,World,0 +crocs-email.com,crocs-email.com,inbound,001,World,0 +crosswalkmail.com,crosswalkmail.com,inbound,001,World,0 +crowdcut.com,crowdcut.com,inbound,001,World,1 +crsend.com,crsend.com,inbound,001,World,0.008688 +crunchyroll.com,crunchyroll.com,inbound,001,World,0 +csas.cz,csas.cz,inbound,001,World,0.999971 +ctrip.com,ctrip.com,inbound,001,World,0.01917 +cudo.com.au,exacttarget.com,inbound,001,World,0 +cuenote.jp,cuenote.jp,inbound,001,World,0 +cumulusdist.net,cumulusdist.net,inbound,001,World,0 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,001,World,1 +cuponatic.com.pe,cuponatic.com.pe,inbound,001,World,1 +cuponicamail.com,fnbox.com,inbound,001,World,0 +cuppon.pl,cuppon.pl,inbound,001,World,0 +curbednetwork.com,curbednetwork.com,inbound,001,World,1 +curriculum.com.br,curriculum.com.br,inbound,001,World,0 +currys.co.uk,currys.co.uk,inbound,001,World,0 +cuspemail.com,neimanmarcusemail.com,inbound,001,World,0 +custom-emailing.com,elabs12.com,inbound,001,World,0 +custombriefings.com,custombriefings.com,inbound,001,World,0 +customercenter.net,customercenter.net,inbound,001,World,0.996453 +customeriomail.com,customeriomail.com,inbound,001,World,1 +cv-library.co.uk,cv-library.co.uk,inbound,001,World,0 +cvbankas.lt,efadm.eu,inbound,001,World,0 +cvent-planner.com,cvent-planner.com,inbound,001,World,0 +cw.com.tw,cw.com.tw,inbound,001,World,0.002535 +cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 +cxomedia.com,cxomedia.com,inbound,001,World,0 +cybercoders.com,cybercoders.com,inbound,001,World,0 +cyberdiet.com.br,allinmedia.com.br,inbound,001,World,0 +cyberlinkmember.com,cyberlinkmember.com,inbound,001,World,0 +d-reizen.nl,dmdelivery.com,inbound,001,World,0 +dabmail.com,iaires.com,inbound,001,World,0 +dabmail.com,mailurja.com,inbound,001,World,0 +dafiti.cl,dafiti.cl,inbound,001,World,0 +dafiti.com.br,fagms.de,inbound,001,World,0 +dailyhoroscope.com,tarot.com,inbound,001,World,0 +dailyom.com,dailyom.com,inbound,001,World,1 +dairyqueen.com,dairyqueen.com,inbound,001,World,0 +datadrivenemail.com,datadrivenemail.com,inbound,001,World,0 +datehookup.com,datehookup.com,inbound,001,World,0 +datingfactory.com,caerussolutions.net,inbound,001,World,0 +datingvipnotifications.com,datingvipnotifications.com,inbound,001,World,0 +daveramsey.com,daveramsey.com,inbound,001,World,0.025924 +daviacalendar.com,daviacalendar.com,inbound,001,World,1 +davidsbridal.com,davidsbridal.com,inbound,001,World,0 +davidstea.com,bronto.com,inbound,001,World,0 +daz3d.com,bronto.com,inbound,001,World,0 +dbgi.co.uk,emc1.co.uk,inbound,001,World,0 +ddc-emails.com,ddc-emails.com,inbound,001,World,0 +deal.com.sg,emarsys.net,inbound,001,World,0 +dealchicken.com,dealchicken.com,inbound,001,World,0 +dealchicken.com,exacttarget.com,inbound,001,World,0 +dealersocket.com,dealersocket.com,inbound,001,World,4e-06 +dealfind.com,dealfind.com,inbound,001,World,0 +dealnews.com,dealnews.com,inbound,001,World,0 +dealsaver.com,secondstreetmedia.com,inbound,001,World,0.999823 +dealsdirect.com.au,dealsdirect.com.au,inbound,001,World,0 +dealspl.us,dealspl.us,inbound,001,World,0 +debian.org,debian.org,inbound,001,World,1 +debshops.com,lstrk.net,inbound,001,World,1 +deezer.com,dms30.com,inbound,001,World,0 +deliasshopemail.com,deliasshopemail.com,inbound,001,World,0 +delivery.net,delivery.net,inbound,001,World,0 +delivery.net,m0.net,inbound,001,World,0 +dell.com,bfi0.com,inbound,001,World,0 +dell.com,dell.com,inbound,001,World,0.969277 +delta.com,delta.com,inbound,001,World,0.092496 +dena.ne.jp,dena.ne.jp,inbound,001,World,0.000249 +dentalsenders.com,dentalsenders.com,inbound,001,World,0 +dermstore.com,exacttarget.com,inbound,001,World,0 +descontos.pt,descontos.pt,inbound,001,World,0 +designerapparel.com,myperfectsale.com,inbound,001,World,1 +despegar.com,despegar.com,inbound,001,World,0 +dhgate.com,chtah.net,inbound,001,World,0 +dhl.com,dhl.com,inbound,001,World,0.994107 +dice.com,dice.com,inbound,001,World,0 +dietaesaude.com.br,dietaesaude.com.br,inbound,001,World,0 +dietnavi.com,data-hotel.net,inbound,001,World,0 +digitalmailer.com,digitalmailer.com,inbound,001,World,0 +digitalmedia-comunicacion.es,chtah.net,inbound,001,World,0 +digitalromanceinc.com,digitalromanceinc.com,inbound,001,World,1 +dinda.com.br,dinda.com.br,inbound,001,World,0.017061 +dip-net.co.jp,dip-net.co.jp,inbound,001,World,0 +directcrm.ru,directcrm.ru,inbound,001,World,0 +directresponsemanager.com,wide.ne.jp,inbound,001,World,0 +directv.com,directv.com,inbound,001,World,0.050604 +directvla.com,directvla.com,inbound,001,World,0 +disc.co.jp,disc.co.jp,inbound,001,World,0.001051 +discover.com,discover.com,inbound,001,World,0 +discover.com,discoverfinancial.com,inbound,001,World,1 +dishtv.co.in,dishtv.co.in,inbound,001,World,0.047664 +disney.co.uk,emv9.com,inbound,001,World,0 +disneydestinations.com,disneyparks.com,inbound,001,World,0 +disneydestinations.com,disneyworld.com,inbound,001,World,0 +disparadordeemails.com,locaweb.com.br,inbound,001,World,0 +disqus.net,disqus.net,inbound,001,World,0 +diynetwork.com,diynetwork.com,inbound,001,World,0 +dks.com.tw,dks.com.tw,inbound,001,World,0.018134 +dmm.com,dmm.com,inbound,001,World,0 +dn.net,naukri.com,inbound,001,World,0 +docomo.ne.jp,docomo.ne.jp,inbound,001,World,0 +docomo.ne.jp,docomo.ne.jp,outbound,001,World,0 +doctoroz.com,email-sharecare2.com,inbound,001,World,0 +docusign.net,docusign.net,inbound,001,World,0.985435 +dollartree.com,email-dollartree.com,inbound,001,World,0 +dominos.com,dominos.com,inbound,001,World,0.011684 +dominos.com.au,dominos.com.au,inbound,001,World,0 +dominosemail.co.uk,dominosemail.co.uk,inbound,001,World,0 +donationnet.net,donationnet.net,inbound,001,World,0 +donuts.ne.jp,dnuts.jp,inbound,001,World,0 +doodle.com,doodle.com,inbound,001,World,1 +dorothyperkins.com,dorothyperkins.com,inbound,001,World,0 +dotmailer-email.com,dotmailer.com,inbound,001,World,0 +dotmailer.co.uk,dotmailer.com,inbound,001,World,0 +dotz.com.br,dotz.com.br,inbound,001,World,0 +doubletakeoffers.com,doubletakeoffers.com,inbound,001,World,0 +dowjones.info,dowjones.info,inbound,001,World,0 +downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,001,World,0 +dpapp.nl,prikbordmailer.nl,inbound,001,World,0 +dpapp.nl,sslsecuref.nl,inbound,001,World,0 +dptagent.biz,dptagent.biz,inbound,001,World,0 +dptagent.net,dptagent.net,inbound,001,World,0 +draftkings.com,draftkings.com,inbound,001,World,0 +dreamhost.com,dreamhost.com,inbound,001,World,0 +dreammail.ne.jp,dreammail.jp,inbound,001,World,0 +dreamwidth.org,dreamwidth.org,inbound,001,World,0 +dreivip.com,dreivip.com,inbound,001,World,0.000301 +dress-for-less.de,privalia.com,inbound,001,World,0 +drhinternet.net,drhinternet.net,inbound,001,World,0 +driftem.com,emce2.in,inbound,001,World,0 +driftem.com,mailurja.com,inbound,001,World,0 +drjays-mail.com,drjays-mail.com,inbound,001,World,0 +dromadaire-news.com,ecmcluster.com,inbound,001,World,0 +dropbox.com,dropbox.com,inbound,001,World,1 +dropboxmail.com,dropbox.com,inbound,001,World,1 +drushim.co.il,drushim.co.il,inbound,001,World,0 +drweb.com,drweb.com,inbound,001,World,0.969315 +dstyleweb.com,dstyleweb.com,inbound,001,World,0.001099 +dsw.com,dsw.com,inbound,001,World,0 +ducks.org,uptilt.com,inbound,001,World,0 +duke.edu,duke.edu,inbound,001,World,0.308101 +dukecareers.com,dukecareers.com,inbound,001,World,1 +duluthtradingemail.com,email-duluthtrading.com,inbound,001,World,0 +dvor.com,dvor.com,inbound,001,World,0 +dynamite-safelist.com,thomas-j-brown.com,inbound,001,World,0 +dynect-mailer.net,dynect.net,inbound,001,World,0 +dynect-mailer.net,sendlabs.com,inbound,001,World,0 +e-activist.com,e-activist.com,inbound,001,World,0 +e-beallsonline.com,e-stagestores.com,inbound,001,World,0 +e-bodyc.com,email-bodycentral.com,inbound,001,World,0 +e-boks.dk,e-boks.dk,inbound,001,World,1 +e-costco.mx,costco.com,inbound,001,World,0 +e-ebuyer.com,e-ebuyer.com,inbound,001,World,0 +e-goodysonline.com,e-stagestores.com,inbound,001,World,0 +e-jobs-ville.com,e-jobs-ville.com,inbound,001,World,1 +e-leclerc.com,e-leclerc.com,inbound,001,World,0.000248 +e-mark.nl,e-mark.nl,inbound,001,World,0 +e-ngine.nl,e-ngine.nl,inbound,001,World,0 +e-peebles.com,e-stagestores.com,inbound,001,World,0 +e-rewards.net,e-rewards.net,inbound,001,World,1 +e-stagestores.com,e-stagestores.com,inbound,001,World,0 +e-travelclub.es,e-travelclub.es,inbound,001,World,0 +e2ma.net,e2ma.net,inbound,001,World,1 +ea.com,ea.com,inbound,001,World,0.001314 +eaccess.net,postini.com,inbound,001,World,0 +earn-e-miles.com,earn-e-miles.com,inbound,001,World,0 +earnerslist.com,traxweb.net,inbound,001,World,9e-06 +earthfare-email.com,edclient2.com,inbound,001,World,0 +earthlink.net,earthlink.net,inbound,001,World,0.031678 +earthlink.net,earthlink.net,outbound,001,World,0 +eastbay.com,footlocker.com,inbound,001,World,0 +easycanvasprints.com,easycanvasprints.com,inbound,001,World,0 +easyhealthoptions.com,easyhealthoptions.com,inbound,001,World,1 +easyhits4u.com,easyhits4u.com,inbound,001,World,1 +easyhits4u.com,relmax.net,inbound,001,World,0 +easyroommate.com,easyroommate.com,inbound,001,World,0.108483 +ebags.com,ebags.com,inbound,001,World,0 +ebates.com,bfi0.com,inbound,001,World,0 +ebay-kleinanzeigen.de,mobile.de,inbound,001,World,1 +ebay.{...},ebay.{...},inbound,001,World,0.99953 +ebay.{...},emarsys.net,inbound,001,World,0 +ebay.{...},postdirect.com,inbound,001,World,0 +ebizac2.com,ebizac2.com,inbound,001,World,0 +ebizac3.com,ebizac3.com,inbound,001,World,0 +eblastengine.com,secondstreetmedia.com,inbound,001,World,0.999827 +ebuildabear.com,ebuildabear.com,inbound,001,World,8e-06 +ec2.internal,amazonaws.com,inbound,001,World,0.769117 +ec21.com,ec21.com,inbound,001,World,0.002261 +ecasend.com,ecasend.com,inbound,001,World,0 +ecnavi.jp,ecnavi.jp,inbound,001,World,0 +ecommzone.com,ecommzone.com,inbound,001,World,0 +ed.gov,leepfrog.com,inbound,001,World,0.106381 +ed10.net,ed10.com,inbound,001,World,0 +ed10.net,postini.com,inbound,001,World,0.083658 +edarling.fr,fagms.de,inbound,001,World,0 +eddiebauer.com,eddiebauer.com,inbound,001,World,0 +edima.hu,edima.hu,inbound,001,World,0 +edirect1.com,ivytech.edu,inbound,001,World,0 +edmodo.com,edmodo.com,inbound,001,World,1.1e-05 +educationzone.co.in,iaires.com,inbound,001,World,0 +eduk.com,eduk.com,inbound,001,World,0 +efamilydollar.com,efamilydollar.com,inbound,001,World,0 +effectivesafelist.com,zoothost.com,inbound,001,World,0 +efox-shop.com,dmdelivery.com,inbound,001,World,0 +eharmony.com,eharmony.com,inbound,001,World,1e-06 +eigbox.net,eigbox.net,inbound,001,World,0 +ejobs.ro,ejobs.ro,inbound,001,World,8.4e-05 +elabs10.com,elabs10.com,inbound,001,World,0 +elabs12.com,elabs12.com,inbound,001,World,0 +elabs3.com,elabs3.com,inbound,001,World,0 +elabs3.com,meritline.com,inbound,001,World,0 +elabs5.com,elabs5.com,inbound,001,World,0 +elabs6.com,elabs6.com,inbound,001,World,0 +elaine-asp.de,artegic.net,inbound,001,World,0.999997 +elanceonline.com,elanceonline.com,inbound,001,World,0 +elcorteingles.es,elcorteingles.es,inbound,001,World,0 +eleadtrack.net,eleadtrack.net,inbound,001,World,0 +elektronskaposta.si,eprvak.si,inbound,001,World,0 +elettershop.de,servicemail24.de,inbound,001,World,1 +elistas.net,elistas.net,inbound,001,World,0 +elitesafelist.com,elitesafelist.com,inbound,001,World,0 +elkjop.no,ec-cluster.com,inbound,001,World,0 +elkjop.no,eccluster.com,inbound,001,World,0 +elo7.com.br,elo7.com.br,inbound,001,World,0 +emag.ro,emag.ro,inbound,001,World,0.005013 +email-1800contacts.com,email-1800contacts.com,inbound,001,World,0 +email-aaa.com,email-aaa.com,inbound,001,World,0 +email-aeriagames.com,email-aeriagames.com,inbound,001,World,0 +email-comparethemarket.com,smartfocusdigital.net,inbound,001,World,0 +email-cooking.com,email-cooking.com,inbound,001,World,0 +email-dressbarn.com,email-dressbarn.com,inbound,001,World,0 +email-firestone.com,reminder-firestone.com,inbound,001,World,0 +email-galls.com,email-galls.com,inbound,001,World,1 +email-honest.com,email-honest.com,inbound,001,World,0 +email-od.com,email-od.com,inbound,001,World,0.999652 +email-od.com,smtprelayserver.com,inbound,001,World,0.999779 +email-petsmart.com,email-petsmart.com,inbound,001,World,0 +email-sportchalet.com,email-sportchalet.com,inbound,001,World,0 +email-telekom.de,ecm-cluster.com,inbound,001,World,0 +email-ticketdada.com,email-ticketdada.com,inbound,001,World,1 +email-totalwine.com,email-totalwine.com,inbound,001,World,0 +email-wildstar-online.com,email-carbine.com,inbound,001,World,0 +email2-beyond.com,messagebus.com,inbound,001,World,0 +email360api.com,email360api.com,inbound,001,World,0 +email365inc.com,email365inc.com,inbound,001,World,0 +email3m.com,email3m.com,inbound,001,World,0 +email4-beyond.com,email4-beyond.com,inbound,001,World,0 +emailcounts.com,secureserver.net,inbound,001,World,0 +emaildir2.com,emaildirect.net,inbound,001,World,0 +emaildir2.com,espsnd.com,inbound,001,World,0 +emailnotify.net,emailnotify.net,inbound,001,World,0.961424 +emailrestaurant.com,emailrestaurant.com,inbound,001,World,0 +emailsbancoestado.cl,emailsbancoestado.cl,inbound,001,World,0 +emailsripley.cl,etarget.cl,inbound,001,World,0 +emailtoryburch.com,emailtoryburch.com,inbound,001,World,0 +emarsys.net,emarsys.net,inbound,001,World,0.092041 +embarqmail.com,centurylink.net,inbound,001,World,0.999918 +embluejet.com,embluejet.com,inbound,001,World,0 +embluejet.com,emblueuser.com,inbound,001,World,0 +emcsend.com,emcsend.com,inbound,001,World,0 +emergencyemail.org,emergencyemail.org,inbound,001,World,0 +eminentinc.com,eminentinc.com,inbound,001,World,0 +emktsender.net,locaweb.com.br,inbound,001,World,0 +emktviajarbarato.com.br,splio.com.br,inbound,001,World,0.875902 +emma.cl,emma.cl,inbound,001,World,0.996895 +emobile.ad.jp,postini.com,inbound,001,World,0 +employboard.com,employboard.com,inbound,001,World,1 +empoweredcomms.com.au,empoweredcomms.com.au,inbound,001,World,0 +emsecure.net,emsecure.net,inbound,001,World,0 +emsmtp.com,emsmtp.com,inbound,001,World,0.175797 +en-japan.com,en-japan.com,inbound,001,World,0.000204 +en25.com,kqed.org,inbound,001,World,0 +enewscartes.net,bp06.net,inbound,001,World,0 +enewsletter.pl,enewsletter.pl,inbound,001,World,0.005395 +enewsletter.pl,mydeal.pl,inbound,001,World,0 +enewsletter.pl,sare25.com,inbound,001,World,0 +enplenitud.com,enplenitud.com,inbound,001,World,0 +entregadeemails.com,locaweb.com.br,inbound,001,World,0 +entregadordecampanhas.net,locaweb.com.br,inbound,001,World,0 +entrepreneur.com,entrepreneur.com,inbound,001,World,0 +enviodecampanhas.net,locaweb.com.br,inbound,001,World,0 +enviodemkt.com.br,locaweb.com.br,inbound,001,World,0 +eonet.ne.jp,eonet.ne.jp,inbound,001,World,0.99955 +epaper.com.tw,epaper.com.tw,inbound,001,World,0 +eplus.jp,eplus.jp,inbound,001,World,0 +epriority.com,epriority.com,inbound,001,World,0 +equifax.com,equifax.com,inbound,001,World,0.936013 +equussafelist.com,equussafelist.com,inbound,001,World,0.000265 +eslitebooks.com,eslitebooks.com,inbound,001,World,0 +espmp-agfr.net,bp06.net,inbound,001,World,0 +esprit-friends.com,esprit-friends.com,inbound,001,World,0 +esri.com,esri.com,inbound,001,World,0.983104 +esteelauder.com,esteelauder.com,inbound,001,World,0 +ethingsremembered.com,ethingsremembered.com,inbound,001,World,0 +etrade.com,etrade.com,inbound,001,World,0.021422 +etransmail.com,etransmail.com,inbound,001,World,0 +etransmail.com,ptransmail.com,inbound,001,World,0 +etrmailbox.com,etrmailbox.com,inbound,001,World,0 +etsy.com,etsy.com,inbound,001,World,0.020844 +euromsg.net,euromsg.net,inbound,001,World,0 +evaair.com,evaair.com,inbound,001,World,0.007363 +evanguard.com,evanguard.com,inbound,001,World,0 +evanscycles.com,msgfocus.com,inbound,001,World,0 +eventbrite.com,eventbrite.com,inbound,001,World,0 +evernote.com,evernote.com,inbound,001,World,1 +eversavelocal.com,eversavelocal.com,inbound,001,World,0 +everydayfamily.com,everydayfamily.com,inbound,001,World,0 +everydayhealthinc.com,waterfrontmedia.net,inbound,001,World,0 +everyjobforme.com,everyjobforme.com,inbound,001,World,0 +everytown.org,everytown.org,inbound,001,World,1 +exacttarget.com,bazaarvoice.com,inbound,001,World,0 +exacttarget.com,booksamillion.com,inbound,001,World,0 +exacttarget.com,exacttarget.com,inbound,001,World,0.000325 +exacttarget.com,msg.com,inbound,001,World,0 +exacttarget.com,redboxinstant.com,inbound,001,World,0 +exacttarget.com,skylinetechnologies.com,inbound,001,World,0 +exchangesolutions.com,exchangesolutions.com,inbound,001,World,0.000143 +exec-u-net-mail.com,exec-u-net-mail.com,inbound,001,World,0 +expediamail.com,airasiago.com,inbound,001,World,1 +expediamail.com,exacttarget.com,inbound,001,World,1 +expediamail.com,expediamail.com,inbound,001,World,0.839662 +expediamail.com,quotitmail.com,inbound,001,World,0 +experteer.com,experteer.com,inbound,001,World,0 +express.com,expressfashion.com,inbound,001,World,0 +exprpt.com,exprpt.com,inbound,001,World,0 +expvtinboxhub.net,expvtinboxhub.net,inbound,001,World,0 +extra.com.br,emv8.com,inbound,001,World,0 +eyepin.com,eyepin.com,inbound,001,World,0 +ezweb.ne.jp,ezweb.ne.jp,inbound,001,World,0.282076 +ezweb.ne.jp,ezweb.ne.jp,outbound,001,World,0 +fabfurnish.com,fagms.de,inbound,001,World,0 +fabletics.com,bronto.com,inbound,001,World,0 +facebook.com,facebook.com,inbound,001,World,0.553197 +facebook.com,facebook.com,outbound,001,World,1 +facebookappmail.com,facebook.com,inbound,001,World,1 +facebookmail.com,facebook.com,inbound,001,World,1 +facebookmail.com,postini.com,inbound,001,World,0.726315 +facebookmail.com,yahoo.{...},inbound,001,World,0.999904 +facilisimo.com,facilisimo.com,inbound,001,World,1 +fagms.net,fagms.de,inbound,001,World,0 +falabella.com,falabella.com,inbound,001,World,0 +familychristianmail.com,familychristianmail.com,inbound,001,World,0 +famousfootwear.com,famousfootwear.com,inbound,001,World,0 +fanatics.com,fanatics.com,inbound,001,World,0 +fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,001,World,0 +fanbridge.com,fanbridge.com,inbound,001,World,0.000198 +fanfiction.com,fictionpress.com,inbound,001,World,1 +fanofannas.com,fanofannas.com,inbound,001,World,0 +fansedge.com,fansedge.com,inbound,001,World,0 +farmers.com,farmers.com,inbound,001,World,0.999984 +farmersonly.com,mailgun.net,inbound,001,World,1 +farmersonly.com,mailgun.us,inbound,001,World,1 +fashion2hub.in,mgenie.in,inbound,001,World,0 +fastcompany.com,fastcompany.com,inbound,001,World,0 +fastgb.com,fastgb.com,inbound,001,World,0 +fastlistmailer.com,zoothost.com,inbound,001,World,0 +fastweb.com,fastweb.com,inbound,001,World,0 +fbi.gov,fbi.gov,inbound,001,World,0 +fbmta.com,fbmta.com,inbound,001,World,0 +fc2.com,fc2.com,inbound,001,World,0.000798 +fedex.com,fedex.com,inbound,001,World,0.921011 +fedoraproject.org,fedoraproject.org,inbound,001,World,5e-06 +feedblitz.com,feedblitz.com,inbound,001,World,0 +feld-ent.com,postdirect.com,inbound,001,World,0 +felissimo.jp,felissimo.jp,inbound,001,World,0 +fellowshiponemail.com,fellowshiponemail.com,inbound,001,World,0 +fetlifemail.com,fetlifemail.com,inbound,001,World,0 +fibertel.com.ar,fibertel.com.ar,inbound,001,World,0.003897 +fidelity.com,fidelity.com,inbound,001,World,1 +fidelizador.org,fidelizador.org,inbound,001,World,0 +financialfreedommail.com,financialfreedommail.com,inbound,001,World,0 +finansbank.com.tr,finansbank.com.tr,inbound,001,World,0.927 +findexpvtinbox.com,findexpvtinbox.com,inbound,001,World,0 +fingerhut.com,fingerhut.com,inbound,001,World,0.020664 +finishline.com,finishline.com,inbound,001,World,0 +finn.no,schibsted-it.no,inbound,001,World,0.002893 +firemountaingems.com,firemountaingems.com,inbound,001,World,0 +fiscosoft.com.br,fiscosoft.com.br,inbound,001,World,0 +fisher-price.com,fisher-price.com,inbound,001,World,0 +fitbit.com,fitbit.com,inbound,001,World,1 +fitnessmagazine.com,meredith.com,inbound,001,World,0 +fiverr.com,fiverr.com,inbound,001,World,0 +fixeads.com,fixeads.com,inbound,001,World,0 +flets.com,flets.com,inbound,001,World,0 +flexmls.com,flexmls.com,inbound,001,World,0.999992 +flightaware.com,flightaware.com,inbound,001,World,0.020698 +flipkart.com,flipkart.com,inbound,001,World,1 +flirchi.com,flirchi.com,inbound,001,World,1.0 +flirt.com,ropot.net,inbound,001,World,0 +flirthookup.com,flirthookup.com,inbound,001,World,1 +flirtlocal.com,flirtlocal.com,inbound,001,World,1 +flmsecure.com,fling.com,inbound,001,World,0 +flmsecure.com,flmsecure.com,inbound,001,World,0 +floridajobdepartment.com,floridajobdepartment.com,inbound,001,World,0 +flyceb.com,flyceb.com,inbound,001,World,0 +flyfrontier.com,flyfrontier.com,inbound,001,World,0 +flymonarchemail.com,flymonarchemail.com,inbound,001,World,0 +fmworld.net,fmworld.net,inbound,001,World,0 +fnac.com,fnac.com,inbound,001,World,0.021985 +fnb.co.za,fnb.co.za,inbound,001,World,0.104983 +fofa.jp,mpme.jp,inbound,001,World,0 +follow-up.se,follow-up.se,inbound,001,World,1 +foodnetwork.com,foodnetwork.com,inbound,001,World,0 +foolsubs.com,foolcs.com,inbound,001,World,0 +foolsubs.com,foolsubs.com,inbound,001,World,0 +footaction.com,footlocker.com,inbound,001,World,0 +footlocker.com,footlocker.com,inbound,001,World,0.000605 +forcemail.in,iaires.com,inbound,001,World,0 +foreseegame.com,iaires.com,inbound,001,World,0 +forever21.com,forever21.com,inbound,001,World,0 +fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,001,World,0 +fotffamily.com,fotffamily.com,inbound,001,World,0 +fotocasa.es,fotocasa.es,inbound,001,World,0 +fotolivro.com.br,fotolivro.com.br,inbound,001,World,0 +fotostrana.ru,fotocdn.net,inbound,001,World,3.4e-05 +foursquare.com,foursquare.com,inbound,001,World,1 +foxnews.com,foxnews.com,inbound,001,World,0.0139 +fpmailerbr.com,fpmailerbr.com,inbound,001,World,0 +fragrancenet.com,fragrancenet.com,inbound,001,World,0.000427 +francescas.com,bronto.com,inbound,001,World,0 +free-lance.ru,free-lance.ru,inbound,001,World,0 +free.fr,free.fr,inbound,001,World,0.984012 +free.fr,free.fr,outbound,001,World,6.9e-05 +freeadsmailer.com,zoothost.com,inbound,001,World,0 +freebeesafelist.com,zoothost.com,inbound,001,World,0 +freebizmag.com,delivery.net,inbound,001,World,0 +freebsd.org,freebsd.org,inbound,001,World,0.999835 +freecycle.org,freecycle.org,inbound,001,World,0.999927 +freedesktop.org,freedesktop.org,inbound,001,World,0 +freeflys.com,freeflys.com,inbound,001,World,0 +freelancer.com,freelancer.com,inbound,001,World,0 +freelancer.com,freelancernotify.com,inbound,001,World,0 +freelancer.com,getafreelancer.com,inbound,001,World,0 +freelists.org,iquest.net,inbound,001,World,0 +freelotto.com,plasmanetinc.com,inbound,001,World,0 +freemail.hu,freemail.hu,outbound,001,World,0 +freeml.com,gmo-media.jp,inbound,001,World,0 +freepeople.com,freepeople.com,inbound,001,World,0 +freesafelistking.com,zoothost.com,inbound,001,World,0 +freesafelistmailer.com,waters-advertising.com,inbound,001,World,0 +freshdesk.com,freshdesk.com,inbound,001,World,1 +freshers2015.com,secureserver.net,inbound,001,World,0 +freshlatesave.com,freshlatesave.com,inbound,001,World,1 +freshmail.pl,freshmail.pl,inbound,001,World,0 +fridays.com,fridays.com,inbound,001,World,0 +friskone.com,mailurja.com,inbound,001,World,0 +frk.com,frk.com,inbound,001,World,0.999995 +frontdoor.com,frontdoor.com,inbound,001,World,0 +frontgate-email.com,frontgate-email.com,inbound,001,World,0 +frontsight.com,frontsight.com,inbound,001,World,0 +frys.com,frys.com,inbound,001,World,0.00388 +frysmail.com,frysmail.com,inbound,001,World,0 +fspeletters.com,agorapub.co.uk,inbound,001,World,0 +ftchinese.com,ftchinese.com,inbound,001,World,0 +fubonshop.com,fubonshop.com,inbound,001,World,0 +fuckbooknet.net,infinitypersonals.com,inbound,001,World,0 +fuelrewards.com,britecast.com,inbound,001,World,0 +fundplaza.co.in,arrowsignindia.com,inbound,001,World,0 +fundplaza.in,fundplaza.in,inbound,001,World,0 +funonthenet.in,funonthenet.in,inbound,001,World,1 +futureshop.com,futureshop.com,inbound,001,World,0 +futurmailer.pt,futurmailer.pt,inbound,001,World,0 +gabbar.info,gabbar.info,inbound,001,World,1 +gaiaonline.com,gaiaonline.com,inbound,001,World,0 +gamecity.ne.jp,gamecity.ne.jp,inbound,001,World,0 +gamefly.com,gamefly.com,inbound,001,World,0.013381 +gamehouse.com,gamehouse.com,inbound,001,World,0 +gamingmails.com,gamingmails.com,inbound,001,World,0 +gap.com,gap.com,inbound,001,World,0 +gap.eu,gap.eu,inbound,001,World,0 +gapcanada.ca,gapcanada.ca,inbound,001,World,0 +garanti.com.tr,euromsg.net,inbound,001,World,0 +garanti.com.tr,garanti.com.tr,inbound,001,World,0.421936 +gardeningclubmail.co.uk,msgfocus.com,inbound,001,World,0 +garnethill-email.com,garnethill-email.com,inbound,001,World,0 +gaylordalert.com,gaylordalert.com,inbound,001,World,0 +gbyguess.com,guess.com,inbound,001,World,0.001787 +gcast.com.au,systemsserver.net,inbound,001,World,0 +gdtsuccess.com,groupdealtools.com,inbound,001,World,1 +geico.com,geico.com,inbound,001,World,0.096879 +gemoney.com,rsys1.com,inbound,001,World,0 +gene.com,roche.com,inbound,001,World,1 +generalmills.com,boxtops4education.com,inbound,001,World,0 +generalmills.com,pillsbury.com,inbound,001,World,0 +gentoo.org,gentoo.org,inbound,001,World,1 +geocaching.com,groundspeak.com,inbound,001,World,1 +geojit.com,geojit.com,inbound,001,World,0.203748 +get-me-jobs.com,get-me-jobs.com,inbound,001,World,0 +gethired.com,gethired.com,inbound,001,World,1 +getinbox.net,getinbox.net,inbound,001,World,0 +getitfree.us,getitfree.us,inbound,001,World,0 +getkeepsafe.com,getkeepsafe.com,inbound,001,World,1 +getmein.com,getmein.com,inbound,001,World,0 +getpaidsolutions.com,getpaidsolutions.com,inbound,001,World,1 +getpocket.com,bronto.com,inbound,001,World,0 +getresponse.com,getresponse.com,inbound,001,World,0 +gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,001,World,0 +ghin.com,ghinconnect.com,inbound,001,World,0 +ghup.in,mgenie.in,inbound,001,World,0 +giffgaff.com,giffgaff.com,inbound,001,World,0 +gillyhicks-email.com,abercrombie-email.com,inbound,001,World,0 +gilt.com,gilt.com,inbound,001,World,1e-06 +gilt.jp,gilt.jp,inbound,001,World,0 +github.com,github.com,inbound,001,World,1 +github.com,github.net,inbound,001,World,1 +github.com,postini.com,inbound,001,World,0.872186 +glassdoor.com,glassdoor.com,inbound,001,World,0.662834 +glasses.com,glasses.com,inbound,001,World,0 +gliq.com,gliq.com,inbound,001,World,0.99852 +globalmembersupport.com,globalmembersupport.com,inbound,001,World,0 +globalsafelist.com,globalsafelist.com,inbound,001,World,0 +globalsources.com,globalsources.com,inbound,001,World,0.007546 +globalspec.com,globalspec.com,inbound,001,World,0 +globaltestmarket.com,globaltestmarket.com,inbound,001,World,0 +globasemail.com,globasemail.com,inbound,001,World,0.951088 +globetel.com.ph,globetel.com.ph,inbound,001,World,1 +gmail.com,02.net,inbound,001,World,0.998007 +gmail.com,amazonaws.com,inbound,001,World,0.988441 +gmail.com,anteldata.net.uy,inbound,001,World,0.998653 +gmail.com,as13285.net,inbound,001,World,0.999943 +gmail.com,asianet.co.th,inbound,001,World,0.998473 +gmail.com,au-net.ne.jp,inbound,001,World,1 +gmail.com,bbox.fr,inbound,001,World,0.919868 +gmail.com,bbtec.net,inbound,001,World,1 +gmail.com,belgacom.be,inbound,001,World,0.822281 +gmail.com,bell.ca,inbound,001,World,0.992482 +gmail.com,bellsouth.net,inbound,001,World,0.9996 +gmail.com,bezeqint.net,inbound,001,World,0.974444 +gmail.com,bigpond.net.au,inbound,001,World,0.999907 +gmail.com,blackberry.com,inbound,001,World,0.996337 +gmail.com,bluewin.ch,inbound,001,World,0.9337 +gmail.com,brasiltelecom.net.br,inbound,001,World,0.99995 +gmail.com,btcentralplus.com,inbound,001,World,0.999969 +gmail.com,centurytel.net,inbound,001,World,0.999415 +gmail.com,cgocable.net,inbound,001,World,0.998291 +gmail.com,charter.com,inbound,001,World,0.999111 +gmail.com,chello.nl,inbound,001,World,0.999951 +gmail.com,claro.net.br,inbound,001,World,1 +gmail.com,comcast.net,inbound,001,World,0.999616 +gmail.com,comcastbusiness.net,inbound,001,World,0.985464 +gmail.com,cox.net,inbound,001,World,0.962636 +gmail.com,data-hotel.net,inbound,001,World,0.000605 +gmail.com,emailsrvr.com,inbound,001,World,1 +gmail.com,embarqhsd.net,inbound,001,World,0.999554 +gmail.com,fastwebnet.it,inbound,001,World,0.984708 +gmail.com,franchiseindia.com,inbound,001,World,1 +gmail.com,frontiernet.net,inbound,001,World,0.995233 +gmail.com,gvt.net.br,inbound,001,World,0.999513 +gmail.com,hinet.net,inbound,001,World,0.97312 +gmail.com,iinet.net.au,inbound,001,World,0.966984 +gmail.com,jazztel.es,inbound,001,World,0.999701 +gmail.com,lorexddns.net,inbound,001,World,0 +gmail.com,majesticmoneymailer.com,inbound,001,World,1 +gmail.com,mchsi.com,inbound,001,World,0.999951 +gmail.com,movistar.cl,inbound,001,World,0.999361 +gmail.com,mtnl.net.in,inbound,001,World,0.999527 +gmail.com,mycingular.net,inbound,001,World,0.999918 +gmail.com,myvzw.com,inbound,001,World,0.999889 +gmail.com,naukri.com,inbound,001,World,0.000998 +gmail.com,net24.it,inbound,001,World,0.999964 +gmail.com,netcabo.pt,inbound,001,World,0.998164 +gmail.com,netvigator.com,inbound,001,World,0.818637 +gmail.com,numericable.fr,inbound,001,World,0.999726 +gmail.com,ocn.ne.jp,inbound,001,World,0.99466 +gmail.com,ono.com,inbound,001,World,0.995991 +gmail.com,optonline.net,inbound,001,World,0.999776 +gmail.com,optusnet.com.au,inbound,001,World,0.992856 +gmail.com,orange.es,inbound,001,World,0.998743 +gmail.com,orange.fr,inbound,001,World,0 +gmail.com,otenet.gr,inbound,001,World,0.957964 +gmail.com,panda-world.ne.jp,inbound,001,World,1 +gmail.com,postini.com,inbound,001,World,0.776029 +gmail.com,proxad.net,inbound,001,World,0.998094 +gmail.com,qwest.net,inbound,001,World,0.997575 +gmail.com,rcn.com,inbound,001,World,0.986768 +gmail.com,rima-tde.net,inbound,001,World,0.99915 +gmail.com,rogers.com,inbound,001,World,0.999917 +gmail.com,rr.com,inbound,001,World,0.967896 +gmail.com,sbcglobal.net,inbound,001,World,0.998817 +gmail.com,secureserver.net,inbound,001,World,0.272513 +gmail.com,seed.net.tw,inbound,001,World,0.992252 +gmail.com,sfr.net,inbound,001,World,0.999878 +gmail.com,shawcable.net,inbound,001,World,0.999998 +gmail.com,singnet.com.sg,inbound,001,World,0.955461 +gmail.com,skybroadband.com,inbound,001,World,0.999854 +gmail.com,spcsdns.net,inbound,001,World,0.999998 +gmail.com,suddenlink.net,inbound,001,World,0.961594 +gmail.com,t-ipconnect.de,inbound,001,World,0.999841 +gmail.com,tdc.net,inbound,001,World,0.999591 +gmail.com,telecom.net.ar,inbound,001,World,0.999664 +gmail.com,telecomitalia.it,inbound,001,World,0.996998 +gmail.com,telekom.hu,inbound,001,World,0.999977 +gmail.com,telenet.be,inbound,001,World,1 +gmail.com,telepac.pt,inbound,001,World,0.999584 +gmail.com,telesp.net.br,inbound,001,World,0.999743 +gmail.com,telia.com,inbound,001,World,1 +gmail.com,telkomadsl.co.za,inbound,001,World,0.999989 +gmail.com,telus.com,inbound,001,World,1 +gmail.com,telus.net,inbound,001,World,0.974677 +gmail.com,threembb.co.uk,inbound,001,World,1 +gmail.com,tmodns.net,inbound,001,World,0.999979 +gmail.com,totbb.net,inbound,001,World,0.999876 +gmail.com,tpgi.com.au,inbound,001,World,0.999649 +gmail.com,tpnet.pl,inbound,001,World,0.999761 +gmail.com,veloxzone.com.br,inbound,001,World,0.999969 +gmail.com,verizon.net,inbound,001,World,0.990214 +gmail.com,videotron.ca,inbound,001,World,0.967213 +gmail.com,virginm.net,inbound,001,World,0.996611 +gmail.com,vodacom.co.za,inbound,001,World,1 +gmail.com,vodafone-ip.de,inbound,001,World,1 +gmail.com,vodafone.pt,inbound,001,World,0.999563 +gmail.com,vodafonedsl.it,inbound,001,World,0.999006 +gmail.com,vtr.net,inbound,001,World,0.999072 +gmail.com,wanadoo.fr,inbound,001,World,0.999753 +gmail.com,websitewelcome.com,inbound,001,World,1 +gmail.com,wideopenwest.com,inbound,001,World,0.999729 +gmail.com,windstream.net,inbound,001,World,0.951847 +gmail.com,yahoo.{...},inbound,001,World,0.999147 +gmail.com,ziggo.nl,inbound,001,World,1 +gmail.com,zoothost.com,inbound,001,World,0.033858 +gmo.jp,gmo-media.jp,inbound,001,World,0 +gmoes.jp,gmoes.jp,inbound,001,World,0 +gmsend.com,gmsend.com,inbound,001,World,0 +gmt.ne.jp,gmt.ne.jp,inbound,001,World,0 +gmx.de,gmx.net,inbound,001,World,1 +gmx.de,gmx.net,outbound,001,World,1 +gmx.net,gmx.net,inbound,001,World,1 +gnavi.co.jp,gnavi.co.jp,inbound,001,World,0.002441 +go.com,starwave.com,inbound,001,World,0.006276 +go4worldbusiness.com,go4worldbusiness.com,inbound,001,World,1 +goalunited.org,ccmdcampaigns.net,inbound,001,World,0 +gob.ar,gob.ar,inbound,001,World,0.159673 +gob.ec,gob.ec,inbound,001,World,0.618006 +godaddy.com,secureserver.net,inbound,001,World,0 +godtubemail.com,godtubemail.com,inbound,001,World,0 +godvinemail.com,godvinemail.com,inbound,001,World,0 +gog.com,gog.com,inbound,001,World,0 +gogecapital.com,rsys1.com,inbound,001,World,0 +gogroopie.com,gogroopie.com,inbound,001,World,0.000283 +gohappy.com.tw,gohappy.com.tw,inbound,001,World,0.000104 +goldenbrands.gr,goldenbrands.gr,inbound,001,World,1 +goldenline.pl,goldenline.pl,inbound,001,World,1 +goldenopsafelist.com,zoothost.com,inbound,001,World,0 +goldstar.com,goldstar.com,inbound,001,World,1 +golfmnb.com,golfmnb.com,inbound,001,World,0 +golfnow.com,email-golfnow.com,inbound,001,World,0 +gomaji.com,gomaji.com,inbound,001,World,0 +goodgame.com,emsmtp.com,inbound,001,World,0 +goodlife.pt,emv8.com,inbound,001,World,0 +google.com,postini.com,inbound,001,World,0.703706 +googlegroups.com,postini.com,inbound,001,World,0.674075 +googlemail.com,t-ipconnect.de,inbound,001,World,0.999957 +gop.com,gop.com,inbound,001,World,0 +gopusamedia.com,gopusamedia.com,inbound,001,World,0 +govdelivery.com,govdelivery.com,inbound,001,World,0 +govdelivery.com,postini.com,inbound,001,World,0.089736 +governmentjobs.com,governmentjobs.com,inbound,001,World,0 +gpmailer.com.br,parperfeito.com,inbound,001,World,0 +grabone-mail-ie.com,grabone-mail-ie.com,inbound,001,World,0 +grabone-mail.com,grabone-mail.com,inbound,001,World,0 +grassrootsaction.com,grassfire.net,inbound,001,World,0 +gratka.pl,gratka.pl,inbound,001,World,0 +greatergood.com,greatergood.com,inbound,001,World,0 +gree.jp,gree.jp,inbound,001,World,0 +grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,001,World,0 +groopdealz.com,groopdealz.com,inbound,001,World,1 +groupalia.es,groupalia.es,inbound,001,World,0 +groupalia.it,groupalia.it,inbound,001,World,0 +groupon.jp,data-hotel.net,inbound,001,World,1e-06 +groupon.{...},chtah.net,inbound,001,World,0 +groupon.{...},groupon.{...},inbound,001,World,0.989844 +groupon.{...},postini.com,inbound,001,World,0.887392 +grouponmail.{...},grouponmail.{...},inbound,001,World,0 +grubhubmail.com,grubhubmail.com,inbound,001,World,0 +grupanya.com,euromsg.net,inbound,001,World,0 +grupos.com.br,grupos.com.br,inbound,001,World,0 +gtbank.com,gtbank.com,inbound,001,World,0.056121 +guess.ca,guess.com,inbound,001,World,0.000807 +guess.com,guess.com,inbound,001,World,0.003805 +guessfactory.com,guess.com,inbound,001,World,0.00164 +gumtree.com,marktplaats.nl,inbound,001,World,0 +gumtree.com.au,kijiji.com,inbound,001,World,0 +gunosy.com,gunosy.com,inbound,001,World,0.998682 +guruin.info,guru.net.in,inbound,001,World,1 +gurunavi.jp,gurunavi.jp,inbound,001,World,0 +gustazos.com,cityoferta.com,inbound,001,World,1 +gymglish.com,gymglish.com,inbound,001,World,0.000788 +habitaclia.com,splio.es,inbound,001,World,0.951406 +hallmark.com,hallmark.com,inbound,001,World,0 +hannaandersson.com,hannaandersson.com,inbound,001,World,0.013533 +harborfreightemail.com,harborfreightemail.com,inbound,001,World,0 +harristeetermail.com,harristeetermail.com,inbound,001,World,0.99673 +harvard.edu,harvard.edu,inbound,001,World,0.1751 +haskell.org,haskell.org,inbound,001,World,0.000703 +hautelook.com,hautelook.com,inbound,001,World,0.000459 +hayneedle.com,hayneedle.com,inbound,001,World,0 +hazteoir.org,hazteoir.org,inbound,001,World,0.31478 +hdfcbank.com,powerelay.com,inbound,001,World,1 +hdfcbank.net,powerelay.com,inbound,001,World,1 +hdfcbank.net,quickvmail.com,inbound,001,World,0 +helpareporter.net,helpareporter.com,inbound,001,World,0 +hepsiburada.com,euromsg.net,inbound,001,World,0 +herbalifemail.com,herbalifemail.com,inbound,001,World,0 +herculist.com,herculist.com,inbound,001,World,0 +heteml.jp,heteml.jp,inbound,001,World,0.950766 +hgtv.com,hgtv.com,inbound,001,World,0 +hh.ru,hh.ru,inbound,001,World,0.996236 +hhgreggemail.com,hhgreggemail.com,inbound,001,World,0 +hilton.com,hiltonemail.com,inbound,001,World,0 +hinet.net,hinet.net,inbound,001,World,0.007093 +hinet.net,hinet.net,outbound,001,World,0.00565 +hipchat.com,hipchat.com,inbound,001,World,1 +hipmunk.com,hipmunk.com,inbound,001,World,0 +hispavista.com,hispavista.com,inbound,001,World,0 +hln.be,persgroep-ops.net,inbound,001,World,0 +hm-f.jp,hm-f.jp,inbound,001,World,0 +hobsonsmail.com,hobsonsmail.com,inbound,001,World,0 +hollister-email.com,abercrombie-email.com,inbound,001,World,0 +home.ne.jp,zaq.ne.jp,inbound,001,World,9e-06 +homeaway.com,haspf.com,inbound,001,World,0 +homebaselife.com,ec-cluster.com,inbound,001,World,0 +homechoice.co.za,homechoice.co.za,inbound,001,World,0 +homedecorators.com,homedecorators.com,inbound,001,World,0 +homedepot.com,homedepot.com,inbound,001,World,1 +homedepotemail.com,homedepotemail.com,inbound,001,World,0 +honto.jp,honto.jp,inbound,001,World,0 +hootsuite.com,hootsuite.com,inbound,001,World,1 +horchowemail.com,horchowemail.com,inbound,001,World,0 +horoscope.com,center.com,inbound,001,World,2e-06 +hostelworld.com,bronto.com,inbound,001,World,0 +hostgator.com,hostgator.com,inbound,001,World,0.976131 +hostgator.com,websitewelcome.com,inbound,001,World,0.999822 +hotel.de,emp-mail.de,inbound,001,World,0 +hotels.com,hotels.com,inbound,001,World,0 +hotelurbano.com.br,allin.com.br,inbound,001,World,0 +hotmail.{...},hotmail.{...},inbound,001,World,0.999968 +hotmail.{...},hotmail.{...},outbound,001,World,1 +hotmail.{...},postini.com,inbound,001,World,0.837967 +hotornot.com,monopost.com,inbound,001,World,0.99472 +hotschedules.com,hotschedules.com,inbound,001,World,0 +hotspotmailer.com,hotspotmailer.com,inbound,001,World,1 +hotukdeals.com,hotukdeals.com,inbound,001,World,1 +hotwire.com,hotwire.com,inbound,001,World,0 +house.gov,house.gov,inbound,001,World,0.999966 +houseoffraser.co.uk,houseoffraser.co.uk,inbound,001,World,0 +houzz.com,houzz.com,inbound,001,World,1 +hp.com,hp.com,inbound,001,World,0.202841 +hpnotifier.nl,hpnotifier.nl,inbound,001,World,0 +hsbc.co.in,hsbc.com.hk,inbound,001,World,1 +hsbc.com.hk,hsbc.com.hk,inbound,001,World,1 +hsn.com,hsn.com,inbound,001,World,0 +htcampusmailer.com,eccluster.com,inbound,001,World,0 +hubspot.com,hubspot.com,inbound,001,World,1 +huinforma.com.br,huinforma.com.br,inbound,001,World,0 +hulumail.com,hulumail.com,inbound,001,World,0 +hungry-girl.com,hungry-girl.com,inbound,001,World,0 +hungryhouse.co.uk,mxmfb.com,inbound,001,World,0 +huntington.com,huntington.com,inbound,001,World,0.987351 +i-part.com.tw,i-part.com.tw,inbound,001,World,0.001865 +i-say.com,ipsos-interactive.com,inbound,001,World,1 +iamlgnd2.com,iamlgnd2.com,inbound,001,World,1 +ibm.com,ibm.com,inbound,001,World,0.94413 +ibps.in,sify.net,inbound,001,World,0 +ibpsorg.org,sify.net,inbound,001,World,0 +ibsys.com,ibsys.com,inbound,001,World,9e-05 +icbc.com.ar,clickexperts.net,inbound,001,World,0 +icbc.com.ar,standardbank.com.ar,inbound,001,World,0 +icelandmail.co.uk,emsg-live.co.uk,inbound,001,World,0 +icicibank.com,icicibank.com,inbound,001,World,0.009835 +icicisecurities.com,icicibank.com,inbound,001,World,0.000803 +icims.com,icims.com,inbound,001,World,0.999939 +icloud.com,apple.com,inbound,001,World,1 +icloud.com,icloud.com,outbound,001,World,1 +icloud.com,mac.com,inbound,001,World,1 +icloud.com,me.com,inbound,001,World,0.999995 +icors.org,lsoft.us,inbound,001,World,0 +icpbounce.com,icpbounce.com,inbound,001,World,0 +idc.email,nmsrv.com,inbound,001,World,1 +idealista.com,idealista.com,inbound,001,World,0.001627 +ideascost.com,ramcorp.in,inbound,001,World,0 +idgconnect-resources.com,idgconnect-resources.com,inbound,001,World,0 +ieee.org,ieee.org,inbound,001,World,0.999912 +ifttt.com,ifttt.com,inbound,001,World,1 +ig.com.br,ig.com.br,inbound,001,World,0 +ig.com.br,ig.com.br,outbound,001,World,0 +ign.com,ign.com,inbound,001,World,0 +ignitionsender.com,ignitionsender.com,inbound,001,World,0 +igot-mails.com,zoothost.com,inbound,001,World,0 +iheart.com,iheart.com,inbound,001,World,0 +iimjobs.com,iimjobs.com,inbound,001,World,1 +ikmultimedianews.com,ikmultimedianews.com,inbound,001,World,0.993319 +illinois.edu,illinois.edu,inbound,001,World,0.866481 +imageshost.ca,imageshost.ca,inbound,001,World,0 +imakenews.net,imakenews.com,inbound,001,World,0 +imi.ne.jp,lifemedia.jp,inbound,001,World,0 +immobilienscout24.de,immobilienscout24.de,inbound,001,World,1 +imo.im,imo.im,inbound,001,World,1 +imodules.com,imodules.com,inbound,001,World,0 +imvu.com,imvu.com,inbound,001,World,7e-06 +in-boxpays.com,in-boxpays.com,inbound,001,World,0 +inboxair.com,inboxair.com,inbound,001,World,0 +inboxdollars.com,inboxdollars.com,inbound,001,World,0 +inboxfirst.com,inboxfirst.com,inbound,001,World,0 +inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,001,World,0.999877 +inboxpays.com,inboxpays.com,inbound,001,World,0 +inboxpounds.co.uk,inboxpounds.co.uk,inbound,001,World,0 +indeed.com,indeed.com,inbound,001,World,0.000121 +indeedemail.com,indeedemail.com,inbound,001,World,0 +independentlivingbullion.com,independentlivingbullion.com,inbound,001,World,0 +indiamart.com,indiamart.com,inbound,001,World,1 +indiaproperty.com,indiaproperty.com,inbound,001,World,0 +indiatimes.com,speakingtree.in,inbound,001,World,0 +indiatimeshop.com,sendpal.in,inbound,001,World,0 +indieroyale.com,desura.com,inbound,001,World,1 +infibeam.com,eccluster.com,inbound,001,World,0 +infobradesco.com.br,infobradesco.com.br,inbound,001,World,0 +infoempleo.com,infoempleo.com,inbound,001,World,0 +infojobs.com.br,anuntis.com,inbound,001,World,0 +infojobs.it,infojobs.it,inbound,001,World,0 +infojobs.net,infojobs.net,inbound,001,World,0 +infomoney.com.br,infomoney.com.br,inbound,001,World,1 +infopanel.jp,mailds.jp,inbound,001,World,0 +infopraca.pl,careesma.com,inbound,001,World,0 +informz.net,informz.net,inbound,001,World,0 +infos-micromania.com,infos-micromania.com,inbound,001,World,0 +infosephora.com,splio.com,inbound,001,World,0.962167 +infoworld.com,infoworld.com,inbound,001,World,0 +infradead.org,infradead.org,inbound,001,World,1 +infusionmail.com,infusionmail.com,inbound,001,World,0 +ingdirect.es,ingdirect.es,inbound,001,World,1 +inman.com,inman.com,inbound,001,World,0 +inmotionhosting.com,inmotionhosting.com,inbound,001,World,0.990051 +innovyx.net,innovyx.net,inbound,001,World,0 +ino.com,ino.com,inbound,001,World,0.999981 +insidehook.com,sailthru.com,inbound,001,World,0 +instagram.com,facebook.com,inbound,001,World,1 +instantprofitlist.com,screenshotads.com,inbound,001,World,0 +intage.co.jp,intage.co.jp,inbound,001,World,0.00557 +inter-chat.com,inter-chat.com,inbound,001,World,0 +interac.ca,certapay.com,inbound,001,World,0 +interactivebrokers.com,interactivebrokers.com,inbound,001,World,1 +interactiverealtyservices.com,interactiverealtyservices.com,inbound,001,World,0 +intercom.io,mailgun.info,inbound,001,World,1 +interdatesa.com,fagms.net,inbound,001,World,0 +interealty.net,interealty.net,inbound,001,World,1 +internations.org,internations.org,inbound,001,World,1 +interweave.com,interweave.com,inbound,001,World,0 +interwell.gr,interwell.gr,inbound,001,World,0 +intliv2.net,internationalliving.com,inbound,001,World,0 +intuit.com,intuit.com,inbound,001,World,0.89114 +invalidemail.com,taleo.net,inbound,001,World,1 +investopedia.com,vclk.net,inbound,001,World,0.999999 +investorplace.com,investorplace.com,inbound,001,World,0.995093 +inx1and1.de,1and1.com,inbound,001,World,1 +inxserver.com,inxserver.de,inbound,001,World,0.952863 +inxserver.de,inxserver.de,inbound,001,World,0.980838 +ipcmedia.co.uk,ipcmedia.co.uk,inbound,001,World,0 +iqelite.com,iqelite.com,inbound,001,World,0 +irctcshopping.com,chtah.net,inbound,001,World,0 +iridium.com,iridium.com,inbound,001,World,0.997882 +isbank.com.tr,isbank.com.tr,inbound,001,World,0.009655 +isendservice.com.br,isendservice.com.br,inbound,001,World,0 +itau-unibanco.com.br,itau.com.br,inbound,001,World,0 +itms.in.ua,itms.in.ua,inbound,001,World,0 +itsmyascent.com,itsmyascent.com,inbound,001,World,0 +ittoolbox.com,ittoolbox.com,inbound,001,World,0 +ittoolbox.com,toolbox.com,inbound,001,World,0 +itunes.com,apple.com,inbound,001,World,0.082921 +itwhitepapers.com,itwhitepapers.com,inbound,001,World,0 +iwantoneofthose.com,thehut.com,inbound,001,World,0 +ixs1.net,ixs1.net,inbound,001,World,0.005131 +jackwills.com,jackwills.com,inbound,001,World,0 +jalag.de,jalag.de,inbound,001,World,1 +jane.com,jane.com,inbound,001,World,1 +jango.com,jango.com,inbound,001,World,0 +jared.com,jared.com,inbound,001,World,0 +jcity.com,jcity.com,inbound,001,World,0 +jcpenney.com,jcpenney.com,inbound,001,World,9e-06 +jdate.com,postdirect.com,inbound,001,World,0 +jeevansathi.com,jeevansathi.com,inbound,001,World,0 +jetprivilege.com,jetprivilege.com,inbound,001,World,0 +jetsetter.com,smartertravelmedia.com,inbound,001,World,0.041362 +jeuxvideo.com,jeuxvideo.com,inbound,001,World,0.148921 +jeweloscoemail.com,email-mywebgrocer2.com,inbound,001,World,0 +jibjab.com,storybots.com,inbound,001,World,0 +jira.com,uc-inf.net,inbound,001,World,1 +jiscmail.ac.uk,lsoft.se,inbound,001,World,0 +joann-mail.com,joann-mail.com,inbound,001,World,0 +jobinsider.com,jobinsider.com,inbound,001,World,0 +jobinthailand.com,jobinthailand.com,inbound,001,World,1 +jobisjob.com,jobisjob.com,inbound,001,World,0 +jobmaster.co.il,jobmaster1.co.il,inbound,001,World,0 +jobomas.com,jobomas.com,inbound,001,World,1 +jobrapidoalert.com,jobrapidoalert.com,inbound,001,World,0 +jobs2web.com,ondemand.com,inbound,001,World,1 +jobscentral.com.sg,mailgun.net,inbound,001,World,1 +jobsdbalert.co.id,jobsdbalert.co.id,inbound,001,World,0 +jobsdbalert.com,jobsdbalert.com,inbound,001,World,0 +jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,001,World,0 +jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,001,World,0 +jobserve.com,jobserve.com,inbound,001,World,0 +jobsindubai.com,jobsindubai.ca,inbound,001,World,0 +jobsite.co.uk,jobsite.co.uk,inbound,001,World,0.000509 +jobson.com,jobsonmail.com,inbound,001,World,0 +jobsradar.com,jobsradar.com,inbound,001,World,0 +jobstreet.com,jobstreet.com,inbound,001,World,0 +jockeycomfort.com,jockeycomfort.com,inbound,001,World,0 +johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,001,World,0 +jomashop.com,lstrk.net,inbound,001,World,1 +joobmailer.com,joobmailer.com,inbound,001,World,0 +josbank.com,josbank.com,inbound,001,World,0 +joshin.co.jp,joshin.co.jp,inbound,001,World,8e-06 +jossandmain.com,jossandmain.com,inbound,001,World,0 +jpcycles.com,jpcycles.com,inbound,001,World,0.001716 +jtv.com,jtv.com,inbound,001,World,0 +jumia.com.ng,fagms.de,inbound,001,World,0 +jungleerummy.com,jungleerummy.com,inbound,001,World,1 +juno.com,untd.com,inbound,001,World,0 +juno.com,untd.com,outbound,001,World,0 +jusbrasil.com.br,jusbrasil.com.br,inbound,001,World,0 +just-eat.co.uk,ec-cluster.com,inbound,001,World,0 +justclick.ru,justclick.ru,inbound,001,World,0 +justdial.com,iaires.com,inbound,001,World,0 +justdial.com,mailurja.com,inbound,001,World,0 +justfab.com,bronto.com,inbound,001,World,0 +justfab.fr,bronto.com,inbound,001,World,0 +k1speed.com,k1speed.com,inbound,001,World,1 +kagoya.net,kagoya.net,inbound,001,World,0.013097 +kalunga.com.br,kalunga.com.br,inbound,001,World,0 +karvy.com,karvy.com,inbound,001,World,0.045186 +kasikornbank.com,kasikornbank.com,inbound,001,World,0 +kaskusnetworks.com,kaskus.com,inbound,001,World,0 +kay.com,kay.com,inbound,001,World,0 +keek.com,keek.com,inbound,001,World,1 +kernel.org,kernel.org,inbound,001,World,0 +kgbdeals.co.uk,email1-kgbdeals.com,inbound,001,World,0 +kgstores.com,kgstores.com,inbound,001,World,0 +kiabi.com,dms-02.net,inbound,001,World,0 +kickstarter.com,kickstarter.com,inbound,001,World,1 +kidsfootlocker.com,footlocker.com,inbound,001,World,0 +kijiji.ca,kijiji.com,inbound,001,World,0 +kik.com,kik.com,inbound,001,World,1 +kimblegroup.com,kimblegroup.com,inbound,001,World,1 +kintera.com,kintera.com,inbound,001,World,0 +kismia.com,kismia.com,inbound,001,World,1 +kiwari.com,kiwari.com,inbound,001,World,9e-06 +klaviyomail.com,klaviyomail.com,inbound,001,World,1 +kliksa.net,euromsg.net,inbound,001,World,0 +kliktoday.com,kliktoday.com,inbound,001,World,0 +klm-mail.com,klm-mail.com,inbound,001,World,0 +klove.com,emfbroadcasting.com,inbound,001,World,0 +kohls.com,kohls.com,inbound,001,World,0 +komando.com,komando.com,inbound,001,World,0 +kongregate.com,kongregate.com,inbound,001,World,0 +kotak.com,kotak.com,inbound,001,World,0.115651 +kp.org,kp.org,inbound,001,World,0.999975 +krogermail.com,bigfootinteractive.com,inbound,001,World,0 +krs.bz,tricorn.net,inbound,001,World,0 +kubra.com,kubra.com,inbound,001,World,1.8e-05 +kundenserver.de,kundenserver.de,inbound,001,World,1 +kvbmail.com,kvbmail.com,inbound,001,World,0 +la-meteo-mail.fr,splio.com,inbound,001,World,1 +laaptuemail.com,laaptuemail.com,inbound,001,World,0 +lakewoodchurch.com,lakewoodchurch.com,inbound,001,World,0 +lancers.jp,lancers.jp,inbound,001,World,0 +landmarketingmailer.com,zoothost.com,inbound,001,World,0 +landofnod.com,landofnod.com,inbound,001,World,0.002525 +landsend.com,email-landsend.com,inbound,001,World,0 +landsend.com,postdirect.com,inbound,001,World,0 +languagepod101.com,eclient10.com,inbound,001,World,0 +languagepod101.com,eddlvr.com,inbound,001,World,0 +languagepod101.com,ednwsltr3.com,inbound,001,World,0 +languagepod101.com,ednwsltr8.com,inbound,001,World,0 +languagepod101.com,emaildirect.net,inbound,001,World,0 +laposte.net,laposte.net,inbound,001,World,0.271722 +laposte.net,laposte.net,outbound,001,World,0 +laptuinvite.com,laptuinvite.com,inbound,001,World,0 +laredoute.fr,laredoute.fr,inbound,001,World,0 +lasenza.com,lasenza.com,inbound,001,World,0 +lastcallemail.com,lastcallemail.com,inbound,001,World,0 +lastminute.com,lastminute.com,inbound,001,World,0.00831 +laterooms.com,laterooms.com,inbound,001,World,0.014746 +latimes.com,latimes.com,inbound,001,World,0 +lauraashley.com,lauraashley.com,inbound,001,World,0 +lazerhits.com,lazerhits.com,inbound,001,World,1 +leadercontato.com.br,leadercontato.com.br,inbound,001,World,0 +leboncoin.fr,leboncoin.fr,inbound,001,World,0 +lefigaro.fr,splio.com,inbound,001,World,1 +leftlanesports.com,auspient.com,inbound,001,World,0.00705 +leftlanesports.com,leftlanesports.com,inbound,001,World,0 +legalshieldassociate.com,legalshield.com,inbound,001,World,0 +lelong.my,lelong.com.my,inbound,001,World,1 +lelong.my,lelong.net.my,inbound,001,World,1 +lemonde.fr,lemonde.fr,inbound,001,World,0.000111 +leparisien.fr,leparisien.fr,inbound,001,World,0 +lexico.com,lexico.com,inbound,001,World,0 +lexpress.fr,bp06.net,inbound,001,World,0 +libero.it,libero.it,inbound,001,World,0.00024 +libero.it,libero.it,outbound,001,World,0 +life360.com,life360.com,inbound,001,World,1 +lifecare-news.com,email-lifecare.com,inbound,001,World,0 +lifecooler.com,1-hostingservice.com,inbound,001,World,0 +lifemiles.com,bigfootinteractive.com,inbound,001,World,0 +lifescript.com,ilinkmd.com,inbound,001,World,0 +lindenlab.com,lindenlab.com,inbound,001,World,0.999444 +line.me,naver.com,inbound,001,World,1 +line6.com,line6.com,inbound,001,World,2.7e-05 +linkedin.com,linkedin.com,inbound,001,World,0.998882 +linkedin.com,postini.com,inbound,001,World,0.835872 +linkedin.com,yahoo.{...},inbound,001,World,0.999927 +linkshare.com,linksynergy.com,inbound,001,World,0 +liquidation.com,liquidation.com,inbound,001,World,6e-06 +listadventure.com,adlabsinc.com,inbound,001,World,0 +listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,001,World,0.00023 +listeneremail.net,listeneremail.net,inbound,001,World,0 +listia.com,listia.com,inbound,001,World,1 +listjoe.com,adlabsinc.com,inbound,001,World,0 +listnerds.com,listnerds.com,inbound,001,World,0 +listreturn.com,zoothost.com,inbound,001,World,0 +listserve.com,listserve.com,inbound,001,World,0 +listvolta.com,listvolta.com,inbound,001,World,0 +listwire.com,listwire.com,inbound,001,World,0 +litres.ru,litres.ru,inbound,001,World,1 +live.{...},hotmail.{...},inbound,001,World,0.999954 +live.{...},hotmail.{...},outbound,001,World,1 +livedoor.com,livedoor.com,inbound,001,World,0 +livefyre.com,andbit.net,inbound,001,World,1 +livejournal.com,livejournal.com,inbound,001,World,0 +livemailservice.com,livemailservice.com,inbound,001,World,0 +livenation.com,exacttarget.com,inbound,001,World,0 +livescribe.com,bronto.com,inbound,001,World,0 +livingsocial.com,livingsocial.com,inbound,001,World,0 +livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,001,World,0 +lmlmgv.com.br,gvarev.com.br,inbound,001,World,0 +localhires.com,localhires.com,inbound,001,World,1 +loccitane.com,neolane.net,inbound,001,World,0 +loft.com,anntaylor.com,inbound,001,World,0 +logentries.com,logentries.com,inbound,001,World,0 +logitech.com,dvsops.com,inbound,001,World,0 +logmein.com,logmein.com,inbound,001,World,0.031467 +lojasmarisa.com.br,lojasmarisa.com.br,inbound,001,World,0 +lolsolos.com,ultimateadsites.net,inbound,001,World,0.999996 +lombardipublishing.com,lombardipublishing.com,inbound,001,World,0 +lonelywifehookup.com,iverificationsystems.com,inbound,001,World,0 +lookout.com,lookout.com,inbound,001,World,1 +lordandtaylor.com,lordandtaylor.com,inbound,001,World,0 +loveaholics.com,ropot.net,inbound,001,World,0 +lovelywholesale.com,lovelywholesale.com,inbound,001,World,1 +loveplanet.ru,pochta.ru,inbound,001,World,0.640035 +lowcostholidays.co.uk,communicatoremail.com,inbound,001,World,0 +lrsmail.com,lrsmail.com,inbound,001,World,0 +lsi.com,postini.com,inbound,001,World,0.981121 +lt02.net,listrak.com,inbound,001,World,1 +lt02.net,lstrk.net,inbound,001,World,1 +ltdcommodities.com,ltdcomm.net,inbound,001,World,0 +lua.org,pepperfish.net,inbound,001,World,1 +luckymag.com,mkt4500.com,inbound,001,World,0 +ludokados.com,ludokado.com,inbound,001,World,0 +lulu.com,bronto.com,inbound,001,World,0 +lulus.com,lstrk.net,inbound,001,World,1 +lumosity.com,lumosity.com,inbound,001,World,1 +luxa.jp,luxa.jp,inbound,001,World,0 +lynxmail.in,iaires.com,inbound,001,World,0 +lyris.net,lyris.net,inbound,001,World,0 +lyst.com,lyst.com,inbound,001,World,1 +m1e.net,m1e.net,inbound,001,World,0.000342 +m3.com,m3.com,inbound,001,World,0 +mac.com,icloud.com,outbound,001,World,1 +mac.com,mac.com,inbound,001,World,1 +maccosmetics.com,esteelauder.com,inbound,001,World,0 +macromill.com,macromill.com,inbound,001,World,1e-06 +macupdate.com,mailgun.info,inbound,001,World,1 +macys.com,macys.com,inbound,001,World,0 +madmels.info,ultimateadsites.net,inbound,001,World,1 +madmimi.com,madmimi.com,inbound,001,World,0 +mag2.com,tandem-m.com,inbound,001,World,0 +magicbricks.com,tbsl.in,inbound,001,World,0 +magicjack.com,magicjack.com,inbound,001,World,1 +magix.net,magix.net,inbound,001,World,0.673176 +magnetdev.com,magnetmail.net,inbound,001,World,0 +mail-backcountry.com,email-bcmarketing.com,inbound,001,World,0 +mail-boss.com,mail-boss.com,inbound,001,World,0 +mail-cdiscount.com,mail-cdiscount.com,inbound,001,World,0 +mail-mbank.pl,mail-mbank.pl,inbound,001,World,0 +mail-route.com,mail-route.com,inbound,001,World,0 +mail-thestreet.com,mail-thestreet.com,inbound,001,World,0 +mail.mil,mail.mil,inbound,001,World,0 +mail.ru,mail.ru,inbound,001,World,0.987506 +mail.ru,mail.ru,outbound,001,World,0.00674 +mailaccurate.com,mgenie.in,inbound,001,World,0 +mailchimp.com,mailchimp.com,inbound,001,World,0.967415 +maileclipse.com,emce2.in,inbound,001,World,0 +mailengine1.com,mailengine1.com,inbound,001,World,0 +mailer-service.de,mailer-service.de,inbound,001,World,4.9e-05 +mailer4u.in,elabs10.com,inbound,001,World,0 +mailersend.com,mailersend.com,inbound,001,World,0 +mailfacil.com.br,md02.com,inbound,001,World,0 +mailfeast.com,mgenie.in,inbound,001,World,0 +mailgun.org,mailgun.info,inbound,001,World,1 +mailgun.org,mailgun.net,inbound,001,World,1 +mailgun.org,mailgun.us,inbound,001,World,1 +mailing-list.it,mailing-list.it,inbound,001,World,0 +mailingathome.net,mailingathome.net,inbound,001,World,0.999995 +mailjayde.com,mailjayde.com,inbound,001,World,0 +mailjet.com,mailjet.com,inbound,001,World,0.436025 +mailmachine1050.com,mailmachine1050.com,inbound,001,World,0 +mailmailmail.net,mailmailmail.net,inbound,001,World,0 +mailoct.in,tcmailer14.in,inbound,001,World,0 +mailoct1.in,mailoct1.in,inbound,001,World,0 +mailoct1.in,myntramail2.in,inbound,001,World,0 +mailorama.fr,mailorama.fr,inbound,001,World,0 +mailplus.nl,brightbase.net,inbound,001,World,1 +mailpost.in,iaires.com,inbound,001,World,0 +mailpv.net,pvmailer.net,inbound,001,World,1 +mailquant.com,iaires.com,inbound,001,World,0 +mailsend1.com,mailsend6.com,inbound,001,World,0 +mailsender.com.br,mailsender.com.br,inbound,001,World,0 +maisonsdumonde.com,bp06.net,inbound,001,World,0 +makro.nl,srv2.de,inbound,001,World,0.88256 +manager.com.br,manager.com.br,inbound,001,World,0 +mandrillapp.com,backpage.com,inbound,001,World,1 +mandrillapp.com,mandrillapp.com,inbound,001,World,1 +mandrillapp.com,mcsignup.com,inbound,001,World,1 +mandrillapp.com,myjobhelperalerts.com,inbound,001,World,1 +mango.com,emstechnology2.net,inbound,001,World,0 +manipal.edu,iaires.com,inbound,001,World,0 +manta.com,exacttarget.com,inbound,001,World,0 +mapfre.com,emv5.com,inbound,001,World,0 +mar0.net,mar0.net,inbound,001,World,0.976409 +marcustheatres.com,movio.co,inbound,001,World,0 +markandgraham.com,markandgraham.com,inbound,001,World,0 +markavip.com,markavip.com,inbound,001,World,0 +marketer-safelist.com,jsalfianmarketing.com,inbound,001,World,1 +marketinghq.net,elabs8.com,inbound,001,World,0 +marketingprofs.com,marketingprofs.com,inbound,001,World,0.004412 +marketingstudio.com,marketingstudio.com,inbound,001,World,0 +marksandspencer.com,marksandspencer.com,inbound,001,World,0 +marktplaats.nl,marktplaats.nl,inbound,001,World,0 +marlboro.com,marlboro.com,inbound,001,World,0 +maropost.com,biotrustnews.com,inbound,001,World,0 +maropost.com,mailing-truthaboutabs.com,inbound,001,World,0 +maropost.com,maropost.com,inbound,001,World,0 +maropost.com,mp2200.com,inbound,001,World,0 +maropost.com,mp2201.com,inbound,001,World,0 +maropost.com,survivallife.com,inbound,001,World,0 +marykay.com,marykay.com,inbound,001,World,0 +masivapp.com,masivapp.com,inbound,001,World,1 +massageenvyclinics.com,massageenvyclinics.com,inbound,001,World,0 +masterbase.com,masterbase.com,inbound,001,World,0 +mastercard-email.com,mastercard-email.com,inbound,001,World,0 +match.com,match.com,inbound,001,World,0 +matchwereld.nl,matchwereld.nl,inbound,001,World,0 +mate1.net,mate1.net,inbound,001,World,0 +matrixemailer.com,matrixemailer.com,inbound,001,World,0 +maxpark.com,gidepark.ru,inbound,001,World,1 +mbga.jp,mbga.jp,inbound,001,World,0 +mbna.co.uk,ec-cluster.com,inbound,001,World,0 +mbounces.com,emdbms.com,inbound,001,World,0 +mbstrm.com,mobilestorm.com,inbound,001,World,0 +mcafee.com,mcafee.com,inbound,001,World,0.963535 +mcarthurglen.com,mcarthurglen.com,inbound,001,World,0 +mcdlv.net,mcdlv.net,inbound,001,World,0 +mcdlv.net,postini.com,inbound,001,World,0.082163 +mckinsey.com,bigfootinteractive.com,inbound,001,World,0 +mcsv.net,mcsv.net,inbound,001,World,0 +mcsv.net,postini.com,inbound,001,World,0.101883 +mdirector.com,mdrctr.com,inbound,001,World,0 +mdlinx.com,mdlinx.com,inbound,001,World,0 +me.com,icloud.com,outbound,001,World,1 +me.com,mac.com,inbound,001,World,1 +mec.gov.br,mec.gov.br,inbound,001,World,0 +mecumauction.com,mecumauction.com,inbound,001,World,0 +medallia.com,medallia.com,inbound,001,World,0.999682 +mediabistro.com,iworld.com,inbound,001,World,0.006428 +mediapost.com,mediapost.com,inbound,001,World,0 +medium.com,messagebus.com,inbound,001,World,0 +medpagetoday.com,wc09.net,inbound,001,World,0 +medscape.com,medscape.com,inbound,001,World,0 +meetic.com,meetic.com,inbound,001,World,0 +meetmemail.com,meetmemail.com,inbound,001,World,0 +meetup.com,meetup.com,inbound,001,World,5e-06 +megasenders.com,megasenders.com,inbound,001,World,0.07662 +melaleuca.com,melaleuca.com,inbound,001,World,0.003493 +memberdealsusa.com,memberdealsusa.com,inbound,001,World,0 +menswearhouse.com,menswearhouse.com,inbound,001,World,0 +mequedouno.com,mequedouno.com,inbound,001,World,0 +mercadojobs.com,sendgrid.net,inbound,001,World,1 +mercadolibre.com,mercadolibre.com,inbound,001,World,0 +mercadolivre.com,mercadolibre.com,inbound,001,World,0 +merceworld.com,merceworld.com,inbound,001,World,0.996738 +mercola.com,mercola.com,inbound,001,World,0.000369 +merodea.me,sendgrid.net,inbound,001,World,1 +messagegears.net,messagegears.net,inbound,001,World,0 +met-art.com,hydentra.com,inbound,001,World,1 +metro.co.in,srv2.de,inbound,001,World,0.966947 +metrodeal.com,fagms.de,inbound,001,World,0 +mgmresorts.com,mgmresorts.com,inbound,001,World,0 +mgo.com,bronto.com,inbound,001,World,0 +michaels.com,chtah.net,inbound,001,World,0 +michaels.com,michaels.com,inbound,001,World,0 +microcentermedia.com,bfi0.com,inbound,001,World,0 +microsoft.com,hotmail.{...},inbound,001,World,1 +microsoft.com,msn.com,inbound,001,World,1 +microsoftemail.com,microsoftemail.com,inbound,001,World,0 +microsoftemail.com,microsoftstoreemail.com,inbound,001,World,0 +midnightsunsafelist.com,zoothost.com,inbound,001,World,0 +mightydeals.co.uk,mightydeals.co.uk,inbound,001,World,0 +mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,001,World,0 +milfaholic.com,iverificationsystems.com,inbound,001,World,0 +miltnews.com,miltnews.com,inbound,001,World,0 +mindbodyonline.com,mindbodyonline.com,inbound,001,World,1 +mindfieldonline.com,mindfieldonline.com,inbound,001,World,0 +mindmoviesmail.com,mindmoviesmail.com,inbound,001,World,0.004239 +mindvalleymail3.com,mindvalleymail3.com,inbound,001,World,0 +minhavida.com.br,minhavida.com.br,inbound,001,World,0 +mint.com,mint.com,inbound,001,World,0 +minted.com,messagelabs.com,inbound,001,World,0.999669 +mirtesen.ru,mtml.ru,inbound,001,World,0 +missselfridge.com,wallis-fashion.com,inbound,001,World,0 +mistersafelist.com,zoothost.com,inbound,001,World,0 +mit.edu,mit.edu,inbound,001,World,0.868705 +mitula.net,mitula.org,inbound,001,World,0 +mitula.org,mitula.org,inbound,001,World,0 +mixcloudmail.com,mixcloudmail.com,inbound,001,World,0.995482 +mixi.jp,mixi.jp,inbound,001,World,0 +mjinn.com,mailurja.com,inbound,001,World,0 +mkt015.com,mkt015.com,inbound,001,World,0 +mkt022.com,mkt022.com,inbound,001,World,0 +mkt063.com,mkt063.com,inbound,001,World,0 +mkt1136.com,mkt1136.com,inbound,001,World,0 +mkt1985.com,fmlinks.net,inbound,001,World,0 +mkt2010.com,mkt2010.com,inbound,001,World,0 +mkt2106.com,mkt2106.com,inbound,001,World,0 +mkt2170.com,mkt2170.com,inbound,001,World,0 +mkt2181.com,mkt2181.com,inbound,001,World,0 +mkt2615.com,mkt2615.com,inbound,001,World,0 +mkt2813.com,mkt2813.com,inbound,001,World,0 +mkt2944.com,mkt2944.com,inbound,001,World,0 +mkt3134.com,mkt3134.com,inbound,001,World,0 +mkt3142.com,mkt3142.com,inbound,001,World,0 +mkt3156.com,mkt3156.com,inbound,001,World,0 +mkt3203.com,mkt3203.com,inbound,001,World,0 +mkt346.com,mkt346.com,inbound,001,World,0 +mkt3544.com,mkt3544.com,inbound,001,World,0 +mkt3622.com,mkt3622.com,inbound,001,World,0 +mkt3682.com,mkt3682.com,inbound,001,World,0 +mkt3690.com,mkt3690.com,inbound,001,World,0 +mkt3695.com,mkt3695.com,inbound,001,World,0 +mkt3804.com,mkt3804.com,inbound,001,World,0 +mkt3815.com,mkt3815.com,inbound,001,World,0 +mkt3952.com,xoom.com,inbound,001,World,0 +mkt4355.com,mkt4355.com,inbound,001,World,0 +mkt4364.com,mkt4364.com,inbound,001,World,0 +mkt459.com,mkt459.com,inbound,001,World,0 +mkt4701.com,mkt4701.com,inbound,001,World,0 +mkt4728.com,mkt4728.com,inbound,001,World,0 +mkt4731.com,mkt4731.com,inbound,001,World,0 +mkt4738.com,mkt4738.com,inbound,001,World,0 +mkt5071.com,mkt5071.com,inbound,001,World,0 +mkt5098.com,mkt5098.com,inbound,001,World,0 +mkt5131.com,mkt5131.com,inbound,001,World,0 +mkt5144.com,mkt5144.com,inbound,001,World,0 +mkt5144.com,mkt5980.com,inbound,001,World,0 +mkt5144.com,mkt5981.com,inbound,001,World,0 +mkt5181.com,mkt5181.com,inbound,001,World,0 +mkt5269.com,mkt5269.com,inbound,001,World,0 +mkt529.com,mkt529.com,inbound,001,World,0 +mkt5297.com,mkt5297.com,inbound,001,World,0 +mkt5297.com,mkt5309.com,inbound,001,World,0 +mkt5371.com,mkt5371.com,inbound,001,World,0 +mkt5806.com,mkt5806.com,inbound,001,World,0 +mkt5934.com,mkt5934.com,inbound,001,World,0 +mkt5937.com,mkt5937.com,inbound,001,World,0 +mkt5970.com,mkt5970.com,inbound,001,World,0 +mkt6100.com,mkt6098.com,inbound,001,World,0 +mkt6276.com,mkt6276.com,inbound,001,World,0 +mkt6323.com,mkt6323.com,inbound,001,World,0 +mkt746.com,mkt746.com,inbound,001,World,0 +mkt824.com,mkt869.com,inbound,001,World,0 +mktdillards.com,mktdillards.com,inbound,001,World,0 +mktid10.com,1-hostingservice.com,inbound,001,World,0 +mktomail.com,mktdns.com,inbound,001,World,0 +mktomail.com,mktomail.com,inbound,001,World,0 +mktomail.com,mktroute.com,inbound,001,World,0 +ml.com,bankofamerica.com,inbound,001,World,1 +mlgns.com,mlgns.com,inbound,001,World,0 +mlgnserv.com,mlgnserv.com,inbound,001,World,0 +mlsend.com,mlsend.com,inbound,001,World,0 +mlsend2.com,mlsend2.com,inbound,001,World,0 +mlssoccer.com,mlssoccer.com,inbound,001,World,0 +mmaco.net,mmaco.net,inbound,001,World,0.999998 +mmagic.jp,mmagic.jp,inbound,001,World,0.000531 +mmks.it,mail-maker.it,inbound,001,World,0 +mmorpg.com,mmorpg.com,inbound,001,World,0.002979 +mmsecure.nl,donenad.nl,inbound,001,World,0 +mo1send.com,mo1send.com,inbound,001,World,0 +mobile01.com,mobile01.com,inbound,001,World,0 +mobly.com.br,mobly.com.br,inbound,001,World,0 +mocospace.com,mocospace.com,inbound,001,World,0 +modellsemail.com,n-email.net,inbound,001,World,0 +modnakasta.ua,emv5.com,inbound,001,World,0 +monex.co.jp,monex.co.jp,inbound,001,World,0.019424 +moneycontrol.com,active18.com,inbound,001,World,0 +moneyforward.com,moneyforward.com,inbound,001,World,0 +moneymorning.com,moneymappress.com,inbound,001,World,0 +moneysupermarketmail.com,moneysupermarketmail.com,inbound,001,World,0 +monipla.jp,aainc.co.jp,inbound,001,World,0 +monografias.com,elistas.net,inbound,001,World,0 +monster.co.in,monster.co.in,inbound,001,World,0 +monster.com,monster.com,inbound,001,World,0.000231 +monster.com,tmpw.net,inbound,001,World,0.000503 +monsterindia.com,monster.co.in,inbound,001,World,0 +moon-ray.com,moon-ray.com,inbound,001,World,0.001892 +mooply.co,mailendo.com,inbound,001,World,0 +mooresclothing.com,mooresclothing.com,inbound,001,World,0 +morhipo.com,euromsg.net,inbound,001,World,0 +morningstar.net,morningstar.net,inbound,001,World,0 +mothercaregroup.com,neolane.net,inbound,001,World,0 +motosnap.com,motosnap.com,inbound,001,World,0.994795 +moveon.org,moveon.org,inbound,001,World,0.99577 +moviestarplanet.com,moviestarplanet.com,inbound,001,World,0 +mozilla.org,mozilla.com,inbound,001,World,0.633237 +mpme.jp,mpme.jp,inbound,001,World,0 +mpse.jp,emsaqua.jp,inbound,001,World,0 +mpse.jp,emsbeige.jp,inbound,001,World,0 +mpse.jp,emsbrown.jp,inbound,001,World,0 +mpse.jp,emscyan.jp,inbound,001,World,0 +mpse.jp,emsgold.jp,inbound,001,World,0 +mpse.jp,emslime.jp,inbound,001,World,0 +mpse.jp,emsnavy.jp,inbound,001,World,0 +mpse.jp,emspink.jp,inbound,001,World,0 +mpse.jp,emssnow.jp,inbound,001,World,0 +mpse.jp,mpme.jp,inbound,001,World,0 +mpse.jp,yahoo.co.jp,inbound,001,World,0 +mpsnd.ch,agenceweb.net,inbound,001,World,0 +mrc.org,msgfocus.com,inbound,001,World,0 +mrmlsmatrix.com,mrmlsmatrix.com,inbound,001,World,0 +ms.com,ms.com,inbound,001,World,1 +ms00.net,ms00.net,inbound,001,World,0 +msdp1.com,msdp1.com,inbound,001,World,0 +msgfocus.com,msgfocus.com,inbound,001,World,0.011658 +msn.com,hotmail.{...},inbound,001,World,0.999964 +msn.com,hotmail.{...},outbound,001,World,1 +mta.info,ealert.com,inbound,001,World,0 +mtasv.net,mtasv.net,inbound,001,World,0.999999 +musiciansfriend.com,musiciansfriend.com,inbound,001,World,0 +musicnotes-alerts.com,mybuys.com,inbound,001,World,0 +mustanglist.com,mustanglist.com,inbound,001,World,0 +mxmfb.com,mxmfb.com,inbound,001,World,0 +mycheapoair.com,mycheapoair.com,inbound,001,World,0.957931 +mycolorscreen.com,mta4.net,inbound,001,World,0 +mydailymoment.biz,mydailymoment.biz,inbound,001,World,0 +mydailymoment.info,mydailymoment.info,inbound,001,World,0 +mydailymoment.net,mydailymoment.net,inbound,001,World,0 +mydailymoment.us,mydailymoment.us,inbound,001,World,0 +myfedloan.org,aessuccess.org,inbound,001,World,0.328917 +myfitnesspal.com,messagebus.com,inbound,001,World,0 +myfxbook.com,myfxbook.com,inbound,001,World,0 +mygreatlakes.org,glhec.org,inbound,001,World,0.000247 +mygroupon.co.th,grouponmail.{...},inbound,001,World,0 +myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,001,World,0 +myheritage.com,myheritage.com,inbound,001,World,0 +myideeli.com,myideeli.com,inbound,001,World,0 +mymeijer.com,mymeijer.com,inbound,001,World,0 +mymms.com,fagms.de,inbound,001,World,0 +mynavi.jp,mynavi.jp,inbound,001,World,3.5e-05 +myngp.com,ngpweb.com,inbound,001,World,0 +myntramail.com,iaires.com,inbound,001,World,0 +myntramail.com,myntramail.com,inbound,001,World,0 +myntramails.in,icubes.in,inbound,001,World,0 +myoutlets.in,trustmailer.com,inbound,001,World,0 +myperfectsale.com,myperfectsale.com,inbound,001,World,0.999988 +mypoints.com,mypoints.com,inbound,001,World,0 +myprotein.com,thehut.com,inbound,001,World,0 +mysafelistmailer.com,mysafelistmailer.com,inbound,001,World,0.00033 +mysale.my,mysale.my,inbound,001,World,0 +mysale.ph,mysale.ph,inbound,001,World,0 +mysmartprice.com,itzwow.com,inbound,001,World,1 +mysupermarket.co.uk,mysupermarket.co.uk,inbound,001,World,0 +mysurvey.com,mysurvey.com,inbound,001,World,0 +mysurvey.eu,mysurvey.com,inbound,001,World,0 +myvegas.com,myvegas.com,inbound,001,World,1 +myzamanamail.com,myzamanamail.com,inbound,001,World,0 +n-email.net,n-email.net,inbound,001,World,0 +n-email1.net,n-email1.net,inbound,001,World,0 +n-email4.net,n-email4.net,inbound,001,World,0 +naaptoldeals.com,eccluster.com,inbound,001,World,0 +namorico.me,namorico.me,inbound,001,World,1 +nanomail.com.br,araie.com.br,inbound,001,World,1 +napitipp.hu,napitipp.hu,inbound,001,World,0 +nasa.gov,nasa.gov,inbound,001,World,0.082443 +nascar.com,nascar.com,inbound,001,World,0 +nastygal.com,bronto.com,inbound,001,World,0 +nasza-klasa.pl,nasza-klasa.pl,inbound,001,World,0 +nationalexpress.com,nationalexpress.com,inbound,001,World,0 +nationbuilder.com,nationbuilder.com,inbound,001,World,1 +nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,001,World,0 +nature.com,nature.com,inbound,001,World,0 +naukri.com,naukri.com,inbound,001,World,0.002488 +nauta.cu,etecsa.net,inbound,001,World,0 +naver.com,naver.com,inbound,001,World,1 +naver.com,naver.com,outbound,001,World,1 +navy.mil,navy.mil,inbound,001,World,0.152107 +nba.com,nba.com,inbound,001,World,0.005828 +nbaa.org,nbaa.org,inbound,001,World,0 +ncl.com,ncl.com,inbound,001,World,0 +neimanmarcusemail.com,neimanmarcusemail.com,inbound,001,World,0 +nend.net,postini.com,inbound,001,World,0 +neolane.net,neolane.net,inbound,001,World,0.01682 +nesinemail.com,euromsg.net,inbound,001,World,0 +net-a-porter.com,net-a-porter.com,inbound,001,World,0 +net-empregos.com,net-empregos.com,inbound,001,World,0 +net-survey.jp,net-survey.jp,inbound,001,World,0 +netatlantic.com,netatlantic.com,inbound,001,World,0.001085 +netbk.co.jp,netbk.co.jp,inbound,001,World,0.010994 +netcommunity1.com,blackbaud.com,inbound,001,World,0 +netflix.com,amazonses.com,inbound,001,World,0.999999 +netflix.com,netflix.com,inbound,001,World,1 +netlogmail.com,netlogmail.com,inbound,001,World,0 +netopia.pt,netopia.pt,inbound,001,World,0.017294 +netprosoftmail.com,netprosoftmail.com,inbound,001,World,0 +netshoes.com.br,netshoes.com.br,inbound,001,World,0.078768 +netsuite.com,netsuite.com,inbound,001,World,0.57723 +networkworld.com,networkworld.com,inbound,001,World,0 +newegg.com,newegg.com,inbound,001,World,1e-06 +newgrounds.com,newgrounds.com,inbound,001,World,0 +newmarkethealth.com,newmarkethealth.com,inbound,001,World,0 +newrelic.com,sendlabs.com,inbound,001,World,0 +news-h5g.com,news-h5g.com,inbound,001,World,0 +newsletter-verychic.com,splio.es,inbound,001,World,0.9804 +newsmax.com,newsmax.com,inbound,001,World,0.004105 +newspaperdirect.com,newspaperdirect.com,inbound,001,World,0.000177 +newyorktimesinfo.com,newyorktimesinfo.com,inbound,001,World,0 +nexcess.net,nexcess.net,inbound,001,World,0.214803 +next-engine.org,next-engine.org,inbound,001,World,1 +nextdoor.com,mailgun.info,inbound,001,World,1 +nextdoor.com,mailgun.net,inbound,001,World,1 +nextdoor.com,nextdoor.com,inbound,001,World,1 +nfl.com,bfi0.com,inbound,001,World,0 +nflshop.com,nflshop.com,inbound,001,World,0 +nhs.jobs,nhscareersjobs.co.uk,inbound,001,World,0 +nic.in,relayout.nic.in,inbound,001,World,0 +nicovideo.jp,nicovideo.jp,inbound,001,World,0 +nieuwsblad.be,vummail.be,inbound,001,World,0 +nifty.com,nifty.com,inbound,001,World,0.762068 +nih.gov,nih.gov,inbound,001,World,8.8e-05 +nike.com,nike.com,inbound,001,World,0 +nikkei.com,nikkei.co.jp,inbound,001,World,0 +nikkeibp.co.jp,nikkeibp.co.jp,inbound,001,World,4.9e-05 +ninewestmail.com,ninewestmail.com,inbound,001,World,0 +ning.com,ning.com,inbound,001,World,0 +nissen.jp,nissen.jp,inbound,001,World,0 +nixle.com,nixle.com,inbound,001,World,0 +nl00.net,netline.com,inbound,001,World,5e-06 +nl00.net,nl00.net,inbound,001,World,0 +nmp1.com,nmp1.net,inbound,001,World,0 +nokia.com,nokia.com,inbound,001,World,0.001256 +nongnu.org,gnu.org,inbound,001,World,1 +nordstrom.com,taleo.net,inbound,001,World,1 +nortonfromsymantec.com,rsys1.com,inbound,001,World,0 +nos.pt,netcabo.pt,inbound,001,World,6.3e-05 +noticiasaominuto.com,ccmdcampaigns.net,inbound,001,World,0 +noticiasaominuto.com,noticiasaominuto.com,inbound,001,World,0 +novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,001,World,0 +npr.org,npr.org,inbound,001,World,0 +nrholding.net,nrholding.net,inbound,001,World,0 +ns.nl,tripolis.com,inbound,001,World,0 +nsandi.com,mxmfb.com,inbound,001,World,0 +numbersusa.com,numbersusa.com,inbound,001,World,0 +nyandcompany.com,nyandcompany.com,inbound,001,World,0 +nytimes.com,nytimes.com,inbound,001,World,0.00472 +nzsale.co.nz,nzsale.co.nz,inbound,001,World,0 +oakley.com,oakley.com,inbound,001,World,0.000904 +ocadomail.com,ocadomail.com,inbound,001,World,0 +ocmail1.in,tcmail.in,inbound,001,World,0 +ocmail14.in,tcmailer5.in,inbound,001,World,0 +ocmail22.in,tcmailer15.in,inbound,001,World,0 +ocmail22.in,tcmailer4.in,inbound,001,World,0 +ocmail40.in,tcmailer15.in,inbound,001,World,0 +ocmail40.in,tcmailer4.in,inbound,001,World,0 +ocn.ad.jp,ocn.ad.jp,inbound,001,World,0 +ocn.ne.jp,ocn.ad.jp,inbound,001,World,0 +ocnmail.in,ocmail6.in,inbound,001,World,0 +ocnmail.in,tcmail3.in,inbound,001,World,0 +odisseias.com,emv4.net,inbound,001,World,0 +odnoklassniki.ru,odnoklassniki.ru,inbound,001,World,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,001,World,0 +ofertasefacil.com.br,ofertasefacil.com.br,inbound,001,World,0 +ofertix.com,ofertix.com,inbound,001,World,0 +ofertop.pe,icommarketing.com,inbound,001,World,0 +offers.com,offers.com,inbound,001,World,1 +offerum.com,cccampaigns.com,inbound,001,World,0 +offerum.com,ccemails.com,inbound,001,World,0 +officedepot.com,officedepot.com,inbound,001,World,0.012483 +officemax.com,officemax.com,inbound,001,World,0 +officemax.com,officemaxworkplace.com,inbound,001,World,0 +ofsys.com,bulletin-metro.ca,inbound,001,World,0 +oknotify2.com,oknotify2.com,inbound,001,World,0 +oldnavy.ca,oldnavy.ca,inbound,001,World,0 +oldnavy.com,oldnavy.com,inbound,001,World,0 +olx.pt,fixeads.com,inbound,001,World,0 +olympiaedge.net,olympiaedge.net,inbound,001,World,0 +omahasteaks.com,omahasteaks.com,inbound,001,World,0.006139 +oneindia.in,infimail.com,inbound,001,World,0 +oneindia.in,mailurja.com,inbound,001,World,0 +onekingslane.com,onekingslane.com,inbound,001,World,0 +onepatriotplace.com,britecast.com,inbound,001,World,0 +onestopplus.com,neolane.net,inbound,001,World,0 +onetravelspecials.com,onetravelspecials.com,inbound,001,World,0.365386 +online.com,cnet.com,inbound,001,World,0 +onlive.com,ipost.com,inbound,001,World,0 +onmicrosoft.com,outlook.com,inbound,001,World,1 +onthecitymail.org,onthecitymail.org,inbound,001,World,1 +oo155.com,bsftransmit7.com,inbound,001,World,0 +oo155.com,oo155.com,inbound,001,World,0 +openstack.org,openstack.org,inbound,001,World,0.996884 +openstackmail.com,infimail.com,inbound,001,World,0 +opentable.com,opentable.com,inbound,001,World,0.000662 +opinionoutpost.com,opinionoutpost.com,inbound,001,World,0 +opinionsquare.com,opinionsquare.com,inbound,001,World,0 +oprah.com,oprah.com,inbound,001,World,0 +opticsplanet.com,opticsplanet.com,inbound,001,World,0 +optimusmail.in,iaires.com,inbound,001,World,0 +optonline.net,cv.net,inbound,001,World,0 +optonline.net,optonline.net,outbound,001,World,0 +orange.fr,orange.fr,inbound,001,World,0 +orange.fr,orange.fr,outbound,001,World,0 +orderscatalog.com,orderscatalog.com,inbound,001,World,0 +oriental-trading.com,oriental-trading.com,inbound,001,World,0 +oroscopofree.com,adsender.us,inbound,001,World,0 +oroscopofree.com,oroscopofree.com,inbound,001,World,0 +orsay.com,emp-mail.de,inbound,001,World,0 +os-email.com,os-email.com,inbound,001,World,0 +oshkoshbgosh.com,oshkoshbgosh.com,inbound,001,World,6e-06 +osu.edu,outlook.com,inbound,001,World,1 +otto.de,eccluster.com,inbound,001,World,1 +ouffer.com,ouffer.com,inbound,001,World,0.01022 +ourtime.com,seniorpeoplemeet.com,inbound,001,World,0 +outback.com,outback.com,inbound,001,World,0 +outlook.com,hotmail.{...},inbound,001,World,0.999929 +outlook.com,hotmail.{...},outbound,001,World,1 +outspot.be,teneo.be,inbound,001,World,0 +outspot.nl,teneo.be,inbound,001,World,0 +ovenmail.com,iaires.com,inbound,001,World,0 +overnightprints.com,chtah.net,inbound,001,World,0 +overstock.com,overstock.com,inbound,001,World,0 +ovh.net,ovh.net,inbound,001,World,0.207527 +ovuline.com,ovuline.com,inbound,001,World,1 +oxfam.org.uk,msgfocus.com,inbound,001,World,0 +ozsale.com.au,ozsale.com.au,inbound,001,World,0.000192 +p-world.co.jp,p-world.co.jp,inbound,001,World,0 +pagoda.com,zales.com,inbound,001,World,0 +pagseguro.com.br,uol.com.br,inbound,001,World,0 +pair.com,pair.com,inbound,001,World,0.880779 +palmscasinoresort.com,palmscasinoresort.com,inbound,001,World,0 +pampers.com,bfi0.com,inbound,001,World,0 +panasonic.jp,panasonic.jp,inbound,001,World,0 +pandaresearch.com,pandaresearch.com,inbound,001,World,0 +pandora.com,pandora.com,inbound,001,World,1 +pandora.net,pandora.net,inbound,001,World,0.999666 +panelplace.com,smtp.com,inbound,001,World,0 +panerabreadnews.com,panerabreadnews.com,inbound,001,World,0 +pantaloondirect.net,iaires.com,inbound,001,World,0 +papajohns-specials.com,papajohns-specials.com,inbound,001,World,0 +paradisepublishers.com,paradisepublishers.com,inbound,001,World,0.999626 +parents.com,meredith.com,inbound,001,World,0 +parkmobileglobal.com,parkmobile.us,inbound,001,World,0 +path.com,path.com,inbound,001,World,1 +patriotupdate.com,inboxfirst.com,inbound,001,World,0 +payback.de,artegic.net,inbound,001,World,0.999986 +payback.in,chtah.net,inbound,001,World,0 +payback.in,eccluster.com,inbound,001,World,0 +payback.in,ecm-cluster.com,inbound,001,World,0 +payback.in,ecmcluster.com,inbound,001,World,0 +paypal.co.uk,paypal.com,inbound,001,World,1 +paypal.com,paypal.com,inbound,001,World,0.608856 +paypal.com.au,paypal.com,inbound,001,World,1 +paypal.de,paypal.com,inbound,001,World,1 +paytm.com,paytm.com,inbound,001,World,1 +pbteen.com,pbteen.com,inbound,001,World,0 +pccomponentes.com,pccomponentes.com,inbound,001,World,0 +pch.com,ed10.com,inbound,001,World,0 +pchfrontpage.com,ed10.com,inbound,001,World,0 +pchlotto.com,ed10.com,inbound,001,World,0 +pchplayandwin.com,ed10.com,inbound,001,World,0 +pchsearch.com,ed10.com,inbound,001,World,0 +pcmag.com,ittoolbox.com,inbound,001,World,0 +pcworld.com,pcworld.com,inbound,001,World,0 +pd25.com,pd25.com,inbound,001,World,1 +pd25.com,pd27.com,inbound,001,World,1 +peanuthome.info,adopterc.info,inbound,001,World,0 +peanuthome.info,aguitytr.info,inbound,001,World,0 +peanuthome.info,bevest.info,inbound,001,World,0 +peanuthome.info,bluester.info,inbound,001,World,0 +peanuthome.info,burror.info,inbound,001,World,0 +peanuthome.info,bursion.info,inbound,001,World,0 +peanuthome.info,cantexi.info,inbound,001,World,0 +peanuthome.info,caserhi.info,inbound,001,World,0 +peanuthome.info,celect.info,inbound,001,World,0 +peanuthome.info,chintone.info,inbound,001,World,0 +peanuthome.info,citery.info,inbound,001,World,0 +peanuthome.info,cleathal.info,inbound,001,World,0 +peanuthome.info,coherentrequittal.info,inbound,001,World,0 +peanuthome.info,colicom.info,inbound,001,World,0 +peanuthome.info,complec.info,inbound,001,World,0 +peanuthome.info,cyprinoidkaiserdom.info,inbound,001,World,0 +peanuthome.info,deciarc.info,inbound,001,World,0 +peanuthome.info,declaws.info,inbound,001,World,0 +peanuthome.info,dewest.info,inbound,001,World,0 +peanuthome.info,epconce.info,inbound,001,World,0 +peanuthome.info,fnotec.info,inbound,001,World,0 +peanuthome.info,folkswor.info,inbound,001,World,0 +peanuthome.info,forepert.info,inbound,001,World,0 +peanuthome.info,gurgaro.info,inbound,001,World,0 +peanuthome.info,heallyps.info,inbound,001,World,0 +peanuthome.info,holeph.info,inbound,001,World,0 +peanuthome.info,homewor.info,inbound,001,World,0 +peanuthome.info,hydroni.info,inbound,001,World,0 +peanuthome.info,ingenbu.info,inbound,001,World,0 +peanuthome.info,kinklybotaurus.info,inbound,001,World,0 +peanuthome.info,ninetiethwhiffet.info,inbound,001,World,0 +peanuthome.info,unglibshudder.info,inbound,001,World,0 +peanutwebmaster.info,adcrent.info,inbound,001,World,0 +peanutwebmaster.info,addmiel.info,inbound,001,World,0 +peanutwebmaster.info,agilhe.info,inbound,001,World,0 +peanutwebmaster.info,allegap.info,inbound,001,World,0 +peanutwebmaster.info,andvore.info,inbound,001,World,0 +peanutwebmaster.info,angogl.info,inbound,001,World,0 +peanutwebmaster.info,animass.info,inbound,001,World,0 +peanutwebmaster.info,arettery.info,inbound,001,World,0 +peanutwebmaster.info,aribank.info,inbound,001,World,0 +peanutwebmaster.info,arkefoc.info,inbound,001,World,0 +peanutwebmaster.info,avenog.info,inbound,001,World,0 +peanutwebmaster.info,barrave.info,inbound,001,World,0 +peanutwebmaster.info,bindowmo.info,inbound,001,World,0 +peanutwebmaster.info,bitravit.info,inbound,001,World,0 +peanutwebmaster.info,borsand.info,inbound,001,World,0 +peanutwebmaster.info,branti.info,inbound,001,World,0 +peanutwebmaster.info,breaserp.info,inbound,001,World,0 +peanutwebmaster.info,bredogly.info,inbound,001,World,0 +peanutwebmaster.info,briantra.info,inbound,001,World,0 +peanutwebmaster.info,bridea.info,inbound,001,World,0 +peanutwebmaster.info,carial.info,inbound,001,World,0 +peanutwebmaster.info,castac.info,inbound,001,World,0 +peanutwebmaster.info,chedoner.info,inbound,001,World,0 +peanutwebmaster.info,chiquent.info,inbound,001,World,0 +peanutwebmaster.info,cinchoi.info,inbound,001,World,0 +peanutwebmaster.info,cliate.info,inbound,001,World,0 +peanutwebmaster.info,cognn.info,inbound,001,World,0 +peanutwebsite.info,abjibbin.info,inbound,001,World,0 +peanutwebsite.info,audiette.info,inbound,001,World,0 +peanutwebsite.info,blancer.info,inbound,001,World,0 +peanutwebsite.info,bluester.info,inbound,001,World,0 +peanutwebsite.info,burror.info,inbound,001,World,0 +peanutwebsite.info,bursion.info,inbound,001,World,0 +peanutwebsite.info,caserhi.info,inbound,001,World,0 +peanutwebsite.info,celect.info,inbound,001,World,0 +peanutwebsite.info,cleathal.info,inbound,001,World,0 +peanutwebsite.info,coherentrequittal.info,inbound,001,World,0 +peanutwebsite.info,complec.info,inbound,001,World,0 +peanutwebsite.info,condost.info,inbound,001,World,0 +peanutwebsite.info,cyprinoidkaiserdom.info,inbound,001,World,0 +peanutwebsite.info,deciarc.info,inbound,001,World,0 +peanutwebsite.info,declaws.info,inbound,001,World,0 +peanutwebsite.info,epconce.info,inbound,001,World,0 +peanutwebsite.info,ferrayer.info,inbound,001,World,0 +peanutwebsite.info,fnotec.info,inbound,001,World,0 +peanutwebsite.info,folkswor.info,inbound,001,World,0 +peanutwebsite.info,forepert.info,inbound,001,World,0 +peanutwebsite.info,gurgaro.info,inbound,001,World,0 +peanutwebsite.info,holeph.info,inbound,001,World,0 +peanutwebsite.info,homewor.info,inbound,001,World,0 +peanutwebsite.info,hydroni.info,inbound,001,World,0 +peanutwebsite.info,ingenbu.info,inbound,001,World,0 +peanutwebsite.info,kinklybotaurus.info,inbound,001,World,0 +peanutwebsite.info,ninetiethwhiffet.info,inbound,001,World,0 +pearlsofwealth.com,pearlsofwealth.com,inbound,001,World,1 +peartreegreetings.com,rexcraft.com,inbound,001,World,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,001,World,0 +pennwell.com,pennwell.com,inbound,001,World,0 +pepabo.com,pepabo.com,inbound,001,World,0 +pepboys.com,pepboys.com,inbound,001,World,0 +peperoni.de,peperoni.de,inbound,001,World,1 +pepperfry.com,epidm.net,inbound,001,World,0 +perfectpriceindia.com,infimail.com,inbound,001,World,0 +perfectworld.com,perfectworld.com,inbound,001,World,0.000162 +perfora.net,perfora.net,inbound,001,World,0.920896 +permissionresearch.com,permissionresearch.com,inbound,001,World,0 +personalliberty.com,personalliberty.com,inbound,001,World,1 +personare.com.br,personare.com.br,inbound,001,World,0 +peytz.dk,peytz.dk,inbound,001,World,4e-06 +pga.com,pga.com,inbound,001,World,0 +pge.com,pge.com,inbound,001,World,0.149792 +pgeveryday.com,bfi0.com,inbound,001,World,0 +philosophy.com,philosophy.com,inbound,001,World,0 +phoenix.edu,phoenix.edu,inbound,001,World,0 +photobox.com,photobox.com,inbound,001,World,0 +photoprintit.com,photoprintit.com,inbound,001,World,0 +phpclasses.org,phpclasses.org,inbound,001,World,0 +phsmtpbox.com,phsmtpbox.com,inbound,001,World,0 +pia.jp,pia.jp,inbound,001,World,0 +pinger.com,pinger.com,inbound,001,World,0 +pinterest.com,pinterest.com,inbound,001,World,1 +pivotaltracker.com,pivotaltracker.com,inbound,001,World,1 +pixable.com,pixable.com,inbound,001,World,1 +pixum.com,pixum.com,inbound,001,World,0 +pizzahut.com,quikorder.com,inbound,001,World,0 +pizzahutoffers.com,pizzahutoffers.com,inbound,001,World,0 +placedestendances.com,placedestendances.com,inbound,001,World,0 +plaisio.gr,fagms.de,inbound,001,World,0 +planeo.com,planeo.com,inbound,001,World,0 +planeo.pt,planeo.pt,inbound,001,World,0 +playstation.com,playstation.com,inbound,001,World,0 +playstationmail.net,playstationmail.net,inbound,001,World,0 +playtika.com,emv8.com,inbound,001,World,0 +plexapp.com,plex.tv,inbound,001,World,1 +plumdistrict.com,plumdistrict.com,inbound,001,World,1 +pmailus.com,patrontechnology.com,inbound,001,World,0 +pnc.com,messagelabs.com,inbound,001,World,0.997194 +pnetweb.co.za,hosting.co.za,inbound,001,World,0 +pnetweb.co.za,salesnet.co.za,inbound,001,World,0 +pobox.com,pobox.com,inbound,001,World,0.975372 +pof.com,plentyoffish.co.uk,inbound,001,World,0 +pogo.com,pogo.com,inbound,001,World,0 +pointtown.com,gmo-media.jp,inbound,001,World,0 +poinx.com,poinx.com,inbound,001,World,0 +pokerstars.com,pokerstars.eu,inbound,001,World,0 +pokerstars.eu,pokerstars.eu,inbound,001,World,0 +pokupon.by,mailersend.com,inbound,001,World,0 +pokupon.ua,mailersend.com,inbound,001,World,0 +politicoemail.com,politicoemail.com,inbound,001,World,0 +polyvore.com,polyvore.com,inbound,001,World,1 +pontofrio.com.br,emv8.com,inbound,001,World,0 +popsugar.com,popsugar.com,inbound,001,World,0 +postcardfromhell.com,cyberthugs.com,inbound,001,World,1 +postgresql.org,postgresql.org,inbound,001,World,0.996805 +potterybarn.com,potterybarn.com,inbound,001,World,0 +potterybarnkids.com,potterybarnkids.com,inbound,001,World,0 +praca.pl,praca.pl,inbound,001,World,1 +pracuj.pl,pracuj.pl,inbound,001,World,0 +preferredpetclub.com,preferredpetclub.com,inbound,001,World,0 +presslaff.net,dat-e-baseonline.com,inbound,001,World,0 +pressmartmail.com,pressmartmail.com,inbound,001,World,0 +priceline.com,priceline.com,inbound,001,World,1 +princess.com,princess.com,inbound,001,World,0 +printvenue.com,fagms.de,inbound,001,World,0 +priorityoneemail.com,priorityoneemail.com,inbound,001,World,0 +privalia.com,privalia.com,inbound,001,World,3.94913202027326e-07 +private-elist.com,private-elist.com,inbound,001,World,0 +privoscite.si,privoscite.si,inbound,001,World,0 +profitcenteronline.com,groupdealtools.com,inbound,001,World,1 +progressive.com,progressive.com,inbound,001,World,0.897291 +progressiveagent.com,progressive.com,inbound,001,World,1 +promedmail.org,childrenshospital.org,inbound,001,World,1 +promod-news.fr,promod-news.com,inbound,001,World,0 +propertysolutions.com,propertysolutions.com,inbound,001,World,1 +prospectgeysercoop.com,prospectgeysercoop.com,inbound,001,World,1 +protopmail.com,protopmail.com,inbound,001,World,0 +providesupport.com,providesupport.com,inbound,001,World,0.000103 +proxyvote.com,adp-ics.com,inbound,001,World,0.908635 +psu.edu,psu.edu,inbound,001,World,0.073843 +pttplc.com,pttgrp.com,inbound,001,World,0 +publicators.com,publicators.com,inbound,001,World,0 +publix.com,publix.com,inbound,001,World,0.002567 +pucp.edu.pe,pucp.edu.pe,inbound,001,World,0.225541 +puffinmailer.com,zoothost.com,inbound,001,World,0 +pur3.net,pur3.net,inbound,001,World,0.001111 +purewow.com,purewow.com,inbound,001,World,0 +puritan.com,email-nbtyinc.com,inbound,001,World,0 +purlsmail.com,purlsmail.com,inbound,001,World,0 +pxsmail.com,pxsmail.com,inbound,001,World,0 +python.org,python.org,inbound,001,World,1 +q.com,synacor.com,inbound,001,World,0.999729 +qemailserver.com,qemailserver.com,inbound,001,World,0 +qoinpro.com,qoinpro.com,inbound,001,World,1 +qoo10.jp,qoo10.jp,inbound,001,World,2e-06 +qoo10.sg,qoo10.co.id,inbound,001,World,1.9e-05 +qoo10.sg,qoo10.com,inbound,001,World,0 +qoo10.sg,qoo10.my,inbound,001,World,2.2e-05 +qoo10.sg,qoo10.sg,inbound,001,World,8e-06 +qq.com,qq.com,inbound,001,World,0.999978 +qq.com,qq.com,outbound,001,World,1 +qtropnews.com,qtropnews.com,inbound,001,World,1.7e-05 +qualicorp.com.br,qualicorp.com.br,inbound,001,World,1 +quality.net.ua,quality.net.ua,inbound,001,World,0 +qualitysafelist.com,zoothost.com,inbound,001,World,0 +queopinas.com,confirmit.com,inbound,001,World,1 +quickbooks.com,intuit.com,inbound,001,World,0.99908 +quickrewards.net,quickrewards.net,inbound,001,World,0 +quikr.com,quikr.com,inbound,001,World,0.026599 +quinstreet.com,neoquin.com,inbound,001,World,0 +quora.com,quora.com,inbound,001,World,1 +quoramail.com,quoramail.com,inbound,001,World,1 +qvcemail.com,qvcemail.com,inbound,001,World,0 +r-project.org,ethz.ch,inbound,001,World,0.99999 +r51.it,musvc.com,inbound,001,World,0.092498 +r52.it,musvc.com,inbound,001,World,0.000221 +r57.it,musvc.com,inbound,001,World,0.139425 +r67.it,musvc.com,inbound,001,World,0.199845 +r70.it,musvc.com,inbound,001,World,0.311983 +rabota.ua,rabota.ua,inbound,001,World,0 +rackroom-email.com,rackroom-email.com,inbound,001,World,0 +radarsystems.net,radarsystems.net,inbound,001,World,1 +radiantretailapps.com,radiantretailapps.com,inbound,001,World,0 +radioshack.com,radioshack.com,inbound,001,World,0 +railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,001,World,0 +rakuten.co.jp,rakuten.co.jp,inbound,001,World,0 +rakuten.co.jp,shareee.jp,inbound,001,World,0 +rakuten.co.jp,yahoo.co.jp,inbound,001,World,0 +rakuten.com,rakuten.com,inbound,001,World,0 +rakuten.ne.jp,rakuten.co.jp,inbound,001,World,0 +rambler.ru,rambler.ru,inbound,001,World,0.08173 +randomhouse.com,randomhouse.com,inbound,001,World,0 +rapattoni.com,rapmls.com,inbound,001,World,0 +ratedpeople.com,ratedpeople.com,inbound,001,World,0.000979 +rax.ru,rax.ru,inbound,001,World,0.000258 +razerzone.com,chtah.net,inbound,001,World,0 +rbc.com,rbc.com,inbound,001,World,0.001897 +rdio.com,rdio.com,inbound,001,World,1 +reactiveadz.com,downlinebuilderdirect.com,inbound,001,World,0 +realage-mail.com,postdirect.com,inbound,001,World,0 +realestate.com.au,realestate.com.au,inbound,001,World,0 +realtor.org,realtor.org,inbound,001,World,0 +realtytrac.com,realtytrac.com,inbound,001,World,0.001681 +realus.co.jp,realus.co.jp,inbound,001,World,0 +recipe.com,meredith.com,inbound,001,World,0 +recochoku.jp,recochoku.jp,inbound,001,World,0 +recruit.net,recruit.net,inbound,001,World,0 +redbox.com,exacttarget.com,inbound,001,World,0 +redbox.com,redbox.com,inbound,001,World,4e-06 +redcross.org.uk,redcross.org.uk,inbound,001,World,0 +redfin.com,redfin.com,inbound,001,World,0 +rediffmail.com,akadns.net,outbound,001,World,0 +rediffmail.com,rediffmail.com,inbound,001,World,0 +redtri.com,redtri.com,inbound,001,World,0 +reebokusnews.com,reebokusnews.com,inbound,001,World,0 +reebonz.com,ed10.com,inbound,001,World,0 +reebonz.com,reebonz.com,inbound,001,World,0.857615 +reed.co.uk,reed.co.uk,inbound,001,World,0 +regie11.net,odiso.net,inbound,001,World,0 +regionalhelpwanted.com,regionalhelpwanted.com,inbound,001,World,1 +registrar-servers.com,registrar-servers.com,inbound,001,World,0.053619 +registro.br,registro.br,inbound,001,World,0 +relax7.hu,gruppi.hu,inbound,001,World,0 +relianceada.com,relianceada.com,inbound,001,World,0.276515 +rent.com,rent.com,inbound,001,World,0.000397 +rentalcars.com,rentalcars.com,inbound,001,World,1 +renweb.com,renweb.com,inbound,001,World,0 +repica.jp,kakaku.com,inbound,001,World,0 +repica.jp,repica.jp,inbound,001,World,0 +republicwireless.com,republicwireless.com,inbound,001,World,0.033564 +research-panel.jp,research-panel.jp,inbound,001,World,0 +researchgate.net,researchgate.net,inbound,001,World,0 +responder.co.il,responder.co.il,inbound,001,World,1 +restorationhardware.com,restorationhardware.com,inbound,001,World,0 +retailjobinsider.com,retailjobinsider.com,inbound,001,World,0 +retailmenot.com,retailmenot.com,inbound,001,World,3.54930373308668e-07 +reverbnation.com,reverbnation.com,inbound,001,World,0 +revolutiongolf.com,revolutiongolf.com,inbound,001,World,0 +rewardme.in,bfi0.com,inbound,001,World,0 +reyrey.net,reyrey.net,inbound,001,World,0.946402 +ricardoeletro.com.br,allin.com.br,inbound,001,World,0 +richersoundsvip.com,ibwmail.com,inbound,001,World,0 +richyrichmailer.com,maddog-productions.info,inbound,001,World,1 +rightmove.com,rightmove.com,inbound,001,World,0 +rigzonemail.com,rigzonemail.com,inbound,001,World,0 +rikunabi.com,rikunabi.com,inbound,001,World,1e-05 +ringcentral.com,ringcentral.com,inbound,001,World,1 +ripleyperu.com.pe,icommarketing.com,inbound,001,World,0 +riseup.net,riseup.net,inbound,001,World,1 +rivamail.com,mailurja.com,inbound,001,World,0 +rmtr.de,rapidmail.de,inbound,001,World,0 +rnmk.com,rnmk.com,inbound,001,World,0 +roadrunner.com,rr.com,inbound,001,World,0.005479 +roamans.com,neolane.net,inbound,001,World,0 +rocketmail.com,yahoo.{...},inbound,001,World,1 +rocketmail.com,yahoodns.net,outbound,001,World,1 +rockpath.info,rockpath.info,inbound,001,World,1 +rockwellcollins.com,rockwellcollins.com,inbound,001,World,1 +rogers.com,yahoo.{...},inbound,001,World,1 +rogers.com,yahoodns.net,outbound,001,World,1 +rookiestewsemails.com,rookiestewsemails.com,inbound,001,World,0 +roulartamail.be,roulartamail.be,inbound,001,World,0 +royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,001,World,0 +rpinow.org,app-info.net,inbound,001,World,0 +rpo9usa.email,rpo9usa.email,inbound,001,World,0 +rr.com,rr.com,inbound,001,World,0.008342 +rr.com,rr.com,outbound,001,World,0 +rsgsv.net,postini.com,inbound,001,World,0.093513 +rsgsv.net,rsgsv.net,inbound,001,World,0 +rsvpsv.net,rsvpsv.net,inbound,001,World,0 +rsvpsv.net,send.esp.br,inbound,001,World,0 +rsys2.com,amfam.com,inbound,001,World,0 +rsys2.com,cheaptickets.com,inbound,001,World,0 +rsys2.com,dishnetworkmail.com,inbound,001,World,0 +rsys2.com,e-comms.net,inbound,001,World,0 +rsys2.com,eharmony.com,inbound,001,World,0 +rsys2.com,fathead.com,inbound,001,World,0 +rsys2.com,intuit.com,inbound,001,World,0 +rsys2.com,kmart.com,inbound,001,World,0 +rsys2.com,kohlernews.com,inbound,001,World,0 +rsys2.com,lego.com,inbound,001,World,0 +rsys2.com,lenovo.com,inbound,001,World,0 +rsys2.com,modcloth.com,inbound,001,World,0 +rsys2.com,moxieinteractive.com,inbound,001,World,0 +rsys2.com,orbitz.com,inbound,001,World,0 +rsys2.com,payless.com,inbound,001,World,0 +rsys2.com,petsathome.com,inbound,001,World,0 +rsys2.com,quizzle.com,inbound,001,World,0 +rsys2.com,robeez.com,inbound,001,World,0 +rsys2.com,rsys1.com,inbound,001,World,0 +rsys2.com,rsys2.com,inbound,001,World,0 +rsys2.com,rsys3.com,inbound,001,World,0 +rsys2.com,rsys4.com,inbound,001,World,0 +rsys2.com,saucony.com,inbound,001,World,0 +rsys2.com,sears.com,inbound,001,World,0 +rsys2.com,shopbop.com,inbound,001,World,0 +rsys2.com,southwest.com,inbound,001,World,0 +rsys2.com,speeddatemail.com,inbound,001,World,0 +rsys2.com,thecompanystore.com,inbound,001,World,0 +rsys2.com,theknot.com,inbound,001,World,0 +rsys5.com,alibris.com,inbound,001,World,0 +rsys5.com,allstate-email.com,inbound,001,World,0 +rsys5.com,beachmint.com,inbound,001,World,0 +rsys5.com,belleandclive.com,inbound,001,World,0 +rsys5.com,br.dk,inbound,001,World,0 +rsys5.com,charlotterusse.com,inbound,001,World,0 +rsys5.com,comixology.com,inbound,001,World,0 +rsys5.com,cottonon.com,inbound,001,World,0 +rsys5.com,ediblearrangements.com,inbound,001,World,0 +rsys5.com,emailworldmarket.com,inbound,001,World,0 +rsys5.com,farfetch.com,inbound,001,World,0 +rsys5.com,frhiemailcommunications.com,inbound,001,World,0 +rsys5.com,harryanddavid.com,inbound,001,World,0 +rsys5.com,hollandandbarrett.com,inbound,001,World,0 +rsys5.com,icing.com,inbound,001,World,0 +rsys5.com,indigo.ca,inbound,001,World,0 +rsys5.com,jabong.com,inbound,001,World,0 +rsys5.com,jcrew.com,inbound,001,World,0 +rsys5.com,jjill.com,inbound,001,World,0 +rsys5.com,kanui.com.br,inbound,001,World,0 +rsys5.com,kirklands.com,inbound,001,World,0 +rsys5.com,lanebryant.com,inbound,001,World,0 +rsys5.com,lazada.com,inbound,001,World,0 +rsys5.com,leapfrog.com,inbound,001,World,0 +rsys5.com,llbean.com,inbound,001,World,0 +rsys5.com,lojascolombo.com.br,inbound,001,World,0 +rsys5.com,madewell.com,inbound,001,World,0 +rsys5.com,magazineluiza.com.br,inbound,001,World,0 +rsys5.com,missguided.co.uk,inbound,001,World,0 +rsys5.com,moma.org,inbound,001,World,0 +rsys5.com,nationalgeographic.com,inbound,001,World,0 +rsys5.com,neat.com,inbound,001,World,0 +rsys5.com,newbalance.com,inbound,001,World,0 +rsys5.com,news-voeazul.com.br,inbound,001,World,0 +rsys5.com,nordstrom.com,inbound,001,World,0 +rsys5.com,normthompson.com,inbound,001,World,0 +rsys5.com,novomundo.com.br,inbound,001,World,0 +rsys5.com,ourdeal.com.au,inbound,001,World,0 +rsys5.com,pier1.com,inbound,001,World,0 +rsys5.com,postini.com,inbound,001,World,0.0697 +rsys5.com,productmadness.com,inbound,001,World,0 +rsys5.com,rainbowshops.com,inbound,001,World,0 +rsys5.com,rei.com,inbound,001,World,0 +rsys5.com,roadrunnersports.com,inbound,001,World,0 +rsys5.com,rosettastone.com,inbound,001,World,0 +rsys5.com,seamless.com,inbound,001,World,0 +rsys5.com,serenaandlily.com,inbound,001,World,0 +rsys5.com,smiles.com.br,inbound,001,World,0 +rsys5.com,soubarato.com.br,inbound,001,World,0 +rsys5.com,strava.com,inbound,001,World,0 +rsys5.com,submarino.com.br,inbound,001,World,0 +rsys5.com,thewalkingcompany.com,inbound,001,World,0 +rsys5.com,tigerdirect.com,inbound,001,World,0 +rsys5.com,udemy.com,inbound,001,World,0 +rsys5.com,vitaminshoppe.com,inbound,001,World,0 +rsys5.com,vpusa.com,inbound,001,World,0 +rsys5.com,walmart.com.br,inbound,001,World,0 +rsys5.com,worldofwatches.com,inbound,001,World,0 +rsys5.com,xfinity.com,inbound,001,World,0 +rue21email.com,rue21email.com,inbound,001,World,0 +rueducommerce.com,groupe-rueducommerce.fr,inbound,001,World,0 +rummycirclemails.com,eccluster.com,inbound,001,World,0 +runkeeper.com,runkeeper.com,inbound,001,World,0 +runnet.jp,runnet.jp,inbound,001,World,0 +runtastic.com,runtastic.com,inbound,001,World,0 +rutenmail.com.tw,rutenmail.com.tw,inbound,001,World,0 +ruum.com,ruum.com,inbound,001,World,0 +ryanairmail.com,ryanairmail.com,inbound,001,World,0 +rzone.de,rzone.de,inbound,001,World,1 +s3s-br1.net,splio.com.br,inbound,001,World,0.908187 +s3s-main.net,splio.com,inbound,001,World,0.970428 +s4s-pl1.pl,splio.com,inbound,001,World,0.939032 +saavn.com,saavn.com,inbound,001,World,1 +safe-sender.net,safe-sender.net,inbound,001,World,0 +safelistextreme.com,quantumsafelist.com,inbound,001,World,0 +safelistpro.com,safelistpro.com,inbound,001,World,0.00014 +safeway.com,chtah.com,inbound,001,World,0 +safeway.com,safeway.com,inbound,001,World,0.000382 +sahibinden.com,sahibinden.com,inbound,001,World,0 +sailthru.com,sailthru.com,inbound,001,World,0 +saimails.in,infimail.com,inbound,001,World,0 +sainsburys.co.uk,emv5.com,inbound,001,World,0 +saisoncard.co.jp,saisoncard.co.jp,inbound,001,World,0 +saks.com,saks.com,inbound,001,World,0 +saksoff5th.com,saksoff5th.com,inbound,001,World,0 +sakura.ne.jp,sakura.ne.jp,inbound,001,World,0.640465 +salememail.net,salememail.net,inbound,001,World,0 +salesforce.com,postini.com,inbound,001,World,0.866057 +salesforce.com,salesforce.com,inbound,001,World,0.983322 +salesforce.com,salesforce.com,outbound,001,World,1 +salesmanago.pl,salesmanago.pl,inbound,001,World,0 +salliemae.com,salliemae.com,inbound,001,World,1 +salsalabs.net,salsalabs.net,inbound,001,World,0 +samashmusic.com,wc09.net,inbound,001,World,0 +samsclub.com,m0.net,inbound,001,World,0 +samsung.com,samsung.com,inbound,001,World,0.043941 +samsung.ru,samsung.ru,inbound,001,World,0 +samsungusa.com,samsungusa.com,inbound,001,World,0 +sanmina-sci.com,postini.com,inbound,001,World,0.999991 +sanmina.com,postini.com,inbound,001,World,0.9996 +sans.org,sans.org,inbound,001,World,0.497629 +santander.cl,santander.cl,inbound,001,World,0.999992 +santander.cl,santandersantiago.cl,inbound,001,World,1 +sapnetworkmail.com,sap-ag.de,inbound,001,World,1 +sapo.pt,sapo.pt,inbound,001,World,0.242839 +sapo.pt,sapo.pt,outbound,001,World,0 +saramin.co.kr,saramin.co.kr,inbound,001,World,0 +sassieshop.com,sassieshop.com,inbound,001,World,0.025887 +saturday.com,saturday.com,inbound,001,World,0 +savelivefresh.com,livesavemail.com,inbound,001,World,1 +savingdeals.in,infimail.com,inbound,001,World,0 +savingstar.com,savingstar.com,inbound,001,World,1 +sbcglobal.net,yahoo.{...},inbound,001,World,0.999991 +sbcglobal.net,yahoodns.net,outbound,001,World,1 +sbi.co.in,sbi.co.in,inbound,001,World,0 +sbr-inc.co.jp,hdemail.jp,inbound,001,World,0.000528 +sc.com,messagelabs.com,inbound,001,World,0.996156 +sc.com,sc.com,inbound,001,World,0.99683 +schwab.com,schwab.com,inbound,001,World,0.025055 +scmp.com,emarsys.net,inbound,001,World,0 +scoop.it,scoop.it,inbound,001,World,0 +scoopon.com.au,inxserver.de,inbound,001,World,1 +screwfix.info,fwdto.net,inbound,001,World,0 +sears.ca,sears.ca,inbound,001,World,0.005447 +searscard.com,searscard.com,inbound,001,World,1 +seaworld.com,seaworld.com,inbound,001,World,0 +secretescapes.com,secretescapes.com,inbound,001,World,0 +secure.ne.jp,secure.ne.jp,inbound,001,World,0.000356 +securence.com,securence.com,inbound,001,World,0.667957 +secureserver.net,secureserver.net,inbound,001,World,3.6e-05 +seek.com.au,seek.com.au,inbound,001,World,0 +seekingalpha.com,seekingalpha.com,inbound,001,World,1 +seekingalpha.com,sendgrid.net,inbound,001,World,1 +selectacast.net,selectacast.net,inbound,001,World,0.000561 +selection-priceminister.com,selection-priceminister.com,inbound,001,World,0 +semana.com,semana.com,inbound,001,World,0.005867 +senate.gov,senate.gov,inbound,001,World,0.992994 +sendearnings.com,sendearnings.com,inbound,001,World,0 +sender.lt,sritis.lt,inbound,001,World,0.000352 +sendgrid.info,sendgrid.net,inbound,001,World,0.999966 +sendgrid.me,sendgrid.net,inbound,001,World,1 +sendlane.com,sendlane.com,inbound,001,World,0 +sendpal.in,sendpal.in,inbound,001,World,0 +sendsmaily.info,sendsmaily.info,inbound,001,World,0 +seniorplanet.fr,seniorplanet.fr,inbound,001,World,0 +serpadres.es,chtah.net,inbound,001,World,0 +service-now.com,postini.com,inbound,001,World,0.988878 +service-now.com,service-now.com,inbound,001,World,0.998853 +serviciobancomer.com,serviciobancomer.com,inbound,001,World,0 +seznam.cz,seznam.cz,inbound,001,World,0.001781 +seznam.cz,seznam.cz,outbound,001,World,0.001741 +sfid01.com,sfid01.com,inbound,001,World,0 +sfimg.com,sfimarketing.com,inbound,001,World,0 +sfly.com,shutterfly.com,inbound,001,World,0 +sfr.fr,sfr.fr,inbound,001,World,0.236539 +sfr.fr,sfr.fr,outbound,001,World,0.004753 +shaadi.com,shaadi.com,inbound,001,World,0.000154 +shadowshopper.com,shadowshopper.com,inbound,001,World,5.6e-05 +shaw.ca,shaw.ca,inbound,001,World,0 +shaw.ca,shaw.ca,outbound,001,World,1 +sheplers.com,sheplers.com,inbound,001,World,0 +shiftplanning.com,shiftplanning.com,inbound,001,World,0 +shiksha.com,shiksha.com,inbound,001,World,0 +shinseibank.com,shinseibank.com,inbound,001,World,0 +shoedazzle.com,shoedazzle.com,inbound,001,World,0 +shoes.com,famousfootwear.com,inbound,001,World,0 +shop2gether.com.br,shop2gether.com.br,inbound,001,World,0 +shopbonton.com,shopbonton.com,inbound,001,World,0 +shopcluesemail.com,shopcluesemail.com,inbound,001,World,0 +shopcluesmail.com,shopcluesmail.com,inbound,001,World,0 +shophq.com,shophq.com,inbound,001,World,0 +shopjustice.com,shopjustice.com,inbound,001,World,0 +shopkick.com,shopkick.com,inbound,001,World,0 +shopko.com,shopko.com,inbound,001,World,0 +shopnineteenmails.in,iaires.com,inbound,001,World,0 +shoppersoptimum.ca,thindata.net,inbound,001,World,0 +shoppersstop.com,shoppersstop.com,inbound,001,World,0 +shoprite-email.com,email-mywebgrocer.com,inbound,001,World,0 +shoptime.com,shoptime.com,inbound,001,World,0 +shopto.net,shopto.net,inbound,001,World,1 +showingtime.com,showingtime.com,inbound,001,World,0 +showroomprive.com,showroomprive.be,inbound,001,World,0 +showroomprive.com,showroomprive.com,inbound,001,World,0.007102 +showroomprive.com,showroomprive.nl,inbound,001,World,0 +showroomprive.es,showroomprive.es,inbound,001,World,0 +showroomprive.it,showroomprive.pt,inbound,001,World,0 +showroomprive.pt,showroomprive.co.uk,inbound,001,World,0 +shtyle.fm,shtyle.fm,inbound,001,World,0 +shukatsu.jp,shukatsu.jp,inbound,001,World,0 +siella.jp,siella.jp,inbound,001,World,0.000199 +sierratradingpost.com,sierratradingpost.com,inbound,001,World,0.000885 +sigmabeauty.com,lstrk.net,inbound,001,World,1 +sii.cl,sii.cl,inbound,001,World,0 +simplesafelist.com,adminforfree.com,inbound,001,World,1 +simpletextadz.com,web-hosting.com,inbound,001,World,1 +simplyhired.com,simplyhired.com,inbound,001,World,0 +simplymarry.com,tbsl.in,inbound,001,World,0 +singsale.com.sg,singsale.com.sg,inbound,001,World,0 +siriusxm.com,xmradio.com,inbound,001,World,0 +sitecore-mailer.com,sendlabs.com,inbound,001,World,0 +sittercity.com,sittercity.com,inbound,001,World,0 +sixflags.com,sixflags.com,inbound,001,World,0 +skillpages-mailer.com,dynect.net,inbound,001,World,0 +skillpages-mailer.com,sendlabs.com,inbound,001,World,0 +skillpages-mailer.com,skillpagesmail.com,inbound,001,World,0 +sky.com,sky.com,inbound,001,World,8.8e-05 +skymall.com,skymall.com,inbound,001,World,0.001135 +skynet.be,belgacom.be,inbound,001,World,0.005903 +skype.com,delivery.net,inbound,001,World,0 +skype.com,skype.com,inbound,001,World,0 +skyscanner.net,skyscanner.net,inbound,001,World,1 +sld.cu,sld.cu,inbound,001,World,1 +slickdeals.net,slickdeals.net,inbound,001,World,4.9e-05 +slidesharemail.com,newslettergrid.com,inbound,001,World,1 +slidesharemail.com,slideshare.net,inbound,001,World,1 +slidesharemail.com,slidesharemail.com,inbound,001,World,1 +smartbrief.com,smartbrief.com,inbound,001,World,0 +smartdraw.com,smartdraw.com,inbound,001,World,0 +smartertravel.com,smartertravelmedia.com,inbound,001,World,0.021388 +smartphoneexperts.com,mailgun.net,inbound,001,World,1 +smartresponder.ru,smartresponder.ru,inbound,001,World,1 +smp.ne.jp,smp.ne.jp,inbound,001,World,0 +snagajob-email.com,snagajob-email.com,inbound,001,World,0 +snapdeal.com,snapdeal.com,inbound,001,World,0 +snapdealmail.in,snapdealmail.in,inbound,001,World,0 +snaphire.com,snaphire.com,inbound,001,World,0 +snapretail.com,snapretail.com,inbound,001,World,1 +socialappsmail.com,socialappsmail.com,inbound,001,World,1 +socialsex.biz,infinitypersonals.com,inbound,001,World,0 +sofmap.com,sofmap.com,inbound,001,World,0.0092 +softbank.jp,softbank.jp,inbound,001,World,0 +softbank.jp,softbank.jp,outbound,001,World,0 +softbank.ne.jp,softbank.ne.jp,inbound,001,World,0 +softbank.ne.jp,softbank.ne.jp,outbound,001,World,0 +solesociety.com,bronto.com,inbound,001,World,0 +solosenders.com,megasenders.com,inbound,001,World,8.2e-05 +solosenders.com,traxweb.org,inbound,001,World,0 +solveerrors.com,infimail.com,inbound,001,World,0 +soma.com,soma.com,inbound,001,World,0 +someecards.com,someecards.com,inbound,001,World,0 +songkick.com,songkick.com,inbound,001,World,1 +sony-latin.com,sony-latin.com,inbound,001,World,0.01735 +sony.com,sony.com,inbound,001,World,0.020225 +sony.jp,sony.jp,inbound,001,World,0.000654 +sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,001,World,0 +sonyrewards.com,sonyrewards.com,inbound,001,World,0 +soundcloudmail.com,soundcloudmail.com,inbound,001,World,0.999996 +sourceforge.net,sourceforge.net,inbound,001,World,1 +sourcenext.info,sourcenext.info,inbound,001,World,0 +southwest.com,southwest.com,inbound,001,World,0 +sp.gov.br,sp.gov.br,inbound,001,World,0.485842 +spanishdict.com,spanishdict.com,inbound,001,World,0.002698 +spareroom.co.uk,spareroom.co.uk,inbound,001,World,1 +sparklist.com,sparklist.com,inbound,001,World,0 +sparkpeople.com,sparkpeople.com,inbound,001,World,0 +spartoo.com,spartoo.com,inbound,001,World,0 +spectersoft.com,spectersoft.com,inbound,001,World,0 +speedyrewards-email.com,speedyrewards-email.com,inbound,001,World,0 +spencersonline.com,spencersonline.com,inbound,001,World,0 +spiritairlines.com,ctd004.net,inbound,001,World,0 +spiritairlines.com,ctd005.net,inbound,001,World,0 +splitwise.com,splitwise.com,inbound,001,World,1 +sportlobster.com,sportlobster.com,inbound,001,World,1 +sportsdirect.com,sportsdirect.com,inbound,001,World,0 +sportsline.com,cbsig.net,inbound,001,World,1 +sportsmansguide.com,sportsmansguide.com,inbound,001,World,0.736903 +spotifymail.com,spotifymail.com,inbound,001,World,1 +sprint.com,m0.net,inbound,001,World,0 +sqlservercentral.com,sqlservercentral.com,inbound,001,World,0 +square-enix.com,messagelabs.com,inbound,001,World,0.004493 +squareup.com,squareup.com,inbound,001,World,0.986452 +ssgadm.com,ssg.com,inbound,001,World,0 +staffeazymailers.com,iaires.com,inbound,001,World,0 +stakemail.com,iaires.com,inbound,001,World,0 +stampmail.in,iaires.com,inbound,001,World,0 +standaard.be,vummail.be,inbound,001,World,0 +standardbank.co.za,standardbank.co.za,inbound,001,World,0 +stanford.edu,highwire.org,inbound,001,World,1 +stanford.edu,stanford.edu,inbound,001,World,0.93025 +stansberryresearch.com,stansberry-re.net,inbound,001,World,0 +stansberryresearch.com,stansberryresearch.com,inbound,001,World,0.001008 +staples-pt.com,1-hostingservice.com,inbound,001,World,0 +staples.co.uk,ncrwebhost.de,inbound,001,World,0 +staples.com,staples.com,inbound,001,World,0.00638 +starbucks.com,iphmx.com,inbound,001,World,0.999476 +starbucks.com,starbucks.com,inbound,001,World,0 +stardockcorporation.com,stardockcorporation.com,inbound,001,World,0 +stardockentertainment.info,stardockentertainment.info,inbound,001,World,0 +starsports.com,eccluster.com,inbound,001,World,0 +startribune.com,startribune.com,inbound,001,World,0.001728 +startwire.com,jobsreport.com,inbound,001,World,1 +startwire.com,startwire.com,inbound,001,World,1 +starwoodhotels.com,outlook.com,inbound,001,World,1 +state-of-the-art-mailer.com,futurebanners.net,inbound,001,World,0 +state.gov,state.gov,inbound,001,World,0 +statefarm.com,statefarm.com,inbound,001,World,1 +stayfriends.de,stayfriends.de,inbound,001,World,0 +steampowered.com,steampowered.com,inbound,001,World,1 +steinmart.com,steinmart.com,inbound,001,World,0 +stelladot.com,stelladot.com,inbound,001,World,0 +stepstone.de,stepstone.com,inbound,001,World,0.001689 +stevemadden.com,stevemadden.com,inbound,001,World,0 +stjobs.sg,st701.com,inbound,001,World,1 +stjude.org,stjude.org,inbound,001,World,0.006716 +strava.com,strava.com,inbound,001,World,1 +streetauthoritydaily.com,streetauthoritydaily.com,inbound,001,World,0 +streeteasy.com,streeteasy.com,inbound,001,World,0 +striata.com,striata.com,inbound,001,World,0 +stubhub.com,stubhub.com,inbound,001,World,1 +studentbeans.com,emv8.com,inbound,001,World,0 +stumblemail.com,stumblemail.com,inbound,001,World,1 +stylecareers.com,stylecareers.com,inbound,001,World,0 +suafaturanet.com.br,suafaturanet.com.br,inbound,001,World,0.995846 +subito.it,subito.it,inbound,001,World,0 +subscribe.ru,subscribe.ru,inbound,001,World,0 +subscribermail.com,subscribermail.com,inbound,001,World,0 +subtend.info,subtend.info,inbound,001,World,1 +subway.com,subway.com,inbound,001,World,0 +sungard.com,postini.com,inbound,001,World,0.499946 +sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,001,World,1 +superbalist.com,sailthru.com,inbound,001,World,0 +superdeal.com.ua,mailersend.com,inbound,001,World,0 +superdrug.com,superdrug.com,inbound,001,World,0 +superjob.ru,superjob.ru,inbound,001,World,0 +supersafemailer.com,zoothost.com,inbound,001,World,0 +support-love.com,support-love.com,inbound,001,World,0 +supremelist.com,onlinehome-server.com,inbound,001,World,1 +surfmandelivery.com,surfmandelivery.com,inbound,001,World,0 +surlatable.com,surlatable.com,inbound,001,World,0 +surveyhelpcenter.com,jsmtp.net,inbound,001,World,0 +surveyjobopportunities.com,surveyjobopportunities.com,inbound,001,World,3.7e-05 +surveymonkey.com,surveymonkey.com,inbound,001,World,0 +surveysavvy.com,surveysavvy.com,inbound,001,World,0 +surveyspot.com,ssisurveys.com,inbound,001,World,0 +sut1.co.uk,sut1.co.uk,inbound,001,World,0.001789 +sut5.co.uk,sut5.co.uk,inbound,001,World,0 +swanson-vitamins.com,emv5.com,inbound,001,World,0 +sweepstakesalerts.com,sweepstakesalerts.com,inbound,001,World,0 +swimoutlet.com,isport.com,inbound,001,World,0 +sylectus.com,sylectus.com,inbound,001,World,0.361297 +sympatico.ca,hotmail.{...},inbound,001,World,1 +sympatico.ca,hotmail.{...},outbound,001,World,1 +synchronyfinancial.com,bigfootinteractive.com,inbound,001,World,0 +synergy360.jp,crmstyle.com,inbound,001,World,0 +t-online.de,t-online.de,inbound,001,World,1 +t-online.de,t-online.de,outbound,001,World,0.999939 +tadtopmails.com,tadtopmails.com,inbound,001,World,0 +taggedmail.com,taggedmail.com,inbound,001,World,0 +taipeifubon.com.tw,taipeifubon.com.tw,inbound,001,World,0 +taishinbank.com.tw,taishinbank.com.tw,inbound,001,World,0 +take2games.com,take2games.com,inbound,001,World,0 +talkmatch.com,talkmatch.com,inbound,001,World,0 +tamu.edu,tamu.edu,inbound,001,World,0.249656 +tanga.com,tanga.com,inbound,001,World,0 +tangeroutletsusa.com,bronto.com,inbound,001,World,0 +tanningmail.com,tanningmail.com,inbound,001,World,0 +tappingsolutionemail.com,tappingsolutionemail.com,inbound,001,World,0 +target-safelist.com,safelistpro.com,inbound,001,World,0.000215 +target.com,bigfootinteractive.com,inbound,001,World,0 +targetproblaster.com,targetproblaster.com,inbound,001,World,0 +targetx.com,targetx.com,inbound,001,World,0 +tarot.com,tarot.com,inbound,001,World,0 +tastefullysimpleparty.com,bigfootinteractive.com,inbound,001,World,0 +tasteofhome.com,tasteofhome.com,inbound,001,World,0 +tastingtable.com,tastingtable.com,inbound,001,World,0 +tatrabanka.sk,tatrabanka.sk,inbound,001,World,0 +taxi4sure.net,infimail.com,inbound,001,World,0 +tchibo.com.tr,euromsg.net,inbound,001,World,0 +tchibo.de,srv2.de,inbound,001,World,0.980445 +teach12.net,teach12.net,inbound,001,World,0 +teambuymail.com,teambuymail.com,inbound,001,World,0 +teamo.ru,teamo.ru,inbound,001,World,0 +teamsnap.com,teamsnap.com,inbound,001,World,1 +teamviewer.com,teamviewer.com,inbound,001,World,0 +teapartyinfo.org,teapartyinfo.org,inbound,001,World,0 +techgig.com,tbsl.in,inbound,001,World,0 +technolutions.net,technolutions.net,inbound,001,World,1 +techtarget.com,techtarget.com,inbound,001,World,0.001771 +telefonica.com,telefonica.com,inbound,001,World,0.998662 +telegraph.co.uk,telegraph.co.uk,inbound,001,World,0 +telenet.be,telenet-ops.be,inbound,001,World,0.000128 +teleportmyjob.com,clara.net,inbound,001,World,0 +telus.com,telus.com,inbound,001,World,0.000118 +telus.net,telus.net,inbound,001,World,0.006226 +templeandwebster.com.au,templeandwebster.com.au,inbound,001,World,5e-06 +ten24mail.com,ten24mail.com,inbound,001,World,0 +terra.com,terra.com,inbound,001,World,0.000463 +terra.com.br,terra.com,inbound,001,World,0 +terra.com.br,terra.com,outbound,001,World,0 +tesco.com,tesco.com,inbound,001,World,0 +testfunda.com,testfunda.com,inbound,001,World,0 +texasjobdepartment.com,texasjobdepartment.com,inbound,001,World,0 +textnow.me,textnow.me,inbound,001,World,0 +tgw.com,tgw.com,inbound,001,World,0 +theanimalrescuesite.com,theanimalrescuesite.com,inbound,001,World,0 +theatermania.com,wc09.net,inbound,001,World,0 +thebay.com,thebay.com,inbound,001,World,0 +thebodyshop-usa.com,email-bodyshop.com,inbound,001,World,0 +thebodyshop-usa.com,postdirect.com,inbound,001,World,0 +thecarousell.com,thecarousell.com,inbound,001,World,1 +thegrommet.com,lstrk.net,inbound,001,World,1 +theguardian.com,theguardian.com,inbound,001,World,0 +thehut.com,thehut.com,inbound,001,World,0 +theleadmagnet.com,your-server.de,inbound,001,World,1 +thelimited.com,thelimited.com,inbound,001,World,0 +themailbagsafelist.com,thomas-j-brown.com,inbound,001,World,0 +theoutnet.com,theoutnet.com,inbound,001,World,0 +thepamperedchef.com,thepamperedchef.com,inbound,001,World,0 +thephonehouse.es,splio.com,inbound,001,World,0.983578 +thephonehouse.es,splio.es,inbound,001,World,0.983266 +therealreal.com,email-realreal.com,inbound,001,World,0 +theskimm.com,theskimm.com,inbound,001,World,0 +thesource.ca,thesource.ca,inbound,001,World,0.00923 +thesovereigninvestor.com,sovereignsociety.com,inbound,001,World,0 +thewarehouse.co.nz,thewarehouse.co.nz,inbound,001,World,0 +thinkgeek.com,thinkgeek.com,inbound,001,World,1e-06 +thinkvidya.com,thinkvidya.com,inbound,001,World,0 +thirtyonegifts.com,thirtyonegifts.com,inbound,001,World,0 +thomascook.com,eccluster.com,inbound,001,World,0 +thoughtful-mind.com,thoughtful-mind.com,inbound,001,World,0 +thumbtack.com,thumbtack.com,inbound,001,World,1 +ticketmaster.com,ticketmaster.com,inbound,001,World,0.251337 +ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,001,World,0 +ticketmonster.co.kr,ticketmonster.co.kr,inbound,001,World,0 +timehop.com,timehop.com,inbound,001,World,1 +timeout.com,ec-cluster.com,inbound,001,World,0 +timesjobs.com,tbsl.in,inbound,001,World,0 +timesjobsmail.com,tbsl.in,inbound,001,World,0 +timesofindia.com,indiatimes.com,inbound,001,World,0 +timewarnercable.com,bigfootinteractive.com,inbound,001,World,0 +timeweb.ru,timeweb.ru,inbound,001,World,1 +tinyletterapp.com,tinyletterapp.com,inbound,001,World,0 +tiscali.it,tiscali.it,outbound,001,World,0 +tmart.com,chtah.net,inbound,001,World,0 +tmp.com,tmpw.com,inbound,001,World,0 +tobi.com,messagebus.com,inbound,001,World,0 +tobizaru.jp,tobizaru.jp,inbound,001,World,0 +tocoo.jp,aics.ne.jp,inbound,001,World,0 +toluna.com,toluna.com,inbound,001,World,0 +tomtommailer.com,tomtommailer.com,inbound,001,World,0 +topface.com,topface.com,inbound,001,World,1e-06 +topica.com,topica-silver-y.com,inbound,001,World,0 +topqpon.si,topqpon.si,inbound,001,World,0 +topspin.net,topspin.net,inbound,001,World,1 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 +touchbase2.com,infimail.com,inbound,001,World,0 +touchbase2.com,mailurja.com,inbound,001,World,0 +touchbasepro.com,touchbasepro.com,inbound,001,World,0 +tower.jp,tower.jp,inbound,001,World,0 +townhallmail.com,townhallmail.com,inbound,001,World,0 +townnews-mail.com,townnews-mail.com,inbound,001,World,0 +townsquaremedia.info,sailthru.com,inbound,001,World,0 +toysrus.com,epsl1.com,inbound,001,World,0 +trabajar.com,trabajo.org,inbound,001,World,0.999997 +trabalhar.com,trabajo.org,inbound,001,World,0.999994 +tradeloop.com,tradeloop.com,inbound,001,World,1 +trademe.co.nz,trademe.co.nz,inbound,001,World,0.991916 +trafficboostermailer.com,trafficboostermailer.com,inbound,001,World,1 +trafficleads2incomevm.com,zoothost.com,inbound,001,World,0 +trafficprolist.com,thomas-j-brown.com,inbound,001,World,0 +trafficwave.net,trafficwave.net,inbound,001,World,0 +transittraveljobinsider.com,transittraveljobinsider.com,inbound,001,World,0 +transportexchangegroup.com,transportexchangegroup.com,inbound,001,World,0.998703 +transversal.net,transversal.net,inbound,001,World,1 +travelchannel.com,travelchannel.com,inbound,001,World,0 +travelocity.com,travelocity.com,inbound,001,World,0.000194 +travelzoo.com,travelzoo.com,inbound,001,World,0 +travisperkins.co.uk,travisperkins.co.uk,inbound,001,World,0 +trclient.com,trclient.com,inbound,001,World,0 +treemall.com.tw,symphox.com,inbound,001,World,0 +trello.com,mandrillapp.com,inbound,001,World,1 +trialsmith.com,membercentral.com,inbound,001,World,0 +tricaemidia.com.br,tricaemidia.com.br,inbound,001,World,0 +triongames.com,triongames.com,inbound,001,World,0.085718 +tripadvisor.com,tripadvisor.com,inbound,001,World,0.000588 +tripit.com,tripit.com,inbound,001,World,0 +tripolis.com,tripolis.com,inbound,001,World,0 +trovit.com,trovit.com,inbound,001,World,0 +trulia.com,trulia.com,inbound,001,World,0 +tsite.jp,tsite.jp,inbound,001,World,0 +tsmmail.com,tsmmail.com,inbound,001,World,0 +tsutaya.co.jp,tsutaya.co.jp,inbound,001,World,0 +tucasa.com,grupodtm.com,inbound,001,World,0 +tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,001,World,0 +tumblr.com,tumblr.com,inbound,001,World,1 +turbine.com,turbine.com,inbound,001,World,0.013592 +turkcell.com.tr,turkcell.com.tr,inbound,001,World,0.043492 +turner.com,cnn.com,inbound,001,World,0 +twe-safelist.com,adminforfree.com,inbound,001,World,1 +twitch.tv,justin.tv,inbound,001,World,0 +twitter.com,postini.com,inbound,001,World,0.765163 +twitter.com,twitter.com,inbound,001,World,0.999969 +twoomail.com,netlogmail.com,inbound,001,World,0 +twoomail.com,twoomail.com,inbound,001,World,0 +type.jp,type.jp,inbound,001,World,0 +u-shopping.com.tw,u-shopping.com.tw,inbound,001,World,0 +uber.com,uber.com,inbound,001,World,1 +ubi.com,ubi.com,inbound,001,World,0 +ubivox.com,ubivox.com,inbound,001,World,0.956275 +ubuntu.com,canonical.com,inbound,001,World,0.000129 +ucla.edu,ucla.edu,inbound,001,World,0.806204 +udnpaper.com,udnpaper.com,inbound,001,World,0.998858 +udnshopping.com,udnshopping.com,inbound,001,World,0 +uga.edu,outlook.com,inbound,001,World,1 +uhaul.com,uhaul.com,inbound,001,World,0.997947 +uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,001,World,0 +uiuc.edu,illinois.edu,inbound,001,World,0.999815 +ukr.net,fwdcdn.com,inbound,001,World,1 +ukr.net,ukr.net,outbound,001,World,0.999992 +ulta.com,exacttarget.com,inbound,001,World,0 +ulta.com,ulta.com,inbound,001,World,0.034861 +ulteem.com,ulteem.com,inbound,001,World,0 +ultimateadsites.net,ultimateadsites.net,inbound,001,World,1 +umd.edu,umd.edu,inbound,001,World,0.021709 +umn.edu,umn.edu,inbound,001,World,0.966992 +umpiredigital.com,inboxmarketer.com,inbound,001,World,0 +unilever.com,mxlogic.net,inbound,001,World,1 +uniqlo-usa.com,uniqlo-usa.com,inbound,001,World,0 +united.com,coair.com,inbound,001,World,1 +united.com,united.com,inbound,001,World,0 +unitedrepublic.org,unitedrepublic.org,inbound,001,World,1 +universalorlando.com,universalorlando.com,inbound,001,World,0 +unosinsidersclub.com,unosinsidersclub.com,inbound,001,World,0 +unrollmail.com,unrollmail.com,inbound,001,World,1 +uol.com.br,uol.com.br,inbound,001,World,0 +uol.com.br,uol.com.br,outbound,001,World,0 +upenn.edu,upenn.edu,inbound,001,World,0.16521 +upromise.com,delivery.net,inbound,001,World,0 +ups.com,ups.com,inbound,001,World,0.997955 +urbanoutfitters.com,freepeople.com,inbound,001,World,0 +urx.com.br,urx.com.br,inbound,001,World,0 +usaa.com,usaa.com,inbound,001,World,0.639434 +usafisnews.org,usafisnews.org,inbound,001,World,0 +usahockey-email.com,usahockey-email.com,inbound,001,World,0 +usajobs.gov,opm.gov,inbound,001,World,0.931948 +usc.edu,usc.edu,inbound,001,World,0.300314 +uscourts.gov,uscourts.gov,inbound,001,World,0 +uslargestsafelist.com,zoothost.com,inbound,001,World,0 +usndr.com,unisender.com,inbound,001,World,1 +usndr.com,usndr.com,inbound,001,World,1 +usp.br,usp.br,inbound,001,World,0.044176 +usps.com,usps.gov,inbound,001,World,0.909588 +usps.gov,usps.gov,inbound,001,World,0.940315 +ustream.tv,mailgun.net,inbound,001,World,1 +ustream.tv,mailgun.us,inbound,001,World,1 +usx.com.br,uqx.com.br,inbound,001,World,0 +usx.com.br,usx.com.br,inbound,001,World,0 +usx.com.br,utx.com.br,inbound,001,World,0 +utfsm.cl,utfsm.cl,inbound,001,World,0.002318 +utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,001,World,0 +utx.com.br,utx.com.br,inbound,001,World,0 +uvarosa.com.br,uvarosa.com.br,inbound,001,World,0 +vacationstogo.com,vacationstogo.com,inbound,001,World,0 +vagas.com.br,vagas.com.br,inbound,001,World,0 +vakifbank.com.tr,vakifbank.com.tr,inbound,001,World,1 +valuedopinions.co.uk,researchnow-usa.com,inbound,001,World,1 +vanheusenrewards.com,vanheusenrewards.com,inbound,001,World,0 +vd.nl,emsecure.net,inbound,001,World,0 +velocityfrequentflyer.com,virginaustralia.com,inbound,001,World,0.003442 +venca.es,eccluster.com,inbound,001,World,0 +venere.com,kiwari.com,inbound,001,World,0 +vente-exclusive.com,vente-exclusive.com,inbound,001,World,0.000463 +venteprivee.com,venteprivee.com,inbound,001,World,0 +venusswimwear.net,venusswimwear.net,inbound,001,World,0 +verabradleymail.com,verabradleymail.com,inbound,001,World,0 +verizon.com,verizon.com,inbound,001,World,0.999793 +verizon.net,verizon.net,inbound,001,World,0.00713 +verizon.net,verizon.net,outbound,001,World,0 +verizon.net,yahoo.{...},inbound,001,World,0.999435 +verizonwireless.com,verizonwireless.com,inbound,001,World,0.972245 +vfoutletvip.com,vfoutletvip.com,inbound,001,World,0 +vhmnetworkemail.com,jobdiagnosis.com,inbound,001,World,0 +viagogo.com,chtah.net,inbound,001,World,0 +viajanet.com.br,viajanet.com.br,inbound,001,World,0.001345 +vicinity.nl,picsrv.net,inbound,001,World,0.022107 +victoriassecret.com,victoriassecret.com,inbound,001,World,0 +videotron.ca,videotron.ca,inbound,001,World,0.005886 +vietcombank.com.vn,vietcombank.com.vn,inbound,001,World,1 +vikingrivercruises.com,bfi0.com,inbound,001,World,0 +vimeo.com,vimeo.com,inbound,001,World,0 +vindale.com,vindale.com,inbound,001,World,0 +vipvoice.com,npdor.com,inbound,001,World,0 +viraladmagnet.com,viraladmagnet.com,inbound,001,World,1 +viralsender.com,viralsender.com,inbound,001,World,0 +virgilio.it,virgilio.net,inbound,001,World,0 +virginia.edu,virginia.edu,inbound,001,World,0.055153 +virtualtarget.com.br,virtualtarget.com.br,inbound,001,World,0 +vistaprint.com,vistaprint.com,inbound,001,World,0 +vistaprint.com.au,vistaprint.com.au,inbound,001,World,0 +visualsoft.co.uk,visualsoft.co.uk,inbound,001,World,0 +vitacost.com,vitacost.com,inbound,001,World,0 +vitaladviews.com,zoothost.com,inbound,001,World,0 +vitaminworld.com,email-nbtyinc.com,inbound,001,World,0 +vivastreet.com,viwii.net,inbound,001,World,0 +vk.com,vkontakte.ru,inbound,001,World,0 +vocus.com,vocus.com,inbound,001,World,0 +vodafone.com,vodafone.in,inbound,001,World,0.617518 +vonage.com,vonagenetworks.net,inbound,001,World,0.006531 +votervoice.net,votervoice.net,inbound,001,World,0 +vouchercloud.com,vouchercloud.com,inbound,001,World,0 +vovici.com,vovici.com,inbound,001,World,0 +voyageprive.com,cccampaigns.net,inbound,001,World,0 +voyageprive.es,ccmdcampaigns.net,inbound,001,World,0 +voyageprive.it,ccmdcampaigns.net,inbound,001,World,0 +voyages-sncf.com,neolane.net,inbound,001,World,0 +vpass.ne.jp,clickmailer.jp,inbound,001,World,0 +vpcontact.com,vpcontact.com,inbound,001,World,0 +vresp.com,verticalresponse.com,inbound,001,World,0 +vt.edu,vt.edu,inbound,001,World,0.978548 +vtext.com,vtext.com,inbound,001,World,0 +vtext.com,vtext.com,outbound,001,World,1 +vudu.com,vudu.com,inbound,001,World,0 +vueling.com,vueling.com,inbound,001,World,0 +vuezone.com,vuezone.com,inbound,001,World,0 +vzwpix.com,vtext.com,inbound,001,World,0 +wagjag.com,wagjag.com,inbound,001,World,0 +walgreens.com,walgreens.com,inbound,001,World,0.006128 +wallst.com,wallst.com,inbound,001,World,0 +wallstreetdaily.com,wallstreetdaily.com,inbound,001,World,0 +walmart.ca,walmart.ca,inbound,001,World,0 +walmart.com,walmart.com,inbound,001,World,0.315059 +wanadoo.fr,orange.fr,inbound,001,World,0 +wanadoo.fr,orange.fr,outbound,001,World,0 +warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,001,World,0 +waterstones.com,waterstones.com,inbound,001,World,0 +wattpad.com,wattpad.com,inbound,001,World,1 +waves-audio.com,emv8.com,inbound,001,World,0 +way2movies.net,way2movies.net,inbound,001,World,0 +way2sms.biz,way2sms.biz,inbound,001,World,0 +way2sms.in,way2sms.in,inbound,001,World,0 +way2smsemail.com,way2smsemail.com,inbound,001,World,0 +way2smsemails.com,way2smsemails.com,inbound,001,World,0 +way2smsmail.in,way2smsmail.in,inbound,001,World,0 +way2smsmails.com,way2smsmails.com,inbound,001,World,0 +wayfair.com,csnstores.com,inbound,001,World,0 +wealthyaffiliate.com,wealthyaffiliate.com,inbound,001,World,1 +web.de,web.de,inbound,001,World,0.999986 +web.de,web.de,outbound,001,World,1 +webcas.net,webcas.net,inbound,001,World,0 +webmd.com,webmd.com,inbound,001,World,2e-06 +webs.com,epsl1.com,inbound,001,World,0 +websaver.ca,websaver.ca,inbound,001,World,0 +websitesettings.com,stabletransit.com,inbound,001,World,0 +websitewelcome.com,websitewelcome.com,inbound,001,World,1 +webstars2k.com,webstars2k.com,inbound,001,World,1 +wechat.com,qq.com,inbound,001,World,1 +weebly.com,weeblymail.com,inbound,001,World,0 +wegottickets.com,wegottickets.com,inbound,001,World,0 +weheartit.com,weheartit.com,inbound,001,World,1 +wehkamp.nl,wehkamp.nl,inbound,001,World,0 +wellsfargo.com,wellsfargo.com,inbound,001,World,1.0 +wellsfargoadvisors.com,wellsfargo.com,inbound,001,World,1 +wemakeprice.com,wemakeprice.com,inbound,001,World,0 +westelm.com,westelm.com,inbound,001,World,0 +westmarine.com,westmarine.com,inbound,001,World,0 +westwing.com.br,cust-cluster.com,inbound,001,World,0 +westwing.es,ecm-cluster.com,inbound,001,World,0 +westwing.ru,ecm-cluster.com,inbound,001,World,0 +wetransfer.com,wetransfer.com,inbound,001,World,0.999751 +wetsealnewsletter.com,wetsealnewsletter.com,inbound,001,World,0 +wgbh.org,wgbh.org,inbound,001,World,0 +whaakky.com,whaakky.com,inbound,001,World,0 +whatcounts.com,wc09.net,inbound,001,World,0 +whentowork.com,whentowork.com,inbound,001,World,0 +whereareyounow.com,wayn.net,inbound,001,World,0 +whitehouse.gov,whitehouse.gov,inbound,001,World,0 +whitelabelpros.com,whitelabelpros.com,inbound,001,World,0 +wiggle.com,wiggle.com,inbound,001,World,0 +wikia.com,wikia.com,inbound,001,World,0.550228 +wikimedia.org,wikimedia.org,inbound,001,World,0 +william-reed.com,neolane.net,inbound,001,World,0 +williamhill.com,williamhill.com,inbound,001,World,0 +williams-sonoma.com,williams-sonoma.com,inbound,001,World,0 +windstream.net,windstream.net,inbound,001,World,0.002172 +wine.com,wine.com,inbound,001,World,0 +winkalmail.com,fnbox.com,inbound,001,World,0 +wisdomitservices.com,infimail.com,inbound,001,World,0 +wldemail-mailer.com,wldemail.com,inbound,001,World,0 +wldemail.com,emarsys.net,inbound,001,World,0 +wmtransfer.com,wmtransfer.com,inbound,001,World,0 +wnd.com,emv4.net,inbound,001,World,0 +wnd.com,worldnetdaily.com,inbound,001,World,0 +wolfmedia.us,wolfmedia.us,inbound,001,World,0 +womanwithin.com,womanwithin.com,inbound,001,World,0 +woodforest.com,woodforest.com,inbound,001,World,1 +wordfly.com,wordfly.com,inbound,001,World,0 +work.ua,work.ua,inbound,001,World,1 +workcircle.com,workcircle.net,inbound,001,World,0 +workhunter.net,workhunter.net,inbound,001,World,1 +workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,001,World,0 +worldsingles.co,worldsingles.com,inbound,001,World,0 +worldwinner.com,worldwinner.com,inbound,001,World,0.112191 +wotif.com,whatcounts.com,inbound,001,World,0 +wowcher.co.uk,wowcher.co.uk,inbound,001,World,0.000373 +wp.com,wordpress.com,inbound,001,World,0 +wp.pl,wp.pl,inbound,001,World,0.998027 +wp.pl,wp.pl,outbound,001,World,1 +wpengine.com,wpengine.com,inbound,001,World,1 +writers-community.com,writers-community.com,inbound,001,World,0 +writersstore.com,writersstore.com,inbound,001,World,0 +wsjemail.com,wsjemail.com,inbound,001,World,0 +wuaki.tv,chtah.net,inbound,001,World,0 +wustl.edu,app-info.net,inbound,001,World,0 +wwe.com,wwe.com,inbound,001,World,8e-06 +www.gov.tw,hinet.net,inbound,001,World,8e-05 +wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,001,World,0 +xbox.com,xbox.com,inbound,001,World,0 +xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,001,World,0 +xen.org,xen.org,inbound,001,World,1 +xing.com,xing.com,inbound,001,World,0 +xmailix.com,xmailix.com,inbound,001,World,0 +xmeeting.com,xmeeting.com,inbound,001,World,1 +xmr3.com,messagereach.com,inbound,001,World,0.999933 +xoom.com,xoom.com,inbound,001,World,0 +xpnews.com.br,xpnews.com.br,inbound,001,World,1 +xxxconnect.com,infinitypersonals.com,inbound,001,World,0 +yahoo-inc.com,yahoo.{...},inbound,001,World,0.999974 +yahoo.co.jp,yahoo.co.jp,inbound,001,World,1.5e-05 +yahoo.co.jp,yahoo.co.jp,outbound,001,World,0 +yahoo.{...},postini.com,inbound,001,World,0.767368 +yahoo.{...},yahoo.co.jp,inbound,001,World,0 +yahoo.{...},yahoo.{...},inbound,001,World,0.999416 +yahoo.{...},yahoodns.net,outbound,001,World,1 +yahoogroups.com,yahoodns.net,outbound,001,World,1 +yakala.co,euromsg.net,inbound,001,World,0 +yammer.com,yammer.com,inbound,001,World,1 +yandex.ru,yandex.net,inbound,001,World,0.999698 +yandex.ru,yandex.ru,outbound,001,World,1 +yapikredi.com.tr,yapikredi.com.tr,inbound,001,World,1 +yapstone.com,yapstone.com,inbound,001,World,0 +yelp.com,yelpcorp.com,inbound,001,World,0 +yesbank.in,yesbank.in,inbound,001,World,0.108676 +yipit.com,yipit.com,inbound,001,World,1 +ymail.com,yahoo.{...},inbound,001,World,1 +ymail.com,yahoodns.net,outbound,001,World,1 +ymlpserver.net,ymlpserver.net,inbound,001,World,0 +ymlpsrv.net,ymlpsrv.net,inbound,001,World,0 +yodobashi.com,yodobashi.com,inbound,001,World,0 +yoox.com,yoox.com,inbound,001,World,0 +youmail.com,youmail.com,inbound,001,World,0 +youravon.com,email-avonglobal.com,inbound,001,World,0 +youreletters3.com,equitymaster.com,inbound,001,World,0 +yourezads.com,yourezads.com,inbound,001,World,0.002974 +yourezlist.com,simplicityads.com,inbound,001,World,9.5e-05 +yourhostingaccount.com,yourhostingaccount.com,inbound,001,World,0 +yournewsletters.net,everydayhealth.com,inbound,001,World,0 +youversion.com,youversion.com,inbound,001,World,1 +zacks.com,zacks.com,inbound,001,World,0.999188 +zalando.be,fagms.de,inbound,001,World,0 +zalando.dk,fagms.de,inbound,001,World,0 +zalando.fi,fagms.de,inbound,001,World,0 +zalando.it,fagms.de,inbound,001,World,0 +zalando.nl,fagms.de,inbound,001,World,0 +zalando.pl,fagms.de,inbound,001,World,0 +zappos.com,zappos.com,inbound,001,World,0.626758 +zara.com,cheetahmail.com,inbound,001,World,0 +zattoo.com,sendnode.com,inbound,001,World,0 +zelonews.com.br,zelonews.com.br,inbound,001,World,0 +zendesk.com,zdsys.com,inbound,001,World,1 +zgalleriestyle.com,zgalleriestyle.com,inbound,001,World,0 +zibmail.info,zibmail.info,inbound,001,World,2e-06 +zillow.com,zillow.com,inbound,001,World,2.82542783334609e-07 +zinio.com,zinio.com,inbound,001,World,0.006193 +zinio.net,zinio.com,inbound,001,World,1 +zipalerts.com,sendgrid.net,inbound,001,World,1 +zipalerts.com,zipalerts.com,inbound,001,World,1 +ziprealty.com,ziprealty.com,inbound,001,World,0.994545 +ziprecruiter.com,ziprecruiter.com,inbound,001,World,1 +zivamewear.com,infimail.com,inbound,001,World,0 +zizigo.com,euromsg.net,inbound,001,World,0 +zlavadna.sk,zlavadna.sk,inbound,001,World,0 +zoom.com.br,zoom.com.br,inbound,001,World,0 +zoominternet.net,synacor.com,inbound,001,World,0 +zoosk.com,zoosk.com,inbound,001,World,0 +zoothost.com,zoothost.com,inbound,001,World,0.107168 +zorpia.com,zorpia.com,inbound,001,World,0.56104 +zovifashion.com,eccluster.com,inbound,001,World,0 +zulily.com,zulily.com,inbound,001,World,0 +zumzi.com,neogen.ro,inbound,001,World,0 +zumzi.com,zumzi.com,inbound,001,World,0 +zyngamail.com,zyngamail.com,inbound,001,World,0 +zzounds.com,zzounds.com,inbound,001,World,0 +careers24.com,careers24.com,inbound,002,Africa,0 +cpm.co.ma,cpm.co.ma,inbound,002,Africa,0 +fnb.co.za,fnb.co.za,inbound,002,Africa,0 +gmail.com,telkomadsl.co.za,inbound,002,Africa,0.999989 +gmail.com,vodacom.co.za,inbound,002,Africa,1 +gtbank.com,gtbank.com,inbound,002,Africa,0.056121 +pnetweb.co.za,hosting.co.za,inbound,002,Africa,0 +pnetweb.co.za,salesnet.co.za,inbound,002,Africa,0 +bigpond.com,bigpond.com,inbound,009,Oceania,0 +bigpond.com,bigpond.com,outbound,009,Oceania,1 +empoweredcomms.com.au,empoweredcomms.com.au,inbound,009,Oceania,0 +gmail.com,bigpond.net.au,inbound,009,Oceania,0.999907 +gmail.com,iinet.net.au,inbound,009,Oceania,0.966945 +gmail.com,optusnet.com.au,inbound,009,Oceania,0.992845 +gmail.com,tpgi.com.au,inbound,009,Oceania,0.999648 +mbounces.com,emdbms.com,inbound,009,Oceania,0 +realestate.com.au,realestate.com.au,inbound,009,Oceania,0 +snaphire.com,snaphire.com,inbound,009,Oceania,0 +trademe.co.nz,trademe.co.nz,inbound,009,Oceania,0.991916 +1-day.co.nz,1-day.co.nz,inbound,019,Americas,0 +12manage.com,netarrest.com,inbound,019,Americas,0.99998 +1800petmeds.com,1800petmeds.com,inbound,019,Americas,0.00599 +2touchbase.com,infimail.com,inbound,019,Americas,0 +4wheelparts.com,4wheelparts.com,inbound,019,Americas,0 +99acres.com,99acres.com,inbound,019,Americas,0 +aafes.com,aafes.com,inbound,019,Americas,0 +abercrombie-email.com,abercrombie-email.com,inbound,019,Americas,0 +abercrombiekids-email.com,abercrombie-email.com,inbound,019,Americas,0 +about.com,about.com,inbound,019,Americas,0 +about.com,sailthru.com,inbound,019,Americas,0 +acemserv.com,acemserv.com,inbound,019,Americas,0 +activesafelist.com,zoothost.com,inbound,019,Americas,0 +adidasusnews.com,adidasusnews.com,inbound,019,Americas,0 +adjockeys.com,thomas-j-brown.com,inbound,019,Americas,0 +admastersafelist.com,zoothost.com,inbound,019,Americas,0 +adorama.com,adorama.com,inbound,019,Americas,0 +adpirate.net,thomas-j-brown.com,inbound,019,Americas,0 +adtpulse.com,adtpulse.com,inbound,019,Americas,0 +ae.com,ae.com,inbound,019,Americas,0 +aexp.com,aexp.com,inbound,019,Americas,1 +affairalert.com,iverificationsystems.com,inbound,019,Americas,0 +agorafinancial.com,agorafinancial.com,inbound,019,Americas,0 +airbnb.com,airbnb.com,inbound,019,Americas,0.867975 +airbrake.io,mailgun.net,inbound,019,Americas,1 +airfarewatchdog.com,smartertravelmedia.com,inbound,019,Americas,0.012002 +airmiles.ca,bigfootinteractive.com,inbound,019,Americas,0 +akcijatau.lt,akcijatau.lt,inbound,019,Americas,0 +alarm.com,alarm.com,inbound,019,Americas,0 +alarmnet.com,alarmnet.com,inbound,019,Americas,0 +alaskaair.com,alaskaair.com,inbound,019,Americas,0 +alertid.com,alertid.com,inbound,019,Americas,0 +allheart.com,allheart.com,inbound,019,Americas,0 +allmodern.com,allmodern.com,inbound,019,Americas,0 +allout.org,allout.org,inbound,019,Americas,1 +allrecipes.com,allrecipes.com,inbound,019,Americas,0 +allstate.com,rsys1.com,inbound,019,Americas,0 +alm.com,sailthru.com,inbound,019,Americas,0 +alumniclass.com,alumniclass.com,inbound,019,Americas,0 +alumniconnections.com,alumniconnections.com,inbound,019,Americas,0 +amazon.{...},postini.com,inbound,019,Americas,0.658136 +amazonses.com,postini.com,inbound,019,Americas,0.733998 +americanas.com,americanas.com,inbound,019,Americas,0 +americanbar.org,abanet.org,inbound,019,Americas,0.00574 +amubm.com,amubm.com,inbound,019,Americas,1 +ancestry.com,ancestry.com,inbound,019,Americas,0 +angelbroking.in,infimail.com,inbound,019,Americas,0 +anghami.com,mailgun.net,inbound,019,Americas,1 +anthropologie.com,freepeople.com,inbound,019,Americas,0 +aol.com,aol.com,inbound,019,Americas,0.999528 +aol.com,aol.com,outbound,019,Americas,0.999984 +aol.com,sailthru.com,inbound,019,Americas,0 +aol.net,aol.com,inbound,019,Americas,1 +apache.org,apache.org,inbound,019,Americas,0 +apnacomplex.com,apnacomplex.com,inbound,019,Americas,0.999981 +apply-4-jobs.com,apply-4-jobs.com,inbound,019,Americas,0 +aptmail.in,mailurja.com,inbound,019,Americas,0 +arcamax.com,arcamax.com,inbound,019,Americas,1.4e-05 +aritzia.com,aritzia.com,inbound,019,Americas,0 +armaniexchange.com,bronto.com,inbound,019,Americas,0 +asadventure.com,asadventure.com,inbound,019,Americas,0 +asana.com,asana.com,inbound,019,Americas,1 +ashleymadison.com,ashleymadison.com,inbound,019,Americas,1 +asos.com,asos.com,inbound,019,Americas,0 +assembla.com,assembla.com,inbound,019,Americas,1 +astrology.com,astrology.com,inbound,019,Americas,0 +astrology.com,hsnlmailsvc.com,inbound,019,Americas,0 +astrology.com,webstakes.com,inbound,019,Americas,0 +astrology.com,wsafmailsvc.com,inbound,019,Americas,0 +atlassian.net,uc-inf.net,inbound,019,Americas,1 +atrapalo.cl,atrapalo.com,inbound,019,Americas,0 +atrapalo.com,atrapalo.com,inbound,019,Americas,0 +att-mail.com,att-mail.com,inbound,019,Americas,1.9e-05 +att-mail.com,att.com,inbound,019,Americas,0.999966 +att.net,att.net,outbound,019,Americas,0.204629 +att.net,mycingular.net,inbound,019,Americas,0.00021 +att.net,yahoo.{...},inbound,019,Americas,1 +australiagsm.net,australiagsm.net,inbound,019,Americas,0 +autoloop.us,loop28.com,inbound,019,Americas,0 +avaaz.org,avaaz.org,inbound,019,Americas,0 +avalanchesafelist.com,zoothost.com,inbound,019,Americas,0 +aveda.com,esteelauder.com,inbound,019,Americas,0 +avenue.com,avenue.com,inbound,019,Americas,0 +avg.com,avg.com,inbound,019,Americas,0 +avomail.com,avomail.com,inbound,019,Americas,0 +b2b-mail.net,b2b-mail.net,inbound,019,Americas,0 +b2b-mail.net,contact-list.net,inbound,019,Americas,0 +babycenter.com,rsys3.com,inbound,019,Americas,0 +babyoye.com,babyoye.com,inbound,019,Americas,0.011564 +badoo.com,monopost.com,inbound,019,Americas,1 +baligam.co.il,baligam.co.il,inbound,019,Americas,1 +banamex.com,ibrands.es,inbound,019,Americas,0 +bancoahorrofamsa.com,avantel.net.mx,inbound,019,Americas,1 +bancomercorreo.com,bancomercorreo.com,inbound,019,Americas,0 +bandsintown.com,bandsintown.com,inbound,019,Americas,1 +barclaycard.co.uk,barclays.co.uk,inbound,019,Americas,0 +barenecessities.com,barenecessities.com,inbound,019,Americas,0 +barleyment.ca,barleyment.ca,inbound,019,Americas,0 +barneys.com,barneys.com,inbound,019,Americas,0 +baseballsavings.com,baseballsavings.com,inbound,019,Americas,0 +basspronews.com,basspronews.com,inbound,019,Americas,0 +bathandbodyworks.com,bathandbodyworks.com,inbound,019,Americas,0 +baublebar.com,baublebar.com,inbound,019,Americas,0.011219 +bayt.com,bayt.com,inbound,019,Americas,2e-06 +bcp.com.pe,bcp.com.pe,inbound,019,Americas,1 +bellsouth.net,att.net,outbound,019,Americas,0 +bellsouth.net,yahoo.{...},inbound,019,Americas,1 +bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +bespokeoffers.co.uk,chtah.net,inbound,019,Americas,0 +bestbuy.ca,bestbuy.ca,inbound,019,Americas,0 +bestdealsforyou.in,elabs5.com,inbound,019,Americas,0 +bevmo.com,bevmo.com,inbound,019,Americas,0.000967 +bhcosmetics.com,bronto.com,inbound,019,Americas,0 +bhg.com,meredith.com,inbound,019,Americas,0 +bigfishgames.com,bigfishgames.com,inbound,019,Americas,0 +biglist.com,biglist.com,inbound,019,Americas,0 +bigtent.com,carezen.net,inbound,019,Americas,0 +bionexo.com,bionexo.com.br,inbound,019,Americas,0.999594 +birthdayalarm.com,monkeyinferno.net,inbound,019,Americas,0 +bitbucket.org,bitbucket.org,inbound,019,Americas,0 +bitlysupport.com,mailgun.info,inbound,019,Americas,1 +bitlysupport.com,mailgun.us,inbound,019,Americas,1 +bitslane.email,bitslane.email,inbound,019,Americas,0 +bitstatement.org,bitstatement.org,inbound,019,Americas,1 +bizjournals.com,bizjournals.com,inbound,019,Americas,0 +bizmailtoday.com,bizmailtoday.com,inbound,019,Americas,0 +bjs.com,bjs.com,inbound,019,Americas,0 +blablacar.com,blablacar.com,inbound,019,Americas,1 +blackberry.com,blackberry.com,inbound,019,Americas,0 +blackboard.com,blackboard.com,inbound,019,Americas,1 +blissworld.com,lstrk.net,inbound,019,Americas,1 +bloomingdales.com,bloomingdales.com,inbound,019,Americas,0 +bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,019,Americas,0 +bluehost.com,bluehost.com,inbound,019,Americas,0.000971 +bluehost.com,hostmonster.com,inbound,019,Americas,0 +bluehost.com,unifiedlayer.com,inbound,019,Americas,0 +blueshellgames.com,blueshellgames.com,inbound,019,Americas,0 +bluestatedigital.com,bluestatedigital.com,inbound,019,Americas,0 +bluestonemx.com,bluestonemx.com,inbound,019,Americas,1 +bm23.com,bronto.com,inbound,019,Americas,0 +bm324.com,bronto.com,inbound,019,Americas,0 +bmdeda99.com,bmdeda99.com,inbound,019,Americas,0 +bn.com,bn.com,inbound,019,Americas,0 +bnetmail.com,bnetmail.com,inbound,019,Americas,0 +bol.com.br,bol.com.br,inbound,019,Americas,0 +bol.com.br,bol.com.br,outbound,019,Americas,0 +boletinrenuevo.com,boletinrenuevo.com,inbound,019,Americas,0 +bomnegocio.com,bomnegocio.com,inbound,019,Americas,0.588708 +bonobos.com,bronto.com,inbound,019,Americas,0 +bookbub.com,bookbub.com,inbound,019,Americas,1 +booking.com,booking.com,inbound,019,Americas,1 +bookingbuddy.com,smartertravelmedia.com,inbound,019,Americas,0.000487 +boomtownroi.com,boomtownroi.com,inbound,019,Americas,0 +boscovs.com,boscovs.com,inbound,019,Americas,0 +bostonproper.com,bostonproper.com,inbound,019,Americas,0 +boutiquesecret.com,chtah.net,inbound,019,Americas,0 +bradfordexchange.com,bradfordexchange.com,inbound,019,Americas,0 +bradsdeals.com,bradsdeals.com,inbound,019,Americas,0 +brassring.com,brassring.com,inbound,019,Americas,1 +briantracyintl.com,briantracyintl.com,inbound,019,Americas,0 +brijj.com,brijj.com,inbound,019,Americas,0 +brincltd.com,brincltd.com,inbound,019,Americas,0 +bronto.com,bronto.com,inbound,019,Americas,0 +bsf01.com,bsftransmit33.com,inbound,019,Americas,0 +buffalo.edu,buffalo.edu,inbound,019,Americas,0.001059 +bumeran.com,bumeran.com,inbound,019,Americas,0 +burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,019,Americas,0 +burton.co.uk,burton.co.uk,inbound,019,Americas,0 +buscojobs.com,amazonaws.com,inbound,019,Americas,0 +buy123.com.tw,buy123.com.tw,inbound,019,Americas,1 +bzm.mobi,nmsrv.com,inbound,019,Americas,1 +c21stores.com,c21stores.com,inbound,019,Americas,0 +ca.gov,ca.gov,inbound,019,Americas,0.702758 +cafepress.com,cafepress.com,inbound,019,Americas,0.000778 +caixa.gov.br,caixa.gov.br,inbound,019,Americas,0 +californiapsychicsemail.com,californiapsychicsemail.com,inbound,019,Americas,0 +calottery.com,calottery.com,inbound,019,Americas,0.999426 +camel.com,rjrsignup.com,inbound,019,Americas,0 +canadianvisaexpert.net,canadianvisaexpert.net,inbound,019,Americas,0 +cancer.org,delivery.net,inbound,019,Americas,0 +capillary.co.in,capillary.co.in,inbound,019,Americas,1 +capitalone.com,bigfootinteractive.com,inbound,019,Americas,0 +capitalone360.com,ingdirect.com,inbound,019,Americas,0 +care.com,care.com,inbound,019,Americas,0 +care2.com,care2.com,inbound,019,Americas,0 +careerage.com,careerage.com,inbound,019,Americas,0 +careerflash.net,careerflash.net,inbound,019,Americas,1 +carmamail.com,carmamail.com,inbound,019,Americas,0 +carnivalfunmail.com,carnivalfunmail.com,inbound,019,Americas,0 +carolsdaughter.com,carolsdaughter.com,inbound,019,Americas,0 +carwale.com,carwale.com,inbound,019,Americas,0 +case.edu,cwru.edu,inbound,019,Americas,0.999942 +castingnetworks.com,castingnetworks.com,inbound,019,Americas,0.000102 +causes.com,causes.com,inbound,019,Americas,1 +cb2.com,cb2.com,inbound,019,Americas,0 +cbsig.net,cbsig.net,inbound,019,Americas,1 +ccbchurch.com,ccbchurch.com,inbound,019,Americas,1 +ccialerts.com,ccialerts.com,inbound,019,Americas,0 +ccs.com,footlocker.com,inbound,019,Americas,2.5e-05 +cenlat.com,cenlat.com,inbound,019,Americas,0.064459 +cerberusapp.com,cerberusapp.com,inbound,019,Americas,1 +cfmvmail.com,cfmvmail.com,inbound,019,Americas,0 +chabad.org,chabad.org,inbound,019,Americas,0 +champssports.com,footlocker.com,inbound,019,Americas,0.000131 +change.org,change.org,inbound,019,Americas,1 +charlestyrwhitt.com,charlestyrwhitt.com,inbound,019,Americas,0 +charter.net,charter.net,inbound,019,Americas,0 +charter.net,charter.net,outbound,019,Americas,0 +chase.com,jpmchase.com,inbound,019,Americas,0.999999 +chatcitynotifications.com,chatcitynotifications.com,inbound,019,Americas,0 +chaturbate.com,chaturbate.com,inbound,019,Americas,1 +cheapairmailer.com,cheapairmailer.com,inbound,019,Americas,0 +check.me,check.me,inbound,019,Americas,0 +cheekylovers.com,ropot.net,inbound,019,Americas,0 +chess.com,chess.com,inbound,019,Americas,1 +chicagotribune.com,latimes.com,inbound,019,Americas,0 +chicos.com,chicos.com,inbound,019,Americas,0.000156 +childrensplace.com,childrensplace.com,inbound,019,Americas,0 +christianbook.com,christianbook.com,inbound,019,Americas,1 +christianmingle.com,christianmingle.com,inbound,019,Americas,0 +chtah.com,chtah.net,inbound,019,Americas,0 +chtah.net,chtah.net,inbound,019,Americas,0 +cincghq.com,searchhomesingta.com,inbound,019,Americas,1 +circleofmomsmail.com,circleofmomsmail.com,inbound,019,Americas,0 +citibank.com,bigfootinteractive.com,inbound,019,Americas,0 +ck.com,ck.com,inbound,019,Americas,0 +clarks.com,clarks.com,inbound,019,Americas,0 +clickdimensions.com,clickdimensions.com,inbound,019,Americas,0 +clickexperts.net,clickexperts.net,inbound,019,Americas,0 +clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,019,Americas,0 +climber.com,climber.com,inbound,019,Americas,0 +clinique.com,esteelauder.com,inbound,019,Americas,0 +clubcupon.com.ar,clubcupon.com.ar,inbound,019,Americas,0 +cmm01.com,coremotivesmarketing.com,inbound,019,Americas,0 +coach.com,delivery.net,inbound,019,Americas,0 +codeproject.com,codeproject.com,inbound,019,Americas,0 +coldwatercreek.com,coldwatercreek.com,inbound,019,Americas,0 +collectionsetc.com,collectionsetc.com,inbound,019,Americas,0 +columbia.edu,columbia.edu,inbound,019,Americas,0.762355 +comcast.net,comcast.net,inbound,019,Americas,0.919244 +comcast.net,comcast.net,outbound,019,Americas,0.999998 +comenity.net,alldata.net,inbound,019,Americas,1 +comenity.net,bigfootinteractive.com,inbound,019,Americas,0 +commonfloor.com,commonfloor.com,inbound,019,Americas,1 +compute.internal,amazonaws.com,inbound,019,Americas,0.875148 +computerworld.com,computerworld.com,inbound,019,Americas,0 +comunicacaodemkt.com,locaweb.com.br,inbound,019,Americas,0 +constantcontact.com,confirmedcc.com,inbound,019,Americas,0 +constantcontact.com,constantcontact.com,inbound,019,Americas,5.3e-05 +constantcontact.com,postini.com,inbound,019,Americas,0.042128 +containerstore.com,containerstore.com,inbound,019,Americas,0 +convio.net,convio.net,inbound,019,Americas,0 +coremotivesmarketing.com,coremotivesmarketing.com,inbound,019,Americas,0 +cornell.edu,cornell.edu,inbound,019,Americas,0.192285 +correosocc.com,correosocc.com,inbound,019,Americas,1 +costco.co.uk,costco.com,inbound,019,Americas,0 +costco.com,costco.com,inbound,019,Americas,7e-06 +costcophotocenter.com,wc09.net,inbound,019,Americas,0 +costcoservices.com,costco.com,inbound,019,Americas,0 +cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,019,Americas,0 +couchsurfing.org,couchsurfing.com,inbound,019,Americas,0 +couponamama.com,couponamama.com,inbound,019,Americas,1 +cox.com,cox.com,inbound,019,Americas,0.001665 +cox.net,cox.net,inbound,019,Americas,0.009187 +cox.net,cox.net,outbound,019,Americas,0 +coyotelogistics.com,postini.com,inbound,019,Americas,0 +cp20.com,cp20.com,inbound,019,Americas,0 +cpbnc.com,cpbnc.com,inbound,019,Americas,0 +cpbnc.com,fye.com,inbound,019,Americas,0 +crackle.com,crackle.com,inbound,019,Americas,0 +craigslist.org,craigslist.org,inbound,019,Americas,0 +craigslist.org,craigslist.org,outbound,019,Americas,1 +crashlytics.com,crashlytics.com,inbound,019,Americas,1 +crashlytics.com,sendgrid.net,inbound,019,Americas,1 +crateandbarrel.com,crateandbarrel.com,inbound,019,Americas,0 +creationsrewards.net,creationsrewards.net,inbound,019,Americas,0 +creditkarma.com,creditkarma.com,inbound,019,Americas,1 +crosswalkmail.com,crosswalkmail.com,inbound,019,Americas,0 +crowdcut.com,crowdcut.com,inbound,019,Americas,1 +crunchyroll.com,crunchyroll.com,inbound,019,Americas,0 +cumulusdist.net,cumulusdist.net,inbound,019,Americas,0 +cuponatic.com.pe,cuponatic.com.pe,inbound,019,Americas,1 +cuponicamail.com,fnbox.com,inbound,019,Americas,0 +curbednetwork.com,curbednetwork.com,inbound,019,Americas,1 +cuspemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +custombriefings.com,custombriefings.com,inbound,019,Americas,0 +customercenter.net,customercenter.net,inbound,019,Americas,0.996453 +customeriomail.com,customeriomail.com,inbound,019,Americas,1 +cxomedia.com,cxomedia.com,inbound,019,Americas,0 +cybercoders.com,cybercoders.com,inbound,019,Americas,0 +dafiti.cl,dafiti.cl,inbound,019,Americas,0 +dailyhoroscope.com,tarot.com,inbound,019,Americas,0 +datehookup.com,datehookup.com,inbound,019,Americas,0 +datingvipnotifications.com,datingvipnotifications.com,inbound,019,Americas,0 +daviacalendar.com,daviacalendar.com,inbound,019,Americas,1 +davidsbridal.com,davidsbridal.com,inbound,019,Americas,0 +davidstea.com,bronto.com,inbound,019,Americas,0 +daz3d.com,bronto.com,inbound,019,Americas,0 +dealersocket.com,dealersocket.com,inbound,019,Americas,0 +dealsaver.com,secondstreetmedia.com,inbound,019,Americas,0.999823 +debshops.com,lstrk.net,inbound,019,Americas,1 +deliasshopemail.com,deliasshopemail.com,inbound,019,Americas,0 +delivery.net,delivery.net,inbound,019,Americas,0 +delivery.net,m0.net,inbound,019,Americas,0 +dell.com,bfi0.com,inbound,019,Americas,0 +dentalsenders.com,dentalsenders.com,inbound,019,Americas,0 +descontos.pt,descontos.pt,inbound,019,Americas,0 +designerapparel.com,myperfectsale.com,inbound,019,Americas,1 +despegar.com,despegar.com,inbound,019,Americas,0 +dhgate.com,chtah.net,inbound,019,Americas,0 +dice.com,dice.com,inbound,019,Americas,0 +digitalmailer.com,digitalmailer.com,inbound,019,Americas,0 +digitalmedia-comunicacion.es,chtah.net,inbound,019,Americas,0 +digitalromanceinc.com,digitalromanceinc.com,inbound,019,Americas,1 +directv.com,directv.com,inbound,019,Americas,0.026649 +discover.com,discoverfinancial.com,inbound,019,Americas,1 +disparadordeemails.com,locaweb.com.br,inbound,019,Americas,0 +disqus.net,disqus.net,inbound,019,Americas,0 +dn.net,naukri.com,inbound,019,Americas,0 +docusign.net,docusign.net,inbound,019,Americas,0.985435 +donationnet.net,donationnet.net,inbound,019,Americas,0 +dorothyperkins.com,dorothyperkins.com,inbound,019,Americas,0 +dowjones.info,dowjones.info,inbound,019,Americas,0 +dptagent.biz,dptagent.biz,inbound,019,Americas,0 +dptagent.net,dptagent.net,inbound,019,Americas,0 +dreamhost.com,dreamhost.com,inbound,019,Americas,0 +dreamwidth.org,dreamwidth.org,inbound,019,Americas,0 +drhinternet.net,drhinternet.net,inbound,019,Americas,0 +driftem.com,emce2.in,inbound,019,Americas,0 +driftem.com,mailurja.com,inbound,019,Americas,0 +dropbox.com,dropbox.com,inbound,019,Americas,1 +dropboxmail.com,dropbox.com,inbound,019,Americas,1 +dsw.com,dsw.com,inbound,019,Americas,0 +ducks.org,uptilt.com,inbound,019,Americas,0 +duke.edu,duke.edu,inbound,019,Americas,0.308101 +dukecareers.com,dukecareers.com,inbound,019,Americas,1 +dvor.com,dvor.com,inbound,019,Americas,0 +dynamite-safelist.com,thomas-j-brown.com,inbound,019,Americas,0 +dynect-mailer.net,sendlabs.com,inbound,019,Americas,0 +e-activist.com,e-activist.com,inbound,019,Americas,0 +e-beallsonline.com,e-stagestores.com,inbound,019,Americas,0 +e-costco.mx,costco.com,inbound,019,Americas,0 +e-goodysonline.com,e-stagestores.com,inbound,019,Americas,0 +e-peebles.com,e-stagestores.com,inbound,019,Americas,0 +e-rewards.net,e-rewards.net,inbound,019,Americas,1 +e-stagestores.com,e-stagestores.com,inbound,019,Americas,0 +e-travelclub.es,e-travelclub.es,inbound,019,Americas,0 +e2ma.net,e2ma.net,inbound,019,Americas,1 +ea.com,ea.com,inbound,019,Americas,0.001232 +eaccess.net,postini.com,inbound,019,Americas,0 +earn-e-miles.com,earn-e-miles.com,inbound,019,Americas,0 +earnerslist.com,traxweb.net,inbound,019,Americas,9e-06 +earthfare-email.com,edclient2.com,inbound,019,Americas,0 +earthlink.net,earthlink.net,inbound,019,Americas,0.031648 +earthlink.net,earthlink.net,outbound,019,Americas,0 +eastbay.com,footlocker.com,inbound,019,Americas,0 +easyhealthoptions.com,easyhealthoptions.com,inbound,019,Americas,1 +easyroommate.com,easyroommate.com,inbound,019,Americas,0 +ebates.com,bfi0.com,inbound,019,Americas,0 +ebay.{...},ebay.{...},inbound,019,Americas,0.99941 +ebizac2.com,ebizac2.com,inbound,019,Americas,0 +eblastengine.com,secondstreetmedia.com,inbound,019,Americas,0.999827 +ec2.internal,amazonaws.com,inbound,019,Americas,0.769271 +ecasend.com,ecasend.com,inbound,019,Americas,0 +ed.gov,leepfrog.com,inbound,019,Americas,0.106381 +ed10.net,ed10.com,inbound,019,Americas,0 +edirect1.com,ivytech.edu,inbound,019,Americas,0 +edmodo.com,edmodo.com,inbound,019,Americas,1.1e-05 +educationzone.co.in,iaires.com,inbound,019,Americas,0 +effectivesafelist.com,zoothost.com,inbound,019,Americas,0 +eharmony.com,eharmony.com,inbound,019,Americas,1e-06 +eigbox.net,eigbox.net,inbound,019,Americas,0 +elabs3.com,elabs3.com,inbound,019,Americas,0 +elabs3.com,meritline.com,inbound,019,Americas,0 +elabs5.com,elabs5.com,inbound,019,Americas,0 +elabs6.com,elabs6.com,inbound,019,Americas,0 +elanceonline.com,elanceonline.com,inbound,019,Americas,0 +eleadtrack.net,eleadtrack.net,inbound,019,Americas,0 +elitesafelist.com,elitesafelist.com,inbound,019,Americas,0 +email-cooking.com,email-cooking.com,inbound,019,Americas,0 +email-galls.com,email-galls.com,inbound,019,Americas,1 +email-od.com,smtprelayserver.com,inbound,019,Americas,0.999779 +email-ticketdada.com,email-ticketdada.com,inbound,019,Americas,1 +email4-beyond.com,email4-beyond.com,inbound,019,Americas,0 +emailcounts.com,secureserver.net,inbound,019,Americas,0 +emaildir2.com,emaildirect.net,inbound,019,Americas,0 +emaildir2.com,espsnd.com,inbound,019,Americas,0 +emailnotify.net,emailnotify.net,inbound,019,Americas,0.961424 +emailsbancoestado.cl,emailsbancoestado.cl,inbound,019,Americas,0 +emailsripley.cl,etarget.cl,inbound,019,Americas,0 +embluejet.com,embluejet.com,inbound,019,Americas,0 +embluejet.com,emblueuser.com,inbound,019,Americas,0 +emcsend.com,emcsend.com,inbound,019,Americas,0 +emergencyemail.org,emergencyemail.org,inbound,019,Americas,0 +emktsender.net,locaweb.com.br,inbound,019,Americas,0 +emma.cl,emma.cl,inbound,019,Americas,0.996895 +emobile.ad.jp,postini.com,inbound,019,Americas,0 +employboard.com,employboard.com,inbound,019,Americas,1 +entregadeemails.com,locaweb.com.br,inbound,019,Americas,0 +entregadordecampanhas.net,locaweb.com.br,inbound,019,Americas,0 +enviodecampanhas.net,locaweb.com.br,inbound,019,Americas,0 +enviodemkt.com.br,locaweb.com.br,inbound,019,Americas,0 +epriority.com,epriority.com,inbound,019,Americas,0 +equifax.com,equifax.com,inbound,019,Americas,1 +equussafelist.com,equussafelist.com,inbound,019,Americas,0.000265 +esri.com,esri.com,inbound,019,Americas,0.983104 +esteelauder.com,esteelauder.com,inbound,019,Americas,0 +evanguard.com,evanguard.com,inbound,019,Americas,0 +eventbrite.com,eventbrite.com,inbound,019,Americas,0 +eversavelocal.com,eversavelocal.com,inbound,019,Americas,0 +everydayhealthinc.com,waterfrontmedia.net,inbound,019,Americas,0 +everyjobforme.com,everyjobforme.com,inbound,019,Americas,0 +exchangesolutions.com,exchangesolutions.com,inbound,019,Americas,0.000143 +exec-u-net-mail.com,exec-u-net-mail.com,inbound,019,Americas,0 +exprpt.com,exprpt.com,inbound,019,Americas,0 +expvtinboxhub.net,expvtinboxhub.net,inbound,019,Americas,0 +fabletics.com,bronto.com,inbound,019,Americas,0 +facebook.com,facebook.com,inbound,019,Americas,0.719445 +facebook.com,facebook.com,outbound,019,Americas,1 +facebookappmail.com,facebook.com,inbound,019,Americas,1 +facebookmail.com,facebook.com,inbound,019,Americas,1 +facebookmail.com,postini.com,inbound,019,Americas,0.592681 +facebookmail.com,yahoo.{...},inbound,019,Americas,1 +familychristianmail.com,familychristianmail.com,inbound,019,Americas,0 +famousfootwear.com,famousfootwear.com,inbound,019,Americas,0 +fanfiction.com,fictionpress.com,inbound,019,Americas,1 +farmersonly.com,mailgun.us,inbound,019,Americas,1 +fashion2hub.in,mgenie.in,inbound,019,Americas,0 +fastgb.com,fastgb.com,inbound,019,Americas,0 +fastlistmailer.com,zoothost.com,inbound,019,Americas,0 +fastweb.com,fastweb.com,inbound,019,Americas,0 +fbi.gov,fbi.gov,inbound,019,Americas,0 +fbmta.com,fbmta.com,inbound,019,Americas,0 +fc2.com,fc2.com,inbound,019,Americas,0.000601 +fedoraproject.org,fedoraproject.org,inbound,019,Americas,0 +feedblitz.com,feedblitz.com,inbound,019,Americas,0 +fetlifemail.com,fetlifemail.com,inbound,019,Americas,0 +fibertel.com.ar,fibertel.com.ar,inbound,019,Americas,0.003898 +fidelizador.org,fidelizador.org,inbound,019,Americas,0 +findexpvtinbox.com,findexpvtinbox.com,inbound,019,Americas,0 +finishline.com,finishline.com,inbound,019,Americas,0 +firemountaingems.com,firemountaingems.com,inbound,019,Americas,0 +fisher-price.com,fisher-price.com,inbound,019,Americas,0 +fitbit.com,fitbit.com,inbound,019,Americas,1 +fitnessmagazine.com,meredith.com,inbound,019,Americas,0 +fiverr.com,fiverr.com,inbound,019,Americas,0 +flexmls.com,flexmls.com,inbound,019,Americas,0.999992 +flightaware.com,flightaware.com,inbound,019,Americas,0.020698 +flipkart.com,flipkart.com,inbound,019,Americas,1 +flirt.com,ropot.net,inbound,019,Americas,0 +flirthookup.com,flirthookup.com,inbound,019,Americas,1 +flyceb.com,flyceb.com,inbound,019,Americas,0 +flyfrontier.com,flyfrontier.com,inbound,019,Americas,0 +foolsubs.com,foolcs.com,inbound,019,Americas,0 +foolsubs.com,foolsubs.com,inbound,019,Americas,0 +footaction.com,footlocker.com,inbound,019,Americas,0 +footlocker.com,footlocker.com,inbound,019,Americas,0.000605 +forever21.com,forever21.com,inbound,019,Americas,0 +fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,019,Americas,0 +foursquare.com,foursquare.com,inbound,019,Americas,1 +foxnews.com,foxnews.com,inbound,019,Americas,0.0139 +fragrancenet.com,fragrancenet.com,inbound,019,Americas,0.000427 +francescas.com,bronto.com,inbound,019,Americas,0 +freeadsmailer.com,zoothost.com,inbound,019,Americas,0 +freebeesafelist.com,zoothost.com,inbound,019,Americas,0 +freebizmag.com,delivery.net,inbound,019,Americas,0 +freebsd.org,freebsd.org,inbound,019,Americas,0.999845 +freecycle.org,freecycle.org,inbound,019,Americas,1 +freedesktop.org,freedesktop.org,inbound,019,Americas,0 +freeflys.com,freeflys.com,inbound,019,Americas,0 +freelancer.com,freelancer.com,inbound,019,Americas,0 +freelancer.com,freelancernotify.com,inbound,019,Americas,0 +freelancer.com,getafreelancer.com,inbound,019,Americas,0 +freelists.org,iquest.net,inbound,019,Americas,0 +freelotto.com,plasmanetinc.com,inbound,019,Americas,0 +freepeople.com,freepeople.com,inbound,019,Americas,0 +freesafelistking.com,zoothost.com,inbound,019,Americas,0 +freshdesk.com,freshdesk.com,inbound,019,Americas,1 +freshers2015.com,secureserver.net,inbound,019,Americas,0 +freshlatesave.com,freshlatesave.com,inbound,019,Americas,1 +friskone.com,mailurja.com,inbound,019,Americas,0 +frontsight.com,frontsight.com,inbound,019,Americas,0 +frys.com,frys.com,inbound,019,Americas,0.00388 +frysmail.com,frysmail.com,inbound,019,Americas,0 +fspeletters.com,agorapub.co.uk,inbound,019,Americas,0 +fuelrewards.com,britecast.com,inbound,019,Americas,0 +futureshop.com,futureshop.com,inbound,019,Americas,0 +gaiaonline.com,gaiaonline.com,inbound,019,Americas,0 +gamehouse.com,gamehouse.com,inbound,019,Americas,0 +gbyguess.com,guess.com,inbound,019,Americas,0.001787 +gemoney.com,rsys1.com,inbound,019,Americas,0 +generalmills.com,boxtops4education.com,inbound,019,Americas,0 +generalmills.com,pillsbury.com,inbound,019,Americas,0 +gentoo.org,gentoo.org,inbound,019,Americas,1 +get-me-jobs.com,get-me-jobs.com,inbound,019,Americas,0 +gethired.com,gethired.com,inbound,019,Americas,1 +getitfree.us,getitfree.us,inbound,019,Americas,0 +getpaidsolutions.com,getpaidsolutions.com,inbound,019,Americas,1 +getpocket.com,bronto.com,inbound,019,Americas,0 +ghin.com,ghinconnect.com,inbound,019,Americas,0 +ghup.in,mgenie.in,inbound,019,Americas,0 +gillyhicks-email.com,abercrombie-email.com,inbound,019,Americas,0 +github.com,github.com,inbound,019,Americas,1 +github.com,postini.com,inbound,019,Americas,0.837872 +glassdoor.com,glassdoor.com,inbound,019,Americas,1 +glasses.com,glasses.com,inbound,019,Americas,0 +gliq.com,gliq.com,inbound,019,Americas,0.99852 +globalsafelist.com,globalsafelist.com,inbound,019,Americas,0 +globaltestmarket.com,globaltestmarket.com,inbound,019,Americas,0 +gmail.com,amazonaws.com,inbound,019,Americas,0.994381 +gmail.com,anteldata.net.uy,inbound,019,Americas,0.998653 +gmail.com,bell.ca,inbound,019,Americas,0.992481 +gmail.com,bellsouth.net,inbound,019,Americas,0.9996 +gmail.com,blackberry.com,inbound,019,Americas,0.992207 +gmail.com,brasiltelecom.net.br,inbound,019,Americas,0.99995 +gmail.com,centurytel.net,inbound,019,Americas,0.999415 +gmail.com,cgocable.net,inbound,019,Americas,0.99829 +gmail.com,charter.com,inbound,019,Americas,0.999109 +gmail.com,claro.net.br,inbound,019,Americas,1 +gmail.com,comcast.net,inbound,019,Americas,0.999621 +gmail.com,comcastbusiness.net,inbound,019,Americas,0.985462 +gmail.com,cox.net,inbound,019,Americas,0.962619 +gmail.com,embarqhsd.net,inbound,019,Americas,0.999554 +gmail.com,franchiseindia.com,inbound,019,Americas,1 +gmail.com,frontiernet.net,inbound,019,Americas,0.995233 +gmail.com,gvt.net.br,inbound,019,Americas,0.999513 +gmail.com,lorexddns.net,inbound,019,Americas,0 +gmail.com,majesticmoneymailer.com,inbound,019,Americas,1 +gmail.com,mchsi.com,inbound,019,Americas,0.999951 +gmail.com,movistar.cl,inbound,019,Americas,0.999361 +gmail.com,mycingular.net,inbound,019,Americas,0.999918 +gmail.com,myvzw.com,inbound,019,Americas,0.99992 +gmail.com,naukri.com,inbound,019,Americas,0.000998 +gmail.com,optonline.net,inbound,019,Americas,0.999776 +gmail.com,postini.com,inbound,019,Americas,0.650888 +gmail.com,qwest.net,inbound,019,Americas,0.997574 +gmail.com,rcn.com,inbound,019,Americas,0.999275 +gmail.com,rogers.com,inbound,019,Americas,0.999916 +gmail.com,rr.com,inbound,019,Americas,0.986894 +gmail.com,sbcglobal.net,inbound,019,Americas,0.998817 +gmail.com,shawcable.net,inbound,019,Americas,0.999998 +gmail.com,spcsdns.net,inbound,019,Americas,0.999998 +gmail.com,suddenlink.net,inbound,019,Americas,0.961593 +gmail.com,telecom.net.ar,inbound,019,Americas,0.999664 +gmail.com,telesp.net.br,inbound,019,Americas,0.999743 +gmail.com,telus.com,inbound,019,Americas,1 +gmail.com,telus.net,inbound,019,Americas,0.974663 +gmail.com,tmodns.net,inbound,019,Americas,1 +gmail.com,veloxzone.com.br,inbound,019,Americas,0.999969 +gmail.com,verizon.net,inbound,019,Americas,0.990214 +gmail.com,videotron.ca,inbound,019,Americas,0.967196 +gmail.com,vtr.net,inbound,019,Americas,0.999072 +gmail.com,websitewelcome.com,inbound,019,Americas,1 +gmail.com,wideopenwest.com,inbound,019,Americas,0.999729 +gmail.com,windstream.net,inbound,019,Americas,0.951847 +gmail.com,yahoo.{...},inbound,019,Americas,0.999992 +gmail.com,zoothost.com,inbound,019,Americas,0.033858 +gob.ar,gob.ar,inbound,019,Americas,0.160006 +gob.ec,gob.ec,inbound,019,Americas,0.623867 +godaddy.com,secureserver.net,inbound,019,Americas,0 +gogecapital.com,rsys1.com,inbound,019,Americas,0 +goldenopsafelist.com,zoothost.com,inbound,019,Americas,0 +goldstar.com,goldstar.com,inbound,019,Americas,1 +golfmnb.com,golfmnb.com,inbound,019,Americas,0 +google.com,postini.com,inbound,019,Americas,0.584887 +gopusamedia.com,gopusamedia.com,inbound,019,Americas,0 +govdelivery.com,govdelivery.com,inbound,019,Americas,0 +governmentjobs.com,governmentjobs.com,inbound,019,Americas,0 +gpmailer.com.br,parperfeito.com,inbound,019,Americas,0 +grassrootsaction.com,grassfire.net,inbound,019,Americas,0 +greatergood.com,greatergood.com,inbound,019,Americas,0 +groupon.{...},chtah.net,inbound,019,Americas,0 +groupon.{...},groupon.{...},inbound,019,Americas,1.0 +groupon.{...},postini.com,inbound,019,Americas,0.886672 +grouponmail.{...},grouponmail.{...},inbound,019,Americas,0 +grupos.com.br,grupos.com.br,inbound,019,Americas,0 +guess.ca,guess.com,inbound,019,Americas,0.000807 +guess.com,guess.com,inbound,019,Americas,0.003805 +guessfactory.com,guess.com,inbound,019,Americas,0.00164 +gustazos.com,cityoferta.com,inbound,019,Americas,1 +hallmark.com,hallmark.com,inbound,019,Americas,0 +hannaandersson.com,hannaandersson.com,inbound,019,Americas,0.013533 +harristeetermail.com,harristeetermail.com,inbound,019,Americas,0.99673 +harvard.edu,harvard.edu,inbound,019,Americas,0.274906 +hayneedle.com,hayneedle.com,inbound,019,Americas,0 +helpareporter.net,helpareporter.com,inbound,019,Americas,0 +herculist.com,herculist.com,inbound,019,Americas,0 +hilton.com,hiltonemail.com,inbound,019,Americas,0 +hipchat.com,hipchat.com,inbound,019,Americas,1 +hispavista.com,hispavista.com,inbound,019,Americas,0 +hollister-email.com,abercrombie-email.com,inbound,019,Americas,0 +homeaway.com,haspf.com,inbound,019,Americas,0 +homedecorators.com,homedecorators.com,inbound,019,Americas,0 +homedepot.com,homedepot.com,inbound,019,Americas,1 +hootsuite.com,hootsuite.com,inbound,019,Americas,1 +horchowemail.com,horchowemail.com,inbound,019,Americas,0 +hostelworld.com,bronto.com,inbound,019,Americas,0 +hostgator.com,hostgator.com,inbound,019,Americas,0.98061 +hostgator.com,websitewelcome.com,inbound,019,Americas,1 +hotmail.{...},hotmail.{...},inbound,019,Americas,1 +hotmail.{...},hotmail.{...},outbound,019,Americas,1 +hotmail.{...},postini.com,inbound,019,Americas,0.723143 +hotornot.com,monopost.com,inbound,019,Americas,1 +hotschedules.com,hotschedules.com,inbound,019,Americas,0 +house.gov,house.gov,inbound,019,Americas,0.999966 +houseoffraser.co.uk,houseoffraser.co.uk,inbound,019,Americas,0 +houzz.com,houzz.com,inbound,019,Americas,1 +hubspot.com,hubspot.com,inbound,019,Americas,1 +hungry-girl.com,hungry-girl.com,inbound,019,Americas,0 +iamlgnd2.com,iamlgnd2.com,inbound,019,Americas,1 +ibsys.com,ibsys.com,inbound,019,Americas,9e-05 +icbc.com.ar,clickexperts.net,inbound,019,Americas,0 +icbc.com.ar,standardbank.com.ar,inbound,019,Americas,0 +icims.com,icims.com,inbound,019,Americas,0.999939 +icors.org,lsoft.us,inbound,019,Americas,0 +icpbounce.com,icpbounce.com,inbound,019,Americas,0 +idc.email,nmsrv.com,inbound,019,Americas,1 +idgconnect-resources.com,idgconnect-resources.com,inbound,019,Americas,0 +ieee.org,ieee.org,inbound,019,Americas,0.999912 +ig.com.br,ig.com.br,inbound,019,Americas,0 +ig.com.br,ig.com.br,outbound,019,Americas,0 +igot-mails.com,zoothost.com,inbound,019,Americas,0 +iimjobs.com,iimjobs.com,inbound,019,Americas,1 +ikmultimedianews.com,ikmultimedianews.com,inbound,019,Americas,0.993319 +illinois.edu,illinois.edu,inbound,019,Americas,0.866481 +imageshost.ca,imageshost.ca,inbound,019,Americas,0 +imakenews.net,imakenews.com,inbound,019,Americas,0 +imo.im,imo.im,inbound,019,Americas,1 +imodules.com,imodules.com,inbound,019,Americas,0 +imvu.com,imvu.com,inbound,019,Americas,7e-06 +inboxdollars.com,inboxdollars.com,inbound,019,Americas,0 +inboxfirst.com,inboxfirst.com,inbound,019,Americas,0 +inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,019,Americas,0.999877 +inboxpays.com,inboxpays.com,inbound,019,Americas,0 +inboxpounds.co.uk,inboxpounds.co.uk,inbound,019,Americas,0 +indeed.com,indeed.com,inbound,019,Americas,0.000121 +indeedemail.com,indeedemail.com,inbound,019,Americas,0 +independentlivingbullion.com,independentlivingbullion.com,inbound,019,Americas,0 +infobradesco.com.br,infobradesco.com.br,inbound,019,Americas,0 +infojobs.com.br,anuntis.com,inbound,019,Americas,0 +infomoney.com.br,infomoney.com.br,inbound,019,Americas,1 +informz.net,informz.net,inbound,019,Americas,0 +infradead.org,infradead.org,inbound,019,Americas,1 +inman.com,inman.com,inbound,019,Americas,0 +innovyx.net,innovyx.net,inbound,019,Americas,0 +insidehook.com,sailthru.com,inbound,019,Americas,0 +instagram.com,facebook.com,inbound,019,Americas,1 +instantprofitlist.com,screenshotads.com,inbound,019,Americas,0 +interac.ca,certapay.com,inbound,019,Americas,0 +interactivebrokers.com,interactivebrokers.com,inbound,019,Americas,1 +interactiverealtyservices.com,interactiverealtyservices.com,inbound,019,Americas,0 +intercom.io,mailgun.info,inbound,019,Americas,1 +interealty.net,interealty.net,inbound,019,Americas,1 +interweave.com,interweave.com,inbound,019,Americas,0 +intliv2.net,internationalliving.com,inbound,019,Americas,0 +invalidemail.com,taleo.net,inbound,019,Americas,1 +investopedia.com,vclk.net,inbound,019,Americas,0.999999 +investorplace.com,investorplace.com,inbound,019,Americas,0.995093 +irctcshopping.com,chtah.net,inbound,019,Americas,0 +iridium.com,iridium.com,inbound,019,Americas,0.997882 +isendservice.com.br,isendservice.com.br,inbound,019,Americas,0 +itau-unibanco.com.br,itau.com.br,inbound,019,Americas,0 +ittoolbox.com,ittoolbox.com,inbound,019,Americas,0 +ittoolbox.com,toolbox.com,inbound,019,Americas,0 +itwhitepapers.com,itwhitepapers.com,inbound,019,Americas,0 +iwantoneofthose.com,thehut.com,inbound,019,Americas,0 +ixs1.net,ixs1.net,inbound,019,Americas,0.005131 +jcpenney.com,jcpenney.com,inbound,019,Americas,9e-06 +jeevansathi.com,jeevansathi.com,inbound,019,Americas,0 +jetsetter.com,smartertravelmedia.com,inbound,019,Americas,0.041888 +jibjab.com,storybots.com,inbound,019,Americas,0 +jira.com,uc-inf.net,inbound,019,Americas,1 +jobscentral.com.sg,mailgun.net,inbound,019,Americas,1 +jobson.com,jobsonmail.com,inbound,019,Americas,0 +jobsradar.com,jobsradar.com,inbound,019,Americas,0 +jockeycomfort.com,jockeycomfort.com,inbound,019,Americas,0 +johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,019,Americas,0 +jomashop.com,lstrk.net,inbound,019,Americas,1 +josbank.com,josbank.com,inbound,019,Americas,0 +jossandmain.com,jossandmain.com,inbound,019,Americas,0 +jtv.com,jtv.com,inbound,019,Americas,0 +juno.com,untd.com,inbound,019,Americas,0 +juno.com,untd.com,outbound,019,Americas,0 +justdial.com,mailurja.com,inbound,019,Americas,0 +justfab.com,bronto.com,inbound,019,Americas,0 +justfab.fr,bronto.com,inbound,019,Americas,0 +keek.com,keek.com,inbound,019,Americas,1 +kernel.org,kernel.org,inbound,019,Americas,0 +kgstores.com,kgstores.com,inbound,019,Americas,0 +kickstarter.com,kickstarter.com,inbound,019,Americas,1 +kidsfootlocker.com,footlocker.com,inbound,019,Americas,0 +kik.com,kik.com,inbound,019,Americas,1 +kimblegroup.com,kimblegroup.com,inbound,019,Americas,1 +kintera.com,kintera.com,inbound,019,Americas,0 +klaviyomail.com,klaviyomail.com,inbound,019,Americas,1 +klove.com,emfbroadcasting.com,inbound,019,Americas,0 +komando.com,komando.com,inbound,019,Americas,0 +kp.org,kp.org,inbound,019,Americas,0.999975 +krogermail.com,bigfootinteractive.com,inbound,019,Americas,0 +landmarketingmailer.com,zoothost.com,inbound,019,Americas,0 +landofnod.com,landofnod.com,inbound,019,Americas,0.002525 +languagepod101.com,eclient10.com,inbound,019,Americas,0 +languagepod101.com,eddlvr.com,inbound,019,Americas,0 +languagepod101.com,ednwsltr3.com,inbound,019,Americas,0 +languagepod101.com,ednwsltr8.com,inbound,019,Americas,0 +languagepod101.com,emaildirect.net,inbound,019,Americas,0 +lasenza.com,lasenza.com,inbound,019,Americas,0 +lastcallemail.com,lastcallemail.com,inbound,019,Americas,0 +latimes.com,latimes.com,inbound,019,Americas,0 +lauraashley.com,lauraashley.com,inbound,019,Americas,0 +leftlanesports.com,auspient.com,inbound,019,Americas,0.00705 +leftlanesports.com,leftlanesports.com,inbound,019,Americas,0 +legalshieldassociate.com,legalshield.com,inbound,019,Americas,0 +lexico.com,lexico.com,inbound,019,Americas,0 +life360.com,life360.com,inbound,019,Americas,1 +lindenlab.com,lindenlab.com,inbound,019,Americas,0.999444 +linkedin.com,linkedin.com,inbound,019,Americas,0.999873 +linkedin.com,postini.com,inbound,019,Americas,0.713391 +listeneremail.net,listeneremail.net,inbound,019,Americas,0 +listia.com,listia.com,inbound,019,Americas,1 +listnerds.com,listnerds.com,inbound,019,Americas,0 +listreturn.com,zoothost.com,inbound,019,Americas,0 +listserve.com,listserve.com,inbound,019,Americas,0 +listvolta.com,listvolta.com,inbound,019,Americas,0 +listwire.com,listwire.com,inbound,019,Americas,0 +live.{...},hotmail.{...},inbound,019,Americas,1 +live.{...},hotmail.{...},outbound,019,Americas,1 +livefyre.com,andbit.net,inbound,019,Americas,1 +livescribe.com,bronto.com,inbound,019,Americas,0 +livingsocial.com,livingsocial.com,inbound,019,Americas,0 +livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,019,Americas,0 +localhires.com,localhires.com,inbound,019,Americas,1 +logitech.com,dvsops.com,inbound,019,Americas,0 +lojasmarisa.com.br,lojasmarisa.com.br,inbound,019,Americas,0 +lolsolos.com,ultimateadsites.net,inbound,019,Americas,0.999996 +lonelywifehookup.com,iverificationsystems.com,inbound,019,Americas,0 +lordandtaylor.com,lordandtaylor.com,inbound,019,Americas,0 +loveaholics.com,ropot.net,inbound,019,Americas,0 +lovelywholesale.com,lovelywholesale.com,inbound,019,Americas,1 +lrsmail.com,lrsmail.com,inbound,019,Americas,0 +lt02.net,listrak.com,inbound,019,Americas,1 +lt02.net,lstrk.net,inbound,019,Americas,1 +ltdcommodities.com,ltdcomm.net,inbound,019,Americas,0 +luckymag.com,mkt4500.com,inbound,019,Americas,0 +lulu.com,bronto.com,inbound,019,Americas,0 +lulus.com,lstrk.net,inbound,019,Americas,1 +lumosity.com,lumosity.com,inbound,019,Americas,1 +lyst.com,lyst.com,inbound,019,Americas,1 +maccosmetics.com,esteelauder.com,inbound,019,Americas,0 +macupdate.com,mailgun.info,inbound,019,Americas,1 +madmels.info,ultimateadsites.net,inbound,019,Americas,1 +madmimi.com,madmimi.com,inbound,019,Americas,0 +magicjack.com,magicjack.com,inbound,019,Americas,1 +magnetdev.com,magnetmail.net,inbound,019,Americas,0 +mail-route.com,mail-route.com,inbound,019,Americas,0 +mail-thestreet.com,mail-thestreet.com,inbound,019,Americas,0 +mail.mil,mail.mil,inbound,019,Americas,0 +mailaccurate.com,mgenie.in,inbound,019,Americas,0 +mailfacil.com.br,md02.com,inbound,019,Americas,0 +mailfeast.com,mgenie.in,inbound,019,Americas,0 +mailgun.org,mailgun.info,inbound,019,Americas,1 +mailgun.org,mailgun.net,inbound,019,Americas,1 +mailgun.org,mailgun.us,inbound,019,Americas,1 +mailingathome.net,mailingathome.net,inbound,019,Americas,0.999995 +mailjayde.com,mailjayde.com,inbound,019,Americas,0 +mailmachine1050.com,mailmachine1050.com,inbound,019,Americas,0 +mailsend1.com,mailsend6.com,inbound,019,Americas,0 +mailsender.com.br,mailsender.com.br,inbound,019,Americas,0 +manager.com.br,manager.com.br,inbound,019,Americas,0 +mandrillapp.com,mandrillapp.com,inbound,019,Americas,1 +mandrillapp.com,myjobhelperalerts.com,inbound,019,Americas,1 +marcustheatres.com,movio.co,inbound,019,Americas,0 +markandgraham.com,markandgraham.com,inbound,019,Americas,0 +marketer-safelist.com,jsalfianmarketing.com,inbound,019,Americas,1 +marketinghq.net,elabs8.com,inbound,019,Americas,0 +marketingprofs.com,marketingprofs.com,inbound,019,Americas,0.004412 +marlboro.com,marlboro.com,inbound,019,Americas,0 +maropost.com,biotrustnews.com,inbound,019,Americas,0 +maropost.com,mailing-truthaboutabs.com,inbound,019,Americas,0 +maropost.com,maropost.com,inbound,019,Americas,0 +maropost.com,mp2201.com,inbound,019,Americas,0 +masivapp.com,masivapp.com,inbound,019,Americas,1 +massageenvyclinics.com,massageenvyclinics.com,inbound,019,Americas,0 +masterbase.com,masterbase.com,inbound,019,Americas,0 +mastercard-email.com,mastercard-email.com,inbound,019,Americas,0 +mate1.net,mate1.net,inbound,019,Americas,0 +matrixemailer.com,matrixemailer.com,inbound,019,Americas,0 +mbstrm.com,mobilestorm.com,inbound,019,Americas,0 +mcafee.com,mcafee.com,inbound,019,Americas,0.979126 +mcarthurglen.com,mcarthurglen.com,inbound,019,Americas,0 +mcdlv.net,mcdlv.net,inbound,019,Americas,0 +mckinsey.com,bigfootinteractive.com,inbound,019,Americas,0 +mcsv.net,mcsv.net,inbound,019,Americas,0 +mdlinx.com,mdlinx.com,inbound,019,Americas,0 +mec.gov.br,mec.gov.br,inbound,019,Americas,0 +mediabistro.com,iworld.com,inbound,019,Americas,0.006428 +medpagetoday.com,wc09.net,inbound,019,Americas,0 +meetmemail.com,meetmemail.com,inbound,019,Americas,0 +megasenders.com,megasenders.com,inbound,019,Americas,0.07662 +memberdealsusa.com,memberdealsusa.com,inbound,019,Americas,0 +menswearhouse.com,menswearhouse.com,inbound,019,Americas,0 +mercadojobs.com,sendgrid.net,inbound,019,Americas,1 +mercola.com,mercola.com,inbound,019,Americas,0.000369 +messagegears.net,messagegears.net,inbound,019,Americas,0 +met-art.com,hydentra.com,inbound,019,Americas,1 +mgo.com,bronto.com,inbound,019,Americas,0 +michaels.com,chtah.net,inbound,019,Americas,0 +michaels.com,michaels.com,inbound,019,Americas,0 +microcentermedia.com,bfi0.com,inbound,019,Americas,0 +microsoft.com,hotmail.{...},inbound,019,Americas,1 +microsoft.com,msn.com,inbound,019,Americas,1 +midnightsunsafelist.com,zoothost.com,inbound,019,Americas,0 +milfaholic.com,iverificationsystems.com,inbound,019,Americas,0 +miltnews.com,miltnews.com,inbound,019,Americas,0 +mindbodyonline.com,mindbodyonline.com,inbound,019,Americas,1 +mindfieldonline.com,mindfieldonline.com,inbound,019,Americas,0 +mindmoviesmail.com,mindmoviesmail.com,inbound,019,Americas,0.004239 +mindvalleymail3.com,mindvalleymail3.com,inbound,019,Americas,0 +mint.com,mint.com,inbound,019,Americas,0 +minted.com,messagelabs.com,inbound,019,Americas,0.999669 +missselfridge.com,wallis-fashion.com,inbound,019,Americas,0 +mistersafelist.com,zoothost.com,inbound,019,Americas,0 +mit.edu,mit.edu,inbound,019,Americas,0.869568 +mjinn.com,mailurja.com,inbound,019,Americas,0 +mkt015.com,mkt015.com,inbound,019,Americas,0 +mkt022.com,mkt022.com,inbound,019,Americas,0 +mkt063.com,mkt063.com,inbound,019,Americas,0 +mkt1136.com,mkt1136.com,inbound,019,Americas,0 +mkt1985.com,fmlinks.net,inbound,019,Americas,0 +mkt2010.com,mkt2010.com,inbound,019,Americas,0 +mkt2106.com,mkt2106.com,inbound,019,Americas,0 +mkt2170.com,mkt2170.com,inbound,019,Americas,0 +mkt2181.com,mkt2181.com,inbound,019,Americas,0 +mkt2615.com,mkt2615.com,inbound,019,Americas,0 +mkt2813.com,mkt2813.com,inbound,019,Americas,0 +mkt2944.com,mkt2944.com,inbound,019,Americas,0 +mkt3134.com,mkt3134.com,inbound,019,Americas,0 +mkt3142.com,mkt3142.com,inbound,019,Americas,0 +mkt3156.com,mkt3156.com,inbound,019,Americas,0 +mkt3203.com,mkt3203.com,inbound,019,Americas,0 +mkt346.com,mkt346.com,inbound,019,Americas,0 +mkt3544.com,mkt3544.com,inbound,019,Americas,0 +mkt3622.com,mkt3622.com,inbound,019,Americas,0 +mkt3682.com,mkt3682.com,inbound,019,Americas,0 +mkt3690.com,mkt3690.com,inbound,019,Americas,0 +mkt3695.com,mkt3695.com,inbound,019,Americas,0 +mkt3804.com,mkt3804.com,inbound,019,Americas,0 +mkt3815.com,mkt3815.com,inbound,019,Americas,0 +mkt3952.com,xoom.com,inbound,019,Americas,0 +mkt4355.com,mkt4355.com,inbound,019,Americas,0 +mkt4364.com,mkt4364.com,inbound,019,Americas,0 +mkt459.com,mkt459.com,inbound,019,Americas,0 +mkt4701.com,mkt4701.com,inbound,019,Americas,0 +mkt4728.com,mkt4728.com,inbound,019,Americas,0 +mkt4731.com,mkt4731.com,inbound,019,Americas,0 +mkt4738.com,mkt4738.com,inbound,019,Americas,0 +mkt5071.com,mkt5071.com,inbound,019,Americas,0 +mkt5098.com,mkt5098.com,inbound,019,Americas,0 +mkt5131.com,mkt5131.com,inbound,019,Americas,0 +mkt5144.com,mkt5144.com,inbound,019,Americas,0 +mkt5144.com,mkt5980.com,inbound,019,Americas,0 +mkt5144.com,mkt5981.com,inbound,019,Americas,0 +mkt5181.com,mkt5181.com,inbound,019,Americas,0 +mkt5269.com,mkt5269.com,inbound,019,Americas,0 +mkt529.com,mkt529.com,inbound,019,Americas,0 +mkt5297.com,mkt5297.com,inbound,019,Americas,0 +mkt5297.com,mkt5309.com,inbound,019,Americas,0 +mkt5371.com,mkt5371.com,inbound,019,Americas,0 +mkt5806.com,mkt5806.com,inbound,019,Americas,0 +mkt5934.com,mkt5934.com,inbound,019,Americas,0 +mkt5937.com,mkt5937.com,inbound,019,Americas,0 +mkt5970.com,mkt5970.com,inbound,019,Americas,0 +mkt6100.com,mkt6098.com,inbound,019,Americas,0 +mkt6276.com,mkt6276.com,inbound,019,Americas,0 +mkt6323.com,mkt6323.com,inbound,019,Americas,0 +mkt746.com,mkt746.com,inbound,019,Americas,0 +mkt824.com,mkt869.com,inbound,019,Americas,0 +mktdillards.com,mktdillards.com,inbound,019,Americas,0 +mo1send.com,mo1send.com,inbound,019,Americas,0 +mobly.com.br,mobly.com.br,inbound,019,Americas,0 +modellsemail.com,n-email.net,inbound,019,Americas,0 +moneymorning.com,moneymappress.com,inbound,019,Americas,0 +monster.com,monster.com,inbound,019,Americas,0 +monster.com,tmpw.net,inbound,019,Americas,0.000503 +moon-ray.com,moon-ray.com,inbound,019,Americas,0 +mooresclothing.com,mooresclothing.com,inbound,019,Americas,0 +moveon.org,moveon.org,inbound,019,Americas,0.996147 +mrmlsmatrix.com,mrmlsmatrix.com,inbound,019,Americas,0 +ms.com,ms.com,inbound,019,Americas,1 +msn.com,hotmail.{...},inbound,019,Americas,1 +msn.com,hotmail.{...},outbound,019,Americas,1 +mta.info,ealert.com,inbound,019,Americas,0 +mtasv.net,mtasv.net,inbound,019,Americas,0.999999 +musicnotes-alerts.com,mybuys.com,inbound,019,Americas,0 +mustanglist.com,mustanglist.com,inbound,019,Americas,0 +mycheapoair.com,mycheapoair.com,inbound,019,Americas,0.957931 +mydailymoment.biz,mydailymoment.biz,inbound,019,Americas,0 +mydailymoment.info,mydailymoment.info,inbound,019,Americas,0 +mydailymoment.net,mydailymoment.net,inbound,019,Americas,0 +mydailymoment.us,mydailymoment.us,inbound,019,Americas,0 +myfedloan.org,aessuccess.org,inbound,019,Americas,0.328917 +myfxbook.com,myfxbook.com,inbound,019,Americas,0 +mygreatlakes.org,glhec.org,inbound,019,Americas,0.000247 +myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,019,Americas,0 +mymeijer.com,mymeijer.com,inbound,019,Americas,0 +myngp.com,ngpweb.com,inbound,019,Americas,0 +myperfectsale.com,myperfectsale.com,inbound,019,Americas,1 +myprotein.com,thehut.com,inbound,019,Americas,0 +mysafelistmailer.com,mysafelistmailer.com,inbound,019,Americas,0.00033 +mysmartprice.com,itzwow.com,inbound,019,Americas,1 +myzamanamail.com,myzamanamail.com,inbound,019,Americas,0 +n-email.net,n-email.net,inbound,019,Americas,0 +n-email1.net,n-email1.net,inbound,019,Americas,0 +n-email4.net,n-email4.net,inbound,019,Americas,0 +nanomail.com.br,araie.com.br,inbound,019,Americas,1 +napitipp.hu,napitipp.hu,inbound,019,Americas,0 +nasa.gov,nasa.gov,inbound,019,Americas,0.101289 +nastygal.com,bronto.com,inbound,019,Americas,0 +nationbuilder.com,nationbuilder.com,inbound,019,Americas,1 +nature.com,nature.com,inbound,019,Americas,0 +naukri.com,naukri.com,inbound,019,Americas,0.000533 +nauta.cu,etecsa.net,inbound,019,Americas,0 +nba.com,nba.com,inbound,019,Americas,0.005829 +nbaa.org,nbaa.org,inbound,019,Americas,0 +ncl.com,ncl.com,inbound,019,Americas,0 +neimanmarcusemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +net-a-porter.com,net-a-porter.com,inbound,019,Americas,0 +net-empregos.com,net-empregos.com,inbound,019,Americas,0 +netcommunity1.com,blackbaud.com,inbound,019,Americas,0 +netflix.com,netflix.com,inbound,019,Americas,1 +netopia.pt,netopia.pt,inbound,019,Americas,0 +netprosoftmail.com,netprosoftmail.com,inbound,019,Americas,0 +netsuite.com,netsuite.com,inbound,019,Americas,0.60682 +networkworld.com,networkworld.com,inbound,019,Americas,0 +newgrounds.com,newgrounds.com,inbound,019,Americas,0 +newrelic.com,sendlabs.com,inbound,019,Americas,0 +newspaperdirect.com,newspaperdirect.com,inbound,019,Americas,0.000177 +newyorktimesinfo.com,newyorktimesinfo.com,inbound,019,Americas,0 +nexcess.net,nexcess.net,inbound,019,Americas,0.039513 +nextdoor.com,mailgun.info,inbound,019,Americas,1 +nfl.com,bfi0.com,inbound,019,Americas,0 +nih.gov,nih.gov,inbound,019,Americas,8.8e-05 +ninewestmail.com,ninewestmail.com,inbound,019,Americas,0 +ning.com,ning.com,inbound,019,Americas,0 +nixle.com,nixle.com,inbound,019,Americas,0 +nl00.net,netline.com,inbound,019,Americas,5e-06 +nl00.net,nl00.net,inbound,019,Americas,0 +nongnu.org,gnu.org,inbound,019,Americas,1 +nordstrom.com,taleo.net,inbound,019,Americas,1 +nortonfromsymantec.com,rsys1.com,inbound,019,Americas,0 +novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,019,Americas,0 +numbersusa.com,numbersusa.com,inbound,019,Americas,0 +nyandcompany.com,nyandcompany.com,inbound,019,Americas,0 +ocadomail.com,ocadomail.com,inbound,019,Americas,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,019,Americas,0 +ofertasefacil.com.br,ofertasefacil.com.br,inbound,019,Americas,0 +ofertop.pe,icommarketing.com,inbound,019,Americas,0 +officedepot.com,officedepot.com,inbound,019,Americas,2.6e-05 +olympiaedge.net,olympiaedge.net,inbound,019,Americas,0 +oneindia.in,infimail.com,inbound,019,Americas,0 +oneindia.in,mailurja.com,inbound,019,Americas,0 +onepatriotplace.com,britecast.com,inbound,019,Americas,0 +onestopplus.com,neolane.net,inbound,019,Americas,0 +onetravelspecials.com,onetravelspecials.com,inbound,019,Americas,0.365386 +online.com,cnet.com,inbound,019,Americas,0 +onthecitymail.org,onthecitymail.org,inbound,019,Americas,1 +oo155.com,bsftransmit7.com,inbound,019,Americas,0 +oo155.com,oo155.com,inbound,019,Americas,0 +openstack.org,openstack.org,inbound,019,Americas,0.997933 +openstackmail.com,infimail.com,inbound,019,Americas,0 +opentable.com,opentable.com,inbound,019,Americas,2e-06 +opinionoutpost.com,opinionoutpost.com,inbound,019,Americas,0 +opinionsquare.com,opinionsquare.com,inbound,019,Americas,0 +oprah.com,oprah.com,inbound,019,Americas,0 +opticsplanet.com,opticsplanet.com,inbound,019,Americas,0 +optonline.net,cv.net,inbound,019,Americas,0 +optonline.net,optonline.net,outbound,019,Americas,0 +oriental-trading.com,oriental-trading.com,inbound,019,Americas,0 +outlook.com,hotmail.{...},inbound,019,Americas,1 +outlook.com,hotmail.{...},outbound,019,Americas,1 +overnightprints.com,chtah.net,inbound,019,Americas,0 +ovuline.com,ovuline.com,inbound,019,Americas,1 +pagoda.com,zales.com,inbound,019,Americas,0 +pagseguro.com.br,uol.com.br,inbound,019,Americas,0 +pair.com,pair.com,inbound,019,Americas,0.893803 +palmscasinoresort.com,palmscasinoresort.com,inbound,019,Americas,0 +pampers.com,bfi0.com,inbound,019,Americas,0 +pandaresearch.com,pandaresearch.com,inbound,019,Americas,0 +pandora.com,pandora.com,inbound,019,Americas,1 +pandora.net,pandora.net,inbound,019,Americas,1 +parents.com,meredith.com,inbound,019,Americas,0 +parkmobileglobal.com,parkmobile.us,inbound,019,Americas,0 +path.com,path.com,inbound,019,Americas,1 +patriotupdate.com,inboxfirst.com,inbound,019,Americas,0 +payback.in,chtah.net,inbound,019,Americas,0 +paytm.com,paytm.com,inbound,019,Americas,1 +pbteen.com,pbteen.com,inbound,019,Americas,0 +pch.com,ed10.com,inbound,019,Americas,0 +pchfrontpage.com,ed10.com,inbound,019,Americas,0 +pchlotto.com,ed10.com,inbound,019,Americas,0 +pchplayandwin.com,ed10.com,inbound,019,Americas,0 +pchsearch.com,ed10.com,inbound,019,Americas,0 +pcmag.com,ittoolbox.com,inbound,019,Americas,0 +pcworld.com,pcworld.com,inbound,019,Americas,0 +pd25.com,pd25.com,inbound,019,Americas,1 +pearlsofwealth.com,pearlsofwealth.com,inbound,019,Americas,1 +peartreegreetings.com,rexcraft.com,inbound,019,Americas,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,019,Americas,0 +pepboys.com,pepboys.com,inbound,019,Americas,0 +pepperfry.com,epidm.net,inbound,019,Americas,0 +perfectpriceindia.com,infimail.com,inbound,019,Americas,0 +perfora.net,perfora.net,inbound,019,Americas,1 +permissionresearch.com,permissionresearch.com,inbound,019,Americas,0 +personalliberty.com,personalliberty.com,inbound,019,Americas,1 +pga.com,pga.com,inbound,019,Americas,0 +pgeveryday.com,bfi0.com,inbound,019,Americas,0 +phoenix.edu,phoenix.edu,inbound,019,Americas,0 +phsmtpbox.com,phsmtpbox.com,inbound,019,Americas,0 +pinterest.com,pinterest.com,inbound,019,Americas,1 +pivotaltracker.com,pivotaltracker.com,inbound,019,Americas,1 +pixable.com,pixable.com,inbound,019,Americas,1 +pizzahut.com,quikorder.com,inbound,019,Americas,0 +plexapp.com,plex.tv,inbound,019,Americas,1 +pmailus.com,patrontechnology.com,inbound,019,Americas,0 +pnc.com,messagelabs.com,inbound,019,Americas,0.997194 +pobox.com,pobox.com,inbound,019,Americas,0.975372 +pof.com,plentyoffish.co.uk,inbound,019,Americas,0 +pogo.com,pogo.com,inbound,019,Americas,0 +polyvore.com,polyvore.com,inbound,019,Americas,1 +postcardfromhell.com,cyberthugs.com,inbound,019,Americas,1 +potterybarn.com,potterybarn.com,inbound,019,Americas,0 +potterybarnkids.com,potterybarnkids.com,inbound,019,Americas,0 +preferredpetclub.com,preferredpetclub.com,inbound,019,Americas,0 +presslaff.net,dat-e-baseonline.com,inbound,019,Americas,0 +pressmartmail.com,pressmartmail.com,inbound,019,Americas,0 +priorityoneemail.com,priorityoneemail.com,inbound,019,Americas,0 +private-elist.com,private-elist.com,inbound,019,Americas,0 +progressive.com,progressive.com,inbound,019,Americas,0.999893 +promedmail.org,childrenshospital.org,inbound,019,Americas,1 +propertysolutions.com,propertysolutions.com,inbound,019,Americas,1 +prospectgeysercoop.com,prospectgeysercoop.com,inbound,019,Americas,1 +providesupport.com,providesupport.com,inbound,019,Americas,0.000103 +proxyvote.com,adp-ics.com,inbound,019,Americas,0.908635 +psu.edu,psu.edu,inbound,019,Americas,0.073843 +puffinmailer.com,zoothost.com,inbound,019,Americas,0 +purewow.com,purewow.com,inbound,019,Americas,0 +purlsmail.com,purlsmail.com,inbound,019,Americas,0 +pxsmail.com,pxsmail.com,inbound,019,Americas,0 +q.com,synacor.com,inbound,019,Americas,0.999734 +qemailserver.com,qemailserver.com,inbound,019,Americas,0 +qtropnews.com,qtropnews.com,inbound,019,Americas,1.7e-05 +qualicorp.com.br,qualicorp.com.br,inbound,019,Americas,1 +qualitysafelist.com,zoothost.com,inbound,019,Americas,0 +queopinas.com,confirmit.com,inbound,019,Americas,1 +quickrewards.net,quickrewards.net,inbound,019,Americas,0 +quikr.com,quikr.com,inbound,019,Americas,0 +quora.com,quora.com,inbound,019,Americas,1 +qvcemail.com,qvcemail.com,inbound,019,Americas,0 +radarsystems.net,radarsystems.net,inbound,019,Americas,1 +radiantretailapps.com,radiantretailapps.com,inbound,019,Americas,0 +radioshack.com,radioshack.com,inbound,019,Americas,0 +rapattoni.com,rapmls.com,inbound,019,Americas,0 +razerzone.com,chtah.net,inbound,019,Americas,0 +rbc.com,rbc.com,inbound,019,Americas,0.003061 +rdio.com,rdio.com,inbound,019,Americas,1 +realtor.org,realtor.org,inbound,019,Americas,0 +realtytrac.com,realtytrac.com,inbound,019,Americas,0.001681 +recipe.com,meredith.com,inbound,019,Americas,0 +recruit.net,recruit.net,inbound,019,Americas,0 +redbox.com,redbox.com,inbound,019,Americas,0 +redfin.com,redfin.com,inbound,019,Americas,0 +redtri.com,redtri.com,inbound,019,Americas,0 +reebokusnews.com,reebokusnews.com,inbound,019,Americas,0 +reebonz.com,ed10.com,inbound,019,Americas,0 +reebonz.com,reebonz.com,inbound,019,Americas,1 +registro.br,registro.br,inbound,019,Americas,0 +rent.com,rent.com,inbound,019,Americas,0.000397 +rentalcars.com,rentalcars.com,inbound,019,Americas,1 +renweb.com,renweb.com,inbound,019,Americas,0 +researchgate.net,researchgate.net,inbound,019,Americas,0 +restorationhardware.com,restorationhardware.com,inbound,019,Americas,0 +retailmenot.com,retailmenot.com,inbound,019,Americas,0 +reverbnation.com,reverbnation.com,inbound,019,Americas,0 +rewardme.in,bfi0.com,inbound,019,Americas,0 +ricardoeletro.com.br,allin.com.br,inbound,019,Americas,0 +rigzonemail.com,rigzonemail.com,inbound,019,Americas,0 +ripleyperu.com.pe,icommarketing.com,inbound,019,Americas,0 +riseup.net,riseup.net,inbound,019,Americas,1 +rivamail.com,mailurja.com,inbound,019,Americas,0 +rnmk.com,rnmk.com,inbound,019,Americas,0 +roamans.com,neolane.net,inbound,019,Americas,0 +rockwellcollins.com,rockwellcollins.com,inbound,019,Americas,1 +rpinow.org,app-info.net,inbound,019,Americas,0 +rr.com,rr.com,inbound,019,Americas,0.03696 +rr.com,rr.com,outbound,019,Americas,0 +rsgsv.net,rsgsv.net,inbound,019,Americas,0 +rsvpsv.net,rsvpsv.net,inbound,019,Americas,0 +rsvpsv.net,send.esp.br,inbound,019,Americas,0 +rsys2.com,amfam.com,inbound,019,Americas,0 +rsys2.com,cheaptickets.com,inbound,019,Americas,0 +rsys2.com,dishnetworkmail.com,inbound,019,Americas,0 +rsys2.com,e-comms.net,inbound,019,Americas,0 +rsys2.com,eharmony.com,inbound,019,Americas,0 +rsys2.com,fathead.com,inbound,019,Americas,0 +rsys2.com,intuit.com,inbound,019,Americas,0 +rsys2.com,kmart.com,inbound,019,Americas,0 +rsys2.com,kohlernews.com,inbound,019,Americas,0 +rsys2.com,lego.com,inbound,019,Americas,0 +rsys2.com,lenovo.com,inbound,019,Americas,0 +rsys2.com,modcloth.com,inbound,019,Americas,0 +rsys2.com,moxieinteractive.com,inbound,019,Americas,0 +rsys2.com,orbitz.com,inbound,019,Americas,0 +rsys2.com,payless.com,inbound,019,Americas,0 +rsys2.com,petsathome.com,inbound,019,Americas,0 +rsys2.com,quizzle.com,inbound,019,Americas,0 +rsys2.com,robeez.com,inbound,019,Americas,0 +rsys2.com,rsys1.com,inbound,019,Americas,0 +rsys2.com,rsys2.com,inbound,019,Americas,0 +rsys2.com,rsys3.com,inbound,019,Americas,0 +rsys2.com,rsys4.com,inbound,019,Americas,0 +rsys2.com,saucony.com,inbound,019,Americas,0 +rsys2.com,sears.com,inbound,019,Americas,0 +rsys2.com,shopbop.com,inbound,019,Americas,0 +rsys2.com,southwest.com,inbound,019,Americas,0 +rsys2.com,speeddatemail.com,inbound,019,Americas,0 +rsys2.com,thecompanystore.com,inbound,019,Americas,0 +rsys2.com,theknot.com,inbound,019,Americas,0 +rsys5.com,alibris.com,inbound,019,Americas,0 +rsys5.com,allstate-email.com,inbound,019,Americas,0 +rsys5.com,beachmint.com,inbound,019,Americas,0 +rsys5.com,belleandclive.com,inbound,019,Americas,0 +rsys5.com,br.dk,inbound,019,Americas,0 +rsys5.com,charlotterusse.com,inbound,019,Americas,0 +rsys5.com,comixology.com,inbound,019,Americas,0 +rsys5.com,cottonon.com,inbound,019,Americas,0 +rsys5.com,ediblearrangements.com,inbound,019,Americas,0 +rsys5.com,emailworldmarket.com,inbound,019,Americas,0 +rsys5.com,farfetch.com,inbound,019,Americas,0 +rsys5.com,frhiemailcommunications.com,inbound,019,Americas,0 +rsys5.com,harryanddavid.com,inbound,019,Americas,0 +rsys5.com,hollandandbarrett.com,inbound,019,Americas,0 +rsys5.com,icing.com,inbound,019,Americas,0 +rsys5.com,indigo.ca,inbound,019,Americas,0 +rsys5.com,jabong.com,inbound,019,Americas,0 +rsys5.com,jcrew.com,inbound,019,Americas,0 +rsys5.com,jjill.com,inbound,019,Americas,0 +rsys5.com,kanui.com.br,inbound,019,Americas,0 +rsys5.com,kirklands.com,inbound,019,Americas,0 +rsys5.com,lanebryant.com,inbound,019,Americas,0 +rsys5.com,lazada.com,inbound,019,Americas,0 +rsys5.com,leapfrog.com,inbound,019,Americas,0 +rsys5.com,llbean.com,inbound,019,Americas,0 +rsys5.com,lojascolombo.com.br,inbound,019,Americas,0 +rsys5.com,madewell.com,inbound,019,Americas,0 +rsys5.com,magazineluiza.com.br,inbound,019,Americas,0 +rsys5.com,missguided.co.uk,inbound,019,Americas,0 +rsys5.com,moma.org,inbound,019,Americas,0 +rsys5.com,nationalgeographic.com,inbound,019,Americas,0 +rsys5.com,neat.com,inbound,019,Americas,0 +rsys5.com,newbalance.com,inbound,019,Americas,0 +rsys5.com,news-voeazul.com.br,inbound,019,Americas,0 +rsys5.com,nordstrom.com,inbound,019,Americas,0 +rsys5.com,normthompson.com,inbound,019,Americas,0 +rsys5.com,novomundo.com.br,inbound,019,Americas,0 +rsys5.com,ourdeal.com.au,inbound,019,Americas,0 +rsys5.com,pier1.com,inbound,019,Americas,0 +rsys5.com,productmadness.com,inbound,019,Americas,0 +rsys5.com,rainbowshops.com,inbound,019,Americas,0 +rsys5.com,rei.com,inbound,019,Americas,0 +rsys5.com,roadrunnersports.com,inbound,019,Americas,0 +rsys5.com,rosettastone.com,inbound,019,Americas,0 +rsys5.com,seamless.com,inbound,019,Americas,0 +rsys5.com,serenaandlily.com,inbound,019,Americas,0 +rsys5.com,smiles.com.br,inbound,019,Americas,0 +rsys5.com,soubarato.com.br,inbound,019,Americas,0 +rsys5.com,strava.com,inbound,019,Americas,0 +rsys5.com,submarino.com.br,inbound,019,Americas,0 +rsys5.com,thewalkingcompany.com,inbound,019,Americas,0 +rsys5.com,tigerdirect.com,inbound,019,Americas,0 +rsys5.com,udemy.com,inbound,019,Americas,0 +rsys5.com,vitaminshoppe.com,inbound,019,Americas,0 +rsys5.com,vpusa.com,inbound,019,Americas,0 +rsys5.com,walmart.com.br,inbound,019,Americas,0 +rsys5.com,worldofwatches.com,inbound,019,Americas,0 +rsys5.com,xfinity.com,inbound,019,Americas,0 +rue21email.com,rue21email.com,inbound,019,Americas,0 +ruum.com,ruum.com,inbound,019,Americas,0 +saavn.com,saavn.com,inbound,019,Americas,1 +safelistextreme.com,quantumsafelist.com,inbound,019,Americas,0 +safelistpro.com,safelistpro.com,inbound,019,Americas,0.00014 +safeway.com,chtah.com,inbound,019,Americas,0 +sailthru.com,sailthru.com,inbound,019,Americas,0 +saks.com,saks.com,inbound,019,Americas,0 +saksoff5th.com,saksoff5th.com,inbound,019,Americas,0 +salememail.net,salememail.net,inbound,019,Americas,0 +salesforce.com,postini.com,inbound,019,Americas,0.816952 +salesforce.com,salesforce.com,inbound,019,Americas,1 +salesforce.com,salesforce.com,outbound,019,Americas,1 +salliemae.com,salliemae.com,inbound,019,Americas,1 +salsalabs.net,salsalabs.net,inbound,019,Americas,0 +samashmusic.com,wc09.net,inbound,019,Americas,0 +samsclub.com,m0.net,inbound,019,Americas,0 +sans.org,sans.org,inbound,019,Americas,0.497629 +santander.cl,santander.cl,inbound,019,Americas,0.999992 +santander.cl,santandersantiago.cl,inbound,019,Americas,1 +sassieshop.com,sassieshop.com,inbound,019,Americas,0.025887 +savelivefresh.com,livesavemail.com,inbound,019,Americas,1 +savingstar.com,savingstar.com,inbound,019,Americas,1 +sbcglobal.net,yahoo.{...},inbound,019,Americas,1 +sbcglobal.net,yahoodns.net,outbound,019,Americas,1 +sc.com,messagelabs.com,inbound,019,Americas,0.996156 +schwab.com,schwab.com,inbound,019,Americas,0.025055 +seaworld.com,seaworld.com,inbound,019,Americas,0 +securence.com,securence.com,inbound,019,Americas,0.670246 +secureserver.net,secureserver.net,inbound,019,Americas,4.4e-05 +seekingalpha.com,seekingalpha.com,inbound,019,Americas,1 +seekingalpha.com,sendgrid.net,inbound,019,Americas,1 +senate.gov,senate.gov,inbound,019,Americas,0.992994 +sendearnings.com,sendearnings.com,inbound,019,Americas,0 +sendgrid.info,sendgrid.net,inbound,019,Americas,1 +sendgrid.me,sendgrid.net,inbound,019,Americas,1 +sendlane.com,sendlane.com,inbound,019,Americas,0 +serpadres.es,chtah.net,inbound,019,Americas,0 +service-now.com,postini.com,inbound,019,Americas,0.9858 +service-now.com,service-now.com,inbound,019,Americas,0.998562 +sfimg.com,sfimarketing.com,inbound,019,Americas,0 +sfly.com,shutterfly.com,inbound,019,Americas,0 +shaadi.com,shaadi.com,inbound,019,Americas,0.00037 +shadowshopper.com,shadowshopper.com,inbound,019,Americas,5.6e-05 +shaw.ca,shaw.ca,inbound,019,Americas,0 +shaw.ca,shaw.ca,outbound,019,Americas,1 +sheplers.com,sheplers.com,inbound,019,Americas,0 +shiftplanning.com,shiftplanning.com,inbound,019,Americas,0 +shiksha.com,shiksha.com,inbound,019,Americas,0 +shoedazzle.com,shoedazzle.com,inbound,019,Americas,0 +shoes.com,famousfootwear.com,inbound,019,Americas,0 +shopbonton.com,shopbonton.com,inbound,019,Americas,0 +shopcluesemail.com,shopcluesemail.com,inbound,019,Americas,0 +shopcluesmail.com,shopcluesmail.com,inbound,019,Americas,0 +shophq.com,shophq.com,inbound,019,Americas,0 +shopkick.com,shopkick.com,inbound,019,Americas,0 +shopko.com,shopko.com,inbound,019,Americas,0 +shoppersoptimum.ca,thindata.net,inbound,019,Americas,0 +shoptime.com,shoptime.com,inbound,019,Americas,0 +shtyle.fm,shtyle.fm,inbound,019,Americas,0 +sierratradingpost.com,sierratradingpost.com,inbound,019,Americas,0.000885 +sigmabeauty.com,lstrk.net,inbound,019,Americas,1 +sii.cl,sii.cl,inbound,019,Americas,0 +simplyhired.com,simplyhired.com,inbound,019,Americas,0 +siriusxm.com,xmradio.com,inbound,019,Americas,0 +sitecore-mailer.com,sendlabs.com,inbound,019,Americas,0 +sittercity.com,sittercity.com,inbound,019,Americas,0 +sixflags.com,sixflags.com,inbound,019,Americas,0 +skillpages-mailer.com,sendlabs.com,inbound,019,Americas,0 +skillpages-mailer.com,skillpagesmail.com,inbound,019,Americas,0 +sky.com,sky.com,inbound,019,Americas,0 +skype.com,delivery.net,inbound,019,Americas,0 +sld.cu,sld.cu,inbound,019,Americas,1 +slidesharemail.com,newslettergrid.com,inbound,019,Americas,1 +slidesharemail.com,slidesharemail.com,inbound,019,Americas,1 +smartbrief.com,smartbrief.com,inbound,019,Americas,0 +smartdraw.com,smartdraw.com,inbound,019,Americas,0 +smartertravel.com,smartertravelmedia.com,inbound,019,Americas,0.021434 +smartphoneexperts.com,mailgun.net,inbound,019,Americas,1 +snapretail.com,snapretail.com,inbound,019,Americas,1 +socialappsmail.com,socialappsmail.com,inbound,019,Americas,1 +solesociety.com,bronto.com,inbound,019,Americas,0 +solosenders.com,megasenders.com,inbound,019,Americas,8.2e-05 +solosenders.com,traxweb.org,inbound,019,Americas,0 +soma.com,soma.com,inbound,019,Americas,0 +someecards.com,someecards.com,inbound,019,Americas,0 +songkick.com,songkick.com,inbound,019,Americas,1 +sony-latin.com,sony-latin.com,inbound,019,Americas,0.01735 +sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,019,Americas,0 +sourceforge.net,sourceforge.net,inbound,019,Americas,1 +southwest.com,southwest.com,inbound,019,Americas,0 +sp.gov.br,sp.gov.br,inbound,019,Americas,0.496357 +sparkpeople.com,sparkpeople.com,inbound,019,Americas,0 +spectersoft.com,spectersoft.com,inbound,019,Americas,0 +spencersonline.com,spencersonline.com,inbound,019,Americas,0 +spiritairlines.com,ctd004.net,inbound,019,Americas,0 +spiritairlines.com,ctd005.net,inbound,019,Americas,0 +splitwise.com,splitwise.com,inbound,019,Americas,1 +sportlobster.com,sportlobster.com,inbound,019,Americas,1 +sportsdirect.com,sportsdirect.com,inbound,019,Americas,0 +sportsline.com,cbsig.net,inbound,019,Americas,1 +sportsmansguide.com,sportsmansguide.com,inbound,019,Americas,0.736903 +sprint.com,m0.net,inbound,019,Americas,0 +squareup.com,squareup.com,inbound,019,Americas,0.986452 +stanford.edu,highwire.org,inbound,019,Americas,1 +stanford.edu,stanford.edu,inbound,019,Americas,0.93025 +stansberryresearch.com,stansberryresearch.com,inbound,019,Americas,0.001541 +staples.com,staples.com,inbound,019,Americas,0.006379 +starbucks.com,starbucks.com,inbound,019,Americas,0 +stardockcorporation.com,stardockcorporation.com,inbound,019,Americas,0 +stardockentertainment.info,stardockentertainment.info,inbound,019,Americas,0 +startribune.com,startribune.com,inbound,019,Americas,0.001728 +startwire.com,jobsreport.com,inbound,019,Americas,1 +startwire.com,startwire.com,inbound,019,Americas,1 +state.gov,state.gov,inbound,019,Americas,0 +statefarm.com,statefarm.com,inbound,019,Americas,1 +steinmart.com,steinmart.com,inbound,019,Americas,0 +stevemadden.com,stevemadden.com,inbound,019,Americas,0 +streetauthoritydaily.com,streetauthoritydaily.com,inbound,019,Americas,0 +stumblemail.com,stumblemail.com,inbound,019,Americas,1 +suafaturanet.com.br,suafaturanet.com.br,inbound,019,Americas,0.995846 +subscribermail.com,subscribermail.com,inbound,019,Americas,0 +sungard.com,postini.com,inbound,019,Americas,0.498265 +sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,019,Americas,1 +superbalist.com,sailthru.com,inbound,019,Americas,0 +supersafemailer.com,zoothost.com,inbound,019,Americas,0 +supremelist.com,onlinehome-server.com,inbound,019,Americas,1 +surlatable.com,surlatable.com,inbound,019,Americas,0 +surveyjobopportunities.com,surveyjobopportunities.com,inbound,019,Americas,3.7e-05 +surveymonkey.com,surveymonkey.com,inbound,019,Americas,0 +surveysavvy.com,surveysavvy.com,inbound,019,Americas,0 +sylectus.com,sylectus.com,inbound,019,Americas,0.361297 +sympatico.ca,hotmail.{...},outbound,019,Americas,1 +taggedmail.com,taggedmail.com,inbound,019,Americas,0 +tamu.edu,tamu.edu,inbound,019,Americas,0.249876 +tangeroutletsusa.com,bronto.com,inbound,019,Americas,0 +target-safelist.com,safelistpro.com,inbound,019,Americas,0.000215 +targetproblaster.com,targetproblaster.com,inbound,019,Americas,0 +targetx.com,targetx.com,inbound,019,Americas,0 +tarot.com,tarot.com,inbound,019,Americas,0 +tastefullysimpleparty.com,bigfootinteractive.com,inbound,019,Americas,0 +tastingtable.com,tastingtable.com,inbound,019,Americas,0 +taxi4sure.net,infimail.com,inbound,019,Americas,0 +teach12.net,teach12.net,inbound,019,Americas,0 +techtarget.com,techtarget.com,inbound,019,Americas,0.001771 +telus.net,telus.net,inbound,019,Americas,0.006226 +ten24mail.com,ten24mail.com,inbound,019,Americas,0 +terra.com,terra.com,inbound,019,Americas,0.000463 +terra.com.br,terra.com,inbound,019,Americas,0 +terra.com.br,terra.com,outbound,019,Americas,0 +tesco.com,tesco.com,inbound,019,Americas,0 +testfunda.com,testfunda.com,inbound,019,Americas,0 +textnow.me,textnow.me,inbound,019,Americas,0 +tgw.com,tgw.com,inbound,019,Americas,0 +theanimalrescuesite.com,theanimalrescuesite.com,inbound,019,Americas,0 +theatermania.com,wc09.net,inbound,019,Americas,0 +thebay.com,thebay.com,inbound,019,Americas,0 +thegrommet.com,lstrk.net,inbound,019,Americas,1 +thehut.com,thehut.com,inbound,019,Americas,0 +thelimited.com,thelimited.com,inbound,019,Americas,0 +themailbagsafelist.com,thomas-j-brown.com,inbound,019,Americas,0 +theoutnet.com,theoutnet.com,inbound,019,Americas,0 +theskimm.com,theskimm.com,inbound,019,Americas,0 +thesovereigninvestor.com,sovereignsociety.com,inbound,019,Americas,0 +thirtyonegifts.com,thirtyonegifts.com,inbound,019,Americas,0 +thoughtful-mind.com,thoughtful-mind.com,inbound,019,Americas,0 +thumbtack.com,thumbtack.com,inbound,019,Americas,1 +ticketmaster.com,ticketmaster.com,inbound,019,Americas,0.769059 +tmart.com,chtah.net,inbound,019,Americas,0 +tmp.com,tmpw.com,inbound,019,Americas,0 +toluna.com,toluna.com,inbound,019,Americas,0 +tomtommailer.com,tomtommailer.com,inbound,019,Americas,0 +topspin.net,topspin.net,inbound,019,Americas,1 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,019,Americas,0 +touchbase2.com,infimail.com,inbound,019,Americas,0 +touchbase2.com,mailurja.com,inbound,019,Americas,0 +touchbasepro.com,touchbasepro.com,inbound,019,Americas,0 +townhallmail.com,townhallmail.com,inbound,019,Americas,0 +townsquaremedia.info,sailthru.com,inbound,019,Americas,0 +toysrus.com,epsl1.com,inbound,019,Americas,0 +trafficboostermailer.com,trafficboostermailer.com,inbound,019,Americas,1 +trafficleads2incomevm.com,zoothost.com,inbound,019,Americas,0 +trafficprolist.com,thomas-j-brown.com,inbound,019,Americas,0 +trialsmith.com,membercentral.com,inbound,019,Americas,0 +tricaemidia.com.br,tricaemidia.com.br,inbound,019,Americas,0 +tripit.com,tripit.com,inbound,019,Americas,0 +tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,019,Americas,0 +tumblr.com,tumblr.com,inbound,019,Americas,1 +twitch.tv,justin.tv,inbound,019,Americas,0 +uber.com,uber.com,inbound,019,Americas,1 +ubi.com,ubi.com,inbound,019,Americas,0 +ubivox.com,ubivox.com,inbound,019,Americas,0.935106 +ucla.edu,ucla.edu,inbound,019,Americas,0.806204 +uhaul.com,uhaul.com,inbound,019,Americas,0.997947 +uiuc.edu,illinois.edu,inbound,019,Americas,0.999815 +ultimateadsites.net,ultimateadsites.net,inbound,019,Americas,1 +umd.edu,umd.edu,inbound,019,Americas,0.021709 +umn.edu,umn.edu,inbound,019,Americas,0.966964 +umpiredigital.com,inboxmarketer.com,inbound,019,Americas,0 +united.com,coair.com,inbound,019,Americas,1 +united.com,united.com,inbound,019,Americas,0 +universalorlando.com,universalorlando.com,inbound,019,Americas,0 +unrollmail.com,unrollmail.com,inbound,019,Americas,1 +uol.com.br,uol.com.br,inbound,019,Americas,0 +uol.com.br,uol.com.br,outbound,019,Americas,0 +upenn.edu,upenn.edu,inbound,019,Americas,0.16521 +upromise.com,delivery.net,inbound,019,Americas,0 +ups.com,ups.com,inbound,019,Americas,0.997955 +urbanoutfitters.com,freepeople.com,inbound,019,Americas,0 +usaa.com,usaa.com,inbound,019,Americas,0.072606 +usafisnews.org,usafisnews.org,inbound,019,Americas,0 +usajobs.gov,opm.gov,inbound,019,Americas,0.931948 +usc.edu,usc.edu,inbound,019,Americas,0.300314 +uscourts.gov,uscourts.gov,inbound,019,Americas,0 +uslargestsafelist.com,zoothost.com,inbound,019,Americas,0 +usndr.com,unisender.com,inbound,019,Americas,1 +usp.br,usp.br,inbound,019,Americas,0.044595 +usps.com,usps.gov,inbound,019,Americas,0.909591 +usps.gov,usps.gov,inbound,019,Americas,0.940315 +ustream.tv,mailgun.net,inbound,019,Americas,1 +ustream.tv,mailgun.us,inbound,019,Americas,1 +utfsm.cl,utfsm.cl,inbound,019,Americas,0.000352 +vacationstogo.com,vacationstogo.com,inbound,019,Americas,0 +vagas.com.br,vagas.com.br,inbound,019,Americas,0 +valuedopinions.co.uk,researchnow-usa.com,inbound,019,Americas,1 +vanheusenrewards.com,vanheusenrewards.com,inbound,019,Americas,0 +velocityfrequentflyer.com,virginaustralia.com,inbound,019,Americas,0 +vente-exclusive.com,vente-exclusive.com,inbound,019,Americas,0 +venusswimwear.net,venusswimwear.net,inbound,019,Americas,0 +verabradleymail.com,verabradleymail.com,inbound,019,Americas,0 +verizon.net,verizon.net,inbound,019,Americas,0.00713 +verizon.net,verizon.net,outbound,019,Americas,0 +verizonwireless.com,verizonwireless.com,inbound,019,Americas,0.972245 +vhmnetworkemail.com,jobdiagnosis.com,inbound,019,Americas,0 +viagogo.com,chtah.net,inbound,019,Americas,0 +viajanet.com.br,viajanet.com.br,inbound,019,Americas,0 +victoriassecret.com,victoriassecret.com,inbound,019,Americas,0 +videotron.ca,videotron.ca,inbound,019,Americas,0.005886 +vikingrivercruises.com,bfi0.com,inbound,019,Americas,0 +vimeo.com,vimeo.com,inbound,019,Americas,0 +vindale.com,vindale.com,inbound,019,Americas,0 +vipvoice.com,npdor.com,inbound,019,Americas,0 +viraladmagnet.com,viraladmagnet.com,inbound,019,Americas,1 +virginia.edu,virginia.edu,inbound,019,Americas,0.055153 +virtualtarget.com.br,virtualtarget.com.br,inbound,019,Americas,0 +vitacost.com,vitacost.com,inbound,019,Americas,0 +vitaladviews.com,zoothost.com,inbound,019,Americas,0 +vivastreet.com,viwii.net,inbound,019,Americas,0 +votervoice.net,votervoice.net,inbound,019,Americas,0 +vresp.com,verticalresponse.com,inbound,019,Americas,0 +vt.edu,vt.edu,inbound,019,Americas,0.978548 +vtext.com,vtext.com,inbound,019,Americas,0 +vtext.com,vtext.com,outbound,019,Americas,1 +vuezone.com,vuezone.com,inbound,019,Americas,0 +vzwpix.com,vtext.com,inbound,019,Americas,0 +wagjag.com,wagjag.com,inbound,019,Americas,0 +walgreens.com,walgreens.com,inbound,019,Americas,0.006128 +wallst.com,wallst.com,inbound,019,Americas,0 +wallstreetdaily.com,wallstreetdaily.com,inbound,019,Americas,0 +waterstones.com,waterstones.com,inbound,019,Americas,0 +wattpad.com,wattpad.com,inbound,019,Americas,1 +way2movies.net,way2movies.net,inbound,019,Americas,0 +wayfair.com,csnstores.com,inbound,019,Americas,0 +webs.com,epsl1.com,inbound,019,Americas,0 +websaver.ca,websaver.ca,inbound,019,Americas,0 +websitesettings.com,stabletransit.com,inbound,019,Americas,0 +websitewelcome.com,websitewelcome.com,inbound,019,Americas,1 +webstars2k.com,webstars2k.com,inbound,019,Americas,1 +weebly.com,weeblymail.com,inbound,019,Americas,0 +wellsfargo.com,wellsfargo.com,inbound,019,Americas,1.0 +wellsfargoadvisors.com,wellsfargo.com,inbound,019,Americas,1 +westelm.com,westelm.com,inbound,019,Americas,0 +westmarine.com,westmarine.com,inbound,019,Americas,0 +wetransfer.com,wetransfer.com,inbound,019,Americas,1 +wetsealnewsletter.com,wetsealnewsletter.com,inbound,019,Americas,0 +whatcounts.com,wc09.net,inbound,019,Americas,0 +whentowork.com,whentowork.com,inbound,019,Americas,0 +wikia.com,wikia.com,inbound,019,Americas,1 +williams-sonoma.com,williams-sonoma.com,inbound,019,Americas,0 +windstream.net,windstream.net,inbound,019,Americas,0.002172 +wine.com,wine.com,inbound,019,Americas,0 +winkalmail.com,fnbox.com,inbound,019,Americas,0 +wisdomitservices.com,infimail.com,inbound,019,Americas,0 +wldemail-mailer.com,wldemail.com,inbound,019,Americas,0 +womanwithin.com,womanwithin.com,inbound,019,Americas,0 +woodforest.com,woodforest.com,inbound,019,Americas,1 +workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,019,Americas,0 +worldsingles.co,worldsingles.com,inbound,019,Americas,0 +wotif.com,whatcounts.com,inbound,019,Americas,0 +wp.com,wordpress.com,inbound,019,Americas,0 +wuaki.tv,chtah.net,inbound,019,Americas,0 +wustl.edu,app-info.net,inbound,019,Americas,0 +wwe.com,wwe.com,inbound,019,Americas,8e-06 +xen.org,xen.org,inbound,019,Americas,1 +xmeeting.com,xmeeting.com,inbound,019,Americas,1 +xmr3.com,messagereach.com,inbound,019,Americas,0.999933 +xoom.com,xoom.com,inbound,019,Americas,0 +xpnews.com.br,xpnews.com.br,inbound,019,Americas,1 +yahoo.{...},postini.com,inbound,019,Americas,0.664066 +yahoo.{...},yahoo.{...},inbound,019,Americas,0.999 +yahoo.{...},yahoodns.net,outbound,019,Americas,1 +yelp.com,yelpcorp.com,inbound,019,Americas,0 +yipit.com,yipit.com,inbound,019,Americas,1 +ymail.com,yahoo.{...},inbound,019,Americas,1 +ymail.com,yahoodns.net,outbound,019,Americas,1 +yoox.com,yoox.com,inbound,019,Americas,0 +youmail.com,youmail.com,inbound,019,Americas,0 +youreletters3.com,equitymaster.com,inbound,019,Americas,0 +yourezads.com,yourezads.com,inbound,019,Americas,0.002974 +yourezlist.com,simplicityads.com,inbound,019,Americas,9.5e-05 +yourhostingaccount.com,yourhostingaccount.com,inbound,019,Americas,0 +youversion.com,youversion.com,inbound,019,Americas,1 +zacks.com,zacks.com,inbound,019,Americas,0.999687 +zara.com,cheetahmail.com,inbound,019,Americas,0 +zendesk.com,zdsys.com,inbound,019,Americas,1 +zgalleriestyle.com,zgalleriestyle.com,inbound,019,Americas,0 +zibmail.info,zibmail.info,inbound,019,Americas,9e-06 +zinio.com,zinio.com,inbound,019,Americas,0 +zipalerts.com,zipalerts.com,inbound,019,Americas,1 +ziprealty.com,ziprealty.com,inbound,019,Americas,0.994545 +ziprecruiter.com,ziprecruiter.com,inbound,019,Americas,1 +zivamewear.com,infimail.com,inbound,019,Americas,0 +zoothost.com,zoothost.com,inbound,019,Americas,0.107168 +zorpia.com,zorpia.com,inbound,019,Americas,1 +zulily.com,zulily.com,inbound,019,Americas,0 +zzounds.com,zzounds.com,inbound,019,Americas,0 +0101.co.jp,0101.co.jp,inbound,142,Asia,0 +04auto.biz,01auto.biz,inbound,142,Asia,0 +123.com.tw,123.com.tw,inbound,142,Asia,0 +160by2.us,160by2.us,inbound,142,Asia,0 +163.com,163.com,inbound,142,Asia,0.534088 +163.com,netease.com,outbound,142,Asia,1 +17life.com.tw,17life.com.tw,inbound,142,Asia,0 +1lejend.com,asumeru.com,inbound,142,Asia,0 +1lejend.com,asumeru001.com,inbound,142,Asia,0 +33go.com.tw,33go.com.tw,inbound,142,Asia,0 +518.com.tw,518.com.tw,inbound,142,Asia,0 +7net.com.tw,7net.com.tw,inbound,142,Asia,0 +ab0.jp,altovision.co.jp,inbound,142,Asia,0 +activetrail.com,atmailsvr.net,inbound,142,Asia,0 +activetrail.com,mymarketing.co.il,inbound,142,Asia,0 +adchiever.com,kinder-rash-marketing.com,inbound,142,Asia,0 +adityabirla.com,adityabirla.com,inbound,142,Asia,0.00615 +agora.co.il,1host.co.il,inbound,142,Asia,0 +airtel.com,airtel.in,inbound,142,Asia,0.064377 +alertsindia.in,alertsindia.in,inbound,142,Asia,1 +alibaba.com,alibaba.com,inbound,142,Asia,0 +aliexpress.com,alibaba.com,inbound,142,Asia,0 +alipay.com,alipay.com,inbound,142,Asia,1 +alljob.co.il,alljob.co.il,inbound,142,Asia,0 +amazon.{...},amazon.{...},inbound,142,Asia,0 +ana.co.jp,ana.co.jp,inbound,142,Asia,0.534416 +anghami.com,mailgun.net,inbound,142,Asia,1 +apple.com,apple.com,inbound,142,Asia,0.999817 +artscow.com,dyxnet.com,inbound,142,Asia,0.000355 +asus.com,asus.com,inbound,142,Asia,0 +baycrews.co.jp,webcas.net,inbound,142,Asia,0 +beamtele.com,beamtele.com,inbound,142,Asia,0 +beanfun.com,beanfun.com,inbound,142,Asia,1 +belluna.net,belluna.net,inbound,142,Asia,0 +betrend.com,betrend.com,inbound,142,Asia,0 +bharatmatrimony.com,bharatmatrimony.com,inbound,142,Asia,1 +blayn.jp,bserver.jp,inbound,142,Asia,0 +bme.jp,bserver.jp,inbound,142,Asia,0 +bookoffonline.co.jp,bookoffonline.co.jp,inbound,142,Asia,0 +brands4friends.jp,webcas.net,inbound,142,Asia,0 +brandsfever.com,mailgun.net,inbound,142,Asia,1 +buyma.com,buyma.com,inbound,142,Asia,0 +cam2life.com,hinet.net,inbound,142,Asia,0 +camsonline.com,camsonline.com,inbound,142,Asia,0.136261 +careesma.in,careesma.in,inbound,142,Asia,0 +ccavenue.com,avenues.info,inbound,142,Asia,1 +chance.com,data-hotel.net,inbound,142,Asia,0 +chiangcn.com,chiangcn.com,inbound,142,Asia,0 +chinatrust.com.tw,chinatrust.com.tw,inbound,142,Asia,0.001272 +cityheaven.net,cityheaven.net,inbound,142,Asia,0 +clickmailer.jp,clickmailer.jp,inbound,142,Asia,9e-05 +cocacola.co.jp,cocacola.co.jp,inbound,142,Asia,0 +combzmail.jp,combzmail.jp,inbound,142,Asia,0 +communitymatrimony.com,communitymatrimony.com,inbound,142,Asia,1 +conrepmail.com,conrepmail.com,inbound,142,Asia,0 +cookpad.com,cookpad.com,inbound,142,Asia,0 +cpc.gov.in,cpc.gov.in,inbound,142,Asia,0 +crmstyle.com,crmstyle.com,inbound,142,Asia,0 +crocos.jp,crocos.jp,inbound,142,Asia,0 +ctrip.com,ctrip.com,inbound,142,Asia,0.019448 +cuenote.jp,cuenote.jp,inbound,142,Asia,0 +cw.com.tw,cw.com.tw,inbound,142,Asia,0.002535 +cyberlinkmember.com,cyberlinkmember.com,inbound,142,Asia,0 +dena.ne.jp,dena.ne.jp,inbound,142,Asia,0.000249 +dietnavi.com,data-hotel.net,inbound,142,Asia,0 +dip-net.co.jp,dip-net.co.jp,inbound,142,Asia,0 +directresponsemanager.com,wide.ne.jp,inbound,142,Asia,0 +disc.co.jp,disc.co.jp,inbound,142,Asia,0.001051 +dishtv.co.in,dishtv.co.in,inbound,142,Asia,0.047664 +disqus.net,disqus.net,inbound,142,Asia,0 +dks.com.tw,dks.com.tw,inbound,142,Asia,0.018134 +dmm.com,dmm.com,inbound,142,Asia,0 +docomo.ne.jp,docomo.ne.jp,inbound,142,Asia,0 +docomo.ne.jp,docomo.ne.jp,outbound,142,Asia,0 +dreammail.ne.jp,dreammail.jp,inbound,142,Asia,0 +drushim.co.il,drushim.co.il,inbound,142,Asia,0 +dstyleweb.com,dstyleweb.com,inbound,142,Asia,0.001099 +ec21.com,ec21.com,inbound,142,Asia,0.002263 +ecnavi.jp,ecnavi.jp,inbound,142,Asia,0 +en-japan.com,en-japan.com,inbound,142,Asia,0.000204 +eonet.ne.jp,eonet.ne.jp,inbound,142,Asia,0.99955 +epaper.com.tw,epaper.com.tw,inbound,142,Asia,0 +eplus.jp,eplus.jp,inbound,142,Asia,0 +eslitebooks.com,eslitebooks.com,inbound,142,Asia,0 +euromsg.net,euromsg.net,inbound,142,Asia,0 +evaair.com,evaair.com,inbound,142,Asia,0.007363 +ezweb.ne.jp,ezweb.ne.jp,inbound,142,Asia,0.282076 +ezweb.ne.jp,ezweb.ne.jp,outbound,142,Asia,0 +farmersonly.com,mailgun.net,inbound,142,Asia,1 +felissimo.jp,felissimo.jp,inbound,142,Asia,0 +finansbank.com.tr,finansbank.com.tr,inbound,142,Asia,0.927 +fmworld.net,fmworld.net,inbound,142,Asia,0 +fofa.jp,mpme.jp,inbound,142,Asia,0 +freeml.com,gmo-media.jp,inbound,142,Asia,0 +ftchinese.com,ftchinese.com,inbound,142,Asia,0 +fubonshop.com,fubonshop.com,inbound,142,Asia,0 +gamecity.ne.jp,gamecity.ne.jp,inbound,142,Asia,0 +garanti.com.tr,garanti.com.tr,inbound,142,Asia,0.421936 +gilt.jp,gilt.jp,inbound,142,Asia,0 +globalsources.com,globalsources.com,inbound,142,Asia,0.003024 +globetel.com.ph,globetel.com.ph,inbound,142,Asia,1 +gmail.com,asianet.co.th,inbound,142,Asia,0.998511 +gmail.com,au-net.ne.jp,inbound,142,Asia,1 +gmail.com,bbtec.net,inbound,142,Asia,1 +gmail.com,data-hotel.net,inbound,142,Asia,0.000607 +gmail.com,hinet.net,inbound,142,Asia,0.973114 +gmail.com,mtnl.net.in,inbound,142,Asia,0.999527 +gmail.com,netvigator.com,inbound,142,Asia,0.818263 +gmail.com,ocn.ne.jp,inbound,142,Asia,0.99466 +gmail.com,panda-world.ne.jp,inbound,142,Asia,1 +gmail.com,seed.net.tw,inbound,142,Asia,0.992252 +gmail.com,singnet.com.sg,inbound,142,Asia,0.968389 +gmail.com,totbb.net,inbound,142,Asia,0.999876 +gmo.jp,gmo-media.jp,inbound,142,Asia,0 +gmoes.jp,gmoes.jp,inbound,142,Asia,0 +gmt.ne.jp,gmt.ne.jp,inbound,142,Asia,0 +gnavi.co.jp,gnavi.co.jp,inbound,142,Asia,0.002441 +gomaji.com,gomaji.com,inbound,142,Asia,0 +gree.jp,gree.jp,inbound,142,Asia,0 +groupon.{...},groupon.{...},inbound,142,Asia,0 +gunosy.com,gunosy.com,inbound,142,Asia,0.998682 +gurunavi.jp,gurunavi.jp,inbound,142,Asia,0 +hdfcbank.com,powerelay.com,inbound,142,Asia,1 +hdfcbank.net,powerelay.com,inbound,142,Asia,1 +hdfcbank.net,quickvmail.com,inbound,142,Asia,0 +heteml.jp,heteml.jp,inbound,142,Asia,0.950296 +hinet.net,hinet.net,inbound,142,Asia,0.007064 +hinet.net,hinet.net,outbound,142,Asia,0.00565 +home.ne.jp,zaq.ne.jp,inbound,142,Asia,9e-06 +i-part.com.tw,i-part.com.tw,inbound,142,Asia,0.001865 +ibps.in,sify.net,inbound,142,Asia,0 +ibpsorg.org,sify.net,inbound,142,Asia,0 +icicibank.com,icicibank.com,inbound,142,Asia,0.009835 +icicisecurities.com,icicibank.com,inbound,142,Asia,0.000803 +icloud.com,apple.com,inbound,142,Asia,1 +imi.ne.jp,lifemedia.jp,inbound,142,Asia,0 +indiaproperty.com,indiaproperty.com,inbound,142,Asia,0 +intage.co.jp,intage.co.jp,inbound,142,Asia,0.00557 +intuit.com,intuit.com,inbound,142,Asia,0.983698 +isbank.com.tr,isbank.com.tr,inbound,142,Asia,0.009655 +itsmyascent.com,itsmyascent.com,inbound,142,Asia,0 +itunes.com,apple.com,inbound,142,Asia,1 +jcity.com,jcity.com,inbound,142,Asia,0 +jobinthailand.com,jobinthailand.com,inbound,142,Asia,1 +jobmaster.co.il,jobmaster1.co.il,inbound,142,Asia,0 +jobsdbalert.co.id,jobsdbalert.co.id,inbound,142,Asia,0 +jobsdbalert.com,jobsdbalert.com,inbound,142,Asia,0 +jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,142,Asia,0 +jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,142,Asia,0 +jobstreet.com,jobstreet.com,inbound,142,Asia,0 +joshin.co.jp,joshin.co.jp,inbound,142,Asia,8e-06 +kagoya.net,kagoya.net,inbound,142,Asia,0.013052 +karvy.com,karvy.com,inbound,142,Asia,0.045186 +kasikornbank.com,kasikornbank.com,inbound,142,Asia,0 +kotak.com,kotak.com,inbound,142,Asia,0.005566 +krs.bz,tricorn.net,inbound,142,Asia,0 +kvbmail.com,kvbmail.com,inbound,142,Asia,0 +lancers.jp,lancers.jp,inbound,142,Asia,0 +lelong.my,lelong.com.my,inbound,142,Asia,1 +lelong.my,lelong.net.my,inbound,142,Asia,1 +line.me,naver.com,inbound,142,Asia,1 +livedoor.com,livedoor.com,inbound,142,Asia,0 +luxa.jp,luxa.jp,inbound,142,Asia,0 +m3.com,m3.com,inbound,142,Asia,0 +mag2.com,tandem-m.com,inbound,142,Asia,0 +magicbricks.com,tbsl.in,inbound,142,Asia,0 +mail-boss.com,mail-boss.com,inbound,142,Asia,0 +mailgun.org,mailgun.net,inbound,142,Asia,1 +mbga.jp,mbga.jp,inbound,142,Asia,0 +mixi.jp,mixi.jp,inbound,142,Asia,0 +mmagic.jp,mmagic.jp,inbound,142,Asia,0.000531 +mobile01.com,mobile01.com,inbound,142,Asia,0 +monex.co.jp,monex.co.jp,inbound,142,Asia,0.019424 +moneycontrol.com,active18.com,inbound,142,Asia,0 +moneyforward.com,moneyforward.com,inbound,142,Asia,0 +monipla.jp,aainc.co.jp,inbound,142,Asia,0 +monster.co.in,monster.co.in,inbound,142,Asia,0 +morhipo.com,euromsg.net,inbound,142,Asia,0 +mpme.jp,mpme.jp,inbound,142,Asia,0 +mpse.jp,emsaqua.jp,inbound,142,Asia,0 +mpse.jp,emsbeige.jp,inbound,142,Asia,0 +mpse.jp,emsbrown.jp,inbound,142,Asia,0 +mpse.jp,emscyan.jp,inbound,142,Asia,0 +mpse.jp,emsgold.jp,inbound,142,Asia,0 +mpse.jp,emslime.jp,inbound,142,Asia,0 +mpse.jp,emsnavy.jp,inbound,142,Asia,0 +mpse.jp,emspink.jp,inbound,142,Asia,0 +mpse.jp,emssnow.jp,inbound,142,Asia,0 +mpse.jp,mpme.jp,inbound,142,Asia,0 +mpse.jp,yahoo.co.jp,inbound,142,Asia,0 +mynavi.jp,mynavi.jp,inbound,142,Asia,3.5e-05 +naver.com,naver.com,inbound,142,Asia,1 +naver.com,naver.com,outbound,142,Asia,1 +nesinemail.com,euromsg.net,inbound,142,Asia,0 +net-survey.jp,net-survey.jp,inbound,142,Asia,0 +netbk.co.jp,netbk.co.jp,inbound,142,Asia,0.010995 +next-engine.org,next-engine.org,inbound,142,Asia,1 +nextdoor.com,nextdoor.com,inbound,142,Asia,1 +nic.in,relayout.nic.in,inbound,142,Asia,0 +nicovideo.jp,nicovideo.jp,inbound,142,Asia,0 +nifty.com,nifty.com,inbound,142,Asia,0.762068 +nikkei.com,nikkei.co.jp,inbound,142,Asia,0 +nikkeibp.co.jp,nikkeibp.co.jp,inbound,142,Asia,4.9e-05 +nissen.jp,nissen.jp,inbound,142,Asia,0 +ocn.ad.jp,ocn.ad.jp,inbound,142,Asia,0 +ocn.ne.jp,ocn.ad.jp,inbound,142,Asia,0 +p-world.co.jp,p-world.co.jp,inbound,142,Asia,0 +panasonic.jp,panasonic.jp,inbound,142,Asia,0 +pepabo.com,pepabo.com,inbound,142,Asia,0 +pia.jp,pia.jp,inbound,142,Asia,0 +playstation.com,playstation.com,inbound,142,Asia,0 +pointtown.com,gmo-media.jp,inbound,142,Asia,0 +pttplc.com,pttgrp.com,inbound,142,Asia,0 +publicators.com,publicators.com,inbound,142,Asia,0 +qoo10.jp,qoo10.jp,inbound,142,Asia,0 +qq.com,qq.com,inbound,142,Asia,0.999973 +quickbooks.com,intuit.com,inbound,142,Asia,0.99908 +rakuten.co.jp,rakuten.co.jp,inbound,142,Asia,0 +rakuten.co.jp,yahoo.co.jp,inbound,142,Asia,0 +rakuten.ne.jp,rakuten.co.jp,inbound,142,Asia,0 +realus.co.jp,realus.co.jp,inbound,142,Asia,0 +recochoku.jp,recochoku.jp,inbound,142,Asia,0 +rediffmail.com,akadns.net,outbound,142,Asia,0 +rediffmail.com,rediffmail.com,inbound,142,Asia,0 +relianceada.com,relianceada.com,inbound,142,Asia,0.276515 +research-panel.jp,research-panel.jp,inbound,142,Asia,0 +responder.co.il,responder.co.il,inbound,142,Asia,1 +runnet.jp,runnet.jp,inbound,142,Asia,0 +rutenmail.com.tw,rutenmail.com.tw,inbound,142,Asia,0 +sahibinden.com,sahibinden.com,inbound,142,Asia,0 +saisoncard.co.jp,saisoncard.co.jp,inbound,142,Asia,0 +sakura.ne.jp,sakura.ne.jp,inbound,142,Asia,0.673305 +samsung.com,samsung.com,inbound,142,Asia,0.008685 +saramin.co.kr,saramin.co.kr,inbound,142,Asia,0 +sbi.co.in,sbi.co.in,inbound,142,Asia,0 +sbr-inc.co.jp,hdemail.jp,inbound,142,Asia,0 +sc.com,sc.com,inbound,142,Asia,0.99683 +secure.ne.jp,secure.ne.jp,inbound,142,Asia,3.7e-05 +secureserver.net,secureserver.net,inbound,142,Asia,0.000386 +shinseibank.com,shinseibank.com,inbound,142,Asia,0 +shukatsu.jp,shukatsu.jp,inbound,142,Asia,0 +simplymarry.com,tbsl.in,inbound,142,Asia,0 +smartphoneexperts.com,mailgun.net,inbound,142,Asia,1 +smp.ne.jp,smp.ne.jp,inbound,142,Asia,0 +snapdealmail.in,snapdealmail.in,inbound,142,Asia,0 +sofmap.com,sofmap.com,inbound,142,Asia,0.0092 +softbank.jp,softbank.jp,inbound,142,Asia,0 +sony.jp,sony.jp,inbound,142,Asia,0.000654 +sourcenext.info,sourcenext.info,inbound,142,Asia,0 +surfmandelivery.com,surfmandelivery.com,inbound,142,Asia,0 +synergy360.jp,crmstyle.com,inbound,142,Asia,0 +taipeifubon.com.tw,taipeifubon.com.tw,inbound,142,Asia,0 +techgig.com,tbsl.in,inbound,142,Asia,0 +thinkvidya.com,thinkvidya.com,inbound,142,Asia,0 +ticketmonster.co.kr,ticketmonster.co.kr,inbound,142,Asia,0 +timesjobs.com,tbsl.in,inbound,142,Asia,0 +timesjobsmail.com,tbsl.in,inbound,142,Asia,0 +timesofindia.com,indiatimes.com,inbound,142,Asia,0 +tobizaru.jp,tobizaru.jp,inbound,142,Asia,0 +tocoo.jp,aics.ne.jp,inbound,142,Asia,0 +tower.jp,tower.jp,inbound,142,Asia,0 +treemall.com.tw,symphox.com,inbound,142,Asia,0 +tsite.jp,tsite.jp,inbound,142,Asia,0 +tsutaya.co.jp,tsutaya.co.jp,inbound,142,Asia,0 +turkcell.com.tr,turkcell.com.tr,inbound,142,Asia,0.043492 +type.jp,type.jp,inbound,142,Asia,0 +u-shopping.com.tw,u-shopping.com.tw,inbound,142,Asia,0 +udnpaper.com,udnpaper.com,inbound,142,Asia,0.998858 +udnshopping.com,udnshopping.com,inbound,142,Asia,0 +vodafone.com,vodafone.in,inbound,142,Asia,0.617518 +vpass.ne.jp,clickmailer.jp,inbound,142,Asia,0 +vpcontact.com,vpcontact.com,inbound,142,Asia,0 +way2sms.biz,way2sms.biz,inbound,142,Asia,0 +way2sms.in,way2sms.in,inbound,142,Asia,0 +webcas.net,webcas.net,inbound,142,Asia,0 +wechat.com,qq.com,inbound,142,Asia,1 +www.gov.tw,hinet.net,inbound,142,Asia,8e-05 +yahoo.co.jp,yahoo.co.jp,inbound,142,Asia,8e-06 +yahoo.co.jp,yahoo.co.jp,outbound,142,Asia,0 +yahoo.{...},yahoo.co.jp,inbound,142,Asia,0 +yakala.co,euromsg.net,inbound,142,Asia,0 +yesbank.in,yesbank.in,inbound,142,Asia,0.241591 +yodobashi.com,yodobashi.com,inbound,142,Asia,0 +zizigo.com,euromsg.net,inbound,142,Asia,0 +3suisses.be,3suisses.be,inbound,150,Europe,0 +3suisses.fr,3suisses.fr,inbound,150,Europe,0 +aanotifier.nl,aanotifier.nl,inbound,150,Europe,0 +adidas.com,neolane.net,inbound,150,Europe,0 +admail.hu,sanomaonline.hu,inbound,150,Europe,0 +adsender.us,adsender.us,inbound,150,Europe,0 +adverts.ie,adverts.ie,inbound,150,Europe,1 +advfn.com,advfn.com,inbound,150,Europe,0.635382 +agnitas.de,agnitas.de,inbound,150,Europe,0.999265 +airliquide.com,airliquide.com,inbound,150,Europe,1 +alerteimmo.com,alerteimmo.com,inbound,150,Europe,0 +alinea.fr,bp06.net,inbound,150,Europe,0 +allegro.pl,allegro.pl,inbound,150,Europe,0 +allegroup.hu,allegroup.hu,inbound,150,Europe,0 +alza.cz,alza.cz,inbound,150,Europe,0.039449 +alza.sk,alza.cz,inbound,150,Europe,0.027725 +andrewchristian.com,emv8.com,inbound,150,Europe,0 +anpasia.com,anpasia.com,inbound,150,Europe,0 +aprovaconcursos.com.br,eadunicid.com.br,inbound,150,Europe,0 +aruba.it,aruba.it,inbound,150,Europe,0.055666 +ashampoo.com,ashampoo.com,inbound,150,Europe,1 +aswatson.com,emarsys.net,inbound,150,Europe,0 +avira.com,avira.com,inbound,150,Europe,0.042528 +avito.ru,avito.ru,inbound,150,Europe,0.000631 +badoo.com,monopost.com,inbound,150,Europe,1 +balsamik.fr,balsamik.fr,inbound,150,Europe,0 +bancomer.com,postini.com,inbound,150,Europe,1e-05 +bbvacompass.com,postini.com,inbound,150,Europe,0.997006 +be2.com,nmp1.net,inbound,150,Europe,0 +biglion.ru,biglion.ru,inbound,150,Europe,0.999775 +bigmailsender.com,bigmailsender.com,inbound,150,Europe,0 +bioagri.com.br,postini.com,inbound,150,Europe,0.991765 +biomedcentral.com,emv5.com,inbound,150,Europe,0 +blackberry.com,blackberry.com,inbound,150,Europe,0 +blinkboxmusic.com,mediagraft.com,inbound,150,Europe,1 +blogtrottr.com,blogtrottr.com,inbound,150,Europe,0 +blue-compass.com,blue-compass.com,inbound,150,Europe,0 +bmdeda99.com,bmdeda99.com,inbound,150,Europe,0 +bolsfr.fr,colt.net,inbound,150,Europe,0 +bonuszbrigad.hu,bonuszbrigad.hu,inbound,150,Europe,0 +boohooemail.com,smartfocusdigital.net,inbound,150,Europe,0 +booking.com,booking.com,inbound,150,Europe,1 +bouncemanager.it,musvc.com,inbound,150,Europe,0.362026 +br.com,cmailsys.com,inbound,150,Europe,0 +brandalley.com,brandalley.com,inbound,150,Europe,0 +brands4friends.de,emv5.com,inbound,150,Europe,0 +brandsvillage.net,brandsvillage.net,inbound,150,Europe,0 +bt.com,bt.com,inbound,150,Europe,0.409404 +bweeble.com,adlabsinc.com,inbound,150,Europe,0 +cabestan.com,cab07.net,inbound,150,Europe,0 +cadremploi.fr,cadremploi.fr,inbound,150,Europe,0 +cardsys.at,cardsys.at,inbound,150,Europe,1 +carmamail.com,carmamail.com,inbound,150,Europe,0 +casasbahia.com.br,casasbahia.com.br,inbound,150,Europe,0 +catchoftheday.com.au,inxserver.de,inbound,150,Europe,1 +catererglobal.com,madgexjb.com,inbound,150,Europe,0 +caterermail.com,totaljobsmail.co.uk,inbound,150,Europe,0 +cathkidston.com,cathkidston.co.uk,inbound,150,Europe,0 +cccampaigns.com,emv5.com,inbound,150,Europe,0 +cccampaigns.com,emv8.com,inbound,150,Europe,0 +cccampaigns.net,01net.com,inbound,150,Europe,0 +cccampaigns.net,cccampaigns.net,inbound,150,Europe,0 +cccampaigns.net,emv4.net,inbound,150,Europe,0 +ccmbg.com,benchmark.fr,inbound,150,Europe,0 +cdongroup.com,cdongroup.com,inbound,150,Europe,0.000147 +cheapflights.co.uk,cheapflights.co.uk,inbound,150,Europe,0 +cheapflights.com,cheapflights.com,inbound,150,Europe,0 +chelseafc.com,chelseafc.com,inbound,150,Europe,0.003566 +cinesa.es,cccampaigns.com,inbound,150,Europe,0 +cipherzone.com,infimail.com,inbound,150,Europe,0 +citybrands.hu,webinform.hu,inbound,150,Europe,1 +clickon.com.br,clickon.com.br,inbound,150,Europe,0 +clicplan.com,dmdelivery.com,inbound,150,Europe,0 +cobone.com,emarsys.net,inbound,150,Europe,0 +communicatoremail.com,communicatoremail.com,inbound,150,Europe,0 +compute.internal,amazonaws.com,inbound,150,Europe,0.81478 +confirmedoptin.com,confirmedoptin.com,inbound,150,Europe,0 +continente.pt,1-hostingservice.com,inbound,150,Europe,0 +cratusservices.in,ramcorp.in,inbound,150,Europe,0 +cricinfo.com,cricinfo.com,inbound,150,Europe,0 +critsend.com,critsend.com,inbound,150,Europe,0 +crsend.com,crsend.com,inbound,150,Europe,0.008688 +csas.cz,csas.cz,inbound,150,Europe,0.999971 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,150,Europe,1 +cv-library.co.uk,cv-library.co.uk,inbound,150,Europe,0 +cvbankas.lt,efadm.eu,inbound,150,Europe,0 +cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 +d-reizen.nl,dmdelivery.com,inbound,150,Europe,0 +dafiti.com.br,fagms.de,inbound,150,Europe,0 +datingfactory.com,caerussolutions.net,inbound,150,Europe,0 +dbgi.co.uk,emc1.co.uk,inbound,150,Europe,0 +deal.com.sg,emarsys.net,inbound,150,Europe,0 +debian.org,debian.org,inbound,150,Europe,1 +deezer.com,dms30.com,inbound,150,Europe,0 +directcrm.ru,directcrm.ru,inbound,150,Europe,0 +disney.co.uk,emv9.com,inbound,150,Europe,0 +dominosemail.co.uk,dominosemail.co.uk,inbound,150,Europe,0 +doodle.com,doodle.com,inbound,150,Europe,1 +dotmailer-email.com,dotmailer.com,inbound,150,Europe,0 +dotmailer.co.uk,dotmailer.com,inbound,150,Europe,0 +dpapp.nl,prikbordmailer.nl,inbound,150,Europe,0 +dpapp.nl,sslsecuref.nl,inbound,150,Europe,0 +dreivip.com,dreivip.com,inbound,150,Europe,0.000301 +dress-for-less.de,privalia.com,inbound,150,Europe,0 +drweb.com,drweb.com,inbound,150,Europe,0.969315 +e-boks.dk,e-boks.dk,inbound,150,Europe,1 +e-ebuyer.com,e-ebuyer.com,inbound,150,Europe,0 +e-mark.nl,e-mark.nl,inbound,150,Europe,0 +e-ngine.nl,e-ngine.nl,inbound,150,Europe,0 +ebay-kleinanzeigen.de,mobile.de,inbound,150,Europe,1 +ecommzone.com,ecommzone.com,inbound,150,Europe,0 +edarling.fr,fagms.de,inbound,150,Europe,0 +edima.hu,edima.hu,inbound,150,Europe,0 +efox-shop.com,dmdelivery.com,inbound,150,Europe,0 +ejobs.ro,ejobs.ro,inbound,150,Europe,8.5e-05 +elaine-asp.de,artegic.net,inbound,150,Europe,0.999997 +elcorteingles.es,elcorteingles.es,inbound,150,Europe,0 +elektronskaposta.si,eprvak.si,inbound,150,Europe,0 +elettershop.de,servicemail24.de,inbound,150,Europe,1 +emag.ro,emag.ro,inbound,150,Europe,0.005013 +email-comparethemarket.com,smartfocusdigital.net,inbound,150,Europe,0 +email360api.com,email360api.com,inbound,150,Europe,0 +emarsys.net,emarsys.net,inbound,150,Europe,0 +embluejet.com,emblueuser.com,inbound,150,Europe,0 +emsecure.net,emsecure.net,inbound,150,Europe,0 +emsmtp.com,emsmtp.com,inbound,150,Europe,0 +enewsletter.pl,enewsletter.pl,inbound,150,Europe,0.007349 +enewsletter.pl,sare25.com,inbound,150,Europe,0 +espmp-agfr.net,bp06.net,inbound,150,Europe,0 +esprit-friends.com,esprit-friends.com,inbound,150,Europe,0 +evanscycles.com,msgfocus.com,inbound,150,Europe,0 +experteer.com,experteer.com,inbound,150,Europe,0 +extra.com.br,emv8.com,inbound,150,Europe,0 +eyepin.com,eyepin.com,inbound,150,Europe,0 +fabfurnish.com,fagms.de,inbound,150,Europe,0 +facilisimo.com,facilisimo.com,inbound,150,Europe,1 +fagms.net,fagms.de,inbound,150,Europe,0 +finn.no,schibsted-it.no,inbound,150,Europe,0.002893 +fixeads.com,fixeads.com,inbound,150,Europe,0 +flirchi.com,flirchi.com,inbound,150,Europe,1.0 +flymonarchemail.com,flymonarchemail.com,inbound,150,Europe,0 +follow-up.se,follow-up.se,inbound,150,Europe,1 +fotocasa.es,fotocasa.es,inbound,150,Europe,0 +fotostrana.ru,fotocdn.net,inbound,150,Europe,3.4e-05 +free-lance.ru,free-lance.ru,inbound,150,Europe,0 +free.fr,free.fr,inbound,150,Europe,0.984012 +free.fr,free.fr,outbound,150,Europe,6.9e-05 +freecycle.org,freecycle.org,inbound,150,Europe,0.999865 +freemail.hu,freemail.hu,outbound,150,Europe,0 +freshmail.pl,freshmail.pl,inbound,150,Europe,0 +gardeningclubmail.co.uk,msgfocus.com,inbound,150,Europe,0 +giffgaff.com,giffgaff.com,inbound,150,Europe,0 +globasemail.com,globasemail.com,inbound,150,Europe,0.951088 +gmail.com,02.net,inbound,150,Europe,0.999617 +gmail.com,as13285.net,inbound,150,Europe,0.999943 +gmail.com,bbox.fr,inbound,150,Europe,0.919849 +gmail.com,belgacom.be,inbound,150,Europe,0.999444 +gmail.com,blackberry.com,inbound,150,Europe,0.998923 +gmail.com,bluewin.ch,inbound,150,Europe,0.93294 +gmail.com,btcentralplus.com,inbound,150,Europe,0.999969 +gmail.com,chello.nl,inbound,150,Europe,0.999951 +gmail.com,fastwebnet.it,inbound,150,Europe,0.984706 +gmail.com,jazztel.es,inbound,150,Europe,0.999701 +gmail.com,net24.it,inbound,150,Europe,0.999964 +gmail.com,netcabo.pt,inbound,150,Europe,0.998164 +gmail.com,numericable.fr,inbound,150,Europe,0.999723 +gmail.com,ono.com,inbound,150,Europe,0.995991 +gmail.com,orange.es,inbound,150,Europe,0.998742 +gmail.com,orange.fr,inbound,150,Europe,0 +gmail.com,otenet.gr,inbound,150,Europe,0.957917 +gmail.com,postini.com,inbound,150,Europe,0.924066 +gmail.com,proxad.net,inbound,150,Europe,0.998094 +gmail.com,rima-tde.net,inbound,150,Europe,0.99915 +gmail.com,sfr.net,inbound,150,Europe,0.999877 +gmail.com,skybroadband.com,inbound,150,Europe,0.999854 +gmail.com,t-ipconnect.de,inbound,150,Europe,0.999841 +gmail.com,tdc.net,inbound,150,Europe,0.999591 +gmail.com,telecomitalia.it,inbound,150,Europe,0.997 +gmail.com,telekom.hu,inbound,150,Europe,0.999977 +gmail.com,telenet.be,inbound,150,Europe,1 +gmail.com,telepac.pt,inbound,150,Europe,0.999584 +gmail.com,telia.com,inbound,150,Europe,1 +gmail.com,threembb.co.uk,inbound,150,Europe,1 +gmail.com,tpnet.pl,inbound,150,Europe,0.999761 +gmail.com,virginm.net,inbound,150,Europe,0.99661 +gmail.com,vodafone-ip.de,inbound,150,Europe,1 +gmail.com,vodafone.pt,inbound,150,Europe,0.999563 +gmail.com,vodafonedsl.it,inbound,150,Europe,0.999006 +gmail.com,wanadoo.fr,inbound,150,Europe,0.999752 +gmail.com,ziggo.nl,inbound,150,Europe,1 +gmx.de,gmx.net,inbound,150,Europe,1 +gmx.de,gmx.net,outbound,150,Europe,1 +gmx.net,gmx.net,inbound,150,Europe,1 +goalunited.org,ccmdcampaigns.net,inbound,150,Europe,0 +gog.com,gog.com,inbound,150,Europe,0 +gogroopie.com,gogroopie.com,inbound,150,Europe,0.000283 +goldenline.pl,goldenline.pl,inbound,150,Europe,1 +goodgame.com,emsmtp.com,inbound,150,Europe,0 +goodlife.pt,emv8.com,inbound,150,Europe,0 +google.com,postini.com,inbound,150,Europe,0.810567 +googlemail.com,t-ipconnect.de,inbound,150,Europe,0.999957 +gumtree.com,marktplaats.nl,inbound,150,Europe,0 +gumtree.com.au,kijiji.com,inbound,150,Europe,0 +gymglish.com,gymglish.com,inbound,150,Europe,0.000788 +haskell.org,haskell.org,inbound,150,Europe,0.000703 +hazteoir.org,hazteoir.org,inbound,150,Europe,0.31478 +hh.ru,hh.ru,inbound,150,Europe,0.996236 +hotel.de,emp-mail.de,inbound,150,Europe,0 +hotornot.com,monopost.com,inbound,150,Europe,0.983953 +hotukdeals.com,hotukdeals.com,inbound,150,Europe,1 +hpnotifier.nl,hpnotifier.nl,inbound,150,Europe,0 +icelandmail.co.uk,emsg-live.co.uk,inbound,150,Europe,0 +idealista.com,idealista.com,inbound,150,Europe,0.001627 +ideascost.com,ramcorp.in,inbound,150,Europe,0 +inboxair.com,inboxair.com,inbound,150,Europe,0 +infoempleo.com,infoempleo.com,inbound,150,Europe,0 +infojobs.it,infojobs.it,inbound,150,Europe,0 +infojobs.net,infojobs.net,inbound,150,Europe,0 +infopraca.pl,careesma.com,inbound,150,Europe,0 +ingdirect.es,ingdirect.es,inbound,150,Europe,1 +inter-chat.com,inter-chat.com,inbound,150,Europe,0 +interdatesa.com,fagms.net,inbound,150,Europe,0 +internations.org,internations.org,inbound,150,Europe,1 +inx1and1.de,1and1.com,inbound,150,Europe,1 +inxserver.com,inxserver.de,inbound,150,Europe,0.952863 +inxserver.de,inxserver.de,inbound,150,Europe,0.980838 +itms.in.ua,itms.in.ua,inbound,150,Europe,0 +jiscmail.ac.uk,lsoft.se,inbound,150,Europe,0 +jobisjob.com,jobisjob.com,inbound,150,Europe,0 +jobrapidoalert.com,jobrapidoalert.com,inbound,150,Europe,0 +jobs2web.com,ondemand.com,inbound,150,Europe,1 +jobserve.com,jobserve.com,inbound,150,Europe,0 +joobmailer.com,joobmailer.com,inbound,150,Europe,0 +jumia.com.ng,fagms.de,inbound,150,Europe,0 +justclick.ru,justclick.ru,inbound,150,Europe,0 +kalunga.com.br,kalunga.com.br,inbound,150,Europe,0 +kiabi.com,dms-02.net,inbound,150,Europe,0 +kijiji.ca,kijiji.com,inbound,150,Europe,0 +kismia.com,kismia.com,inbound,150,Europe,1 +kiwari.com,kiwari.com,inbound,150,Europe,9e-06 +kundenserver.de,kundenserver.de,inbound,150,Europe,1 +laposte.net,laposte.net,inbound,150,Europe,0.271722 +laposte.net,laposte.net,outbound,150,Europe,0 +laredoute.fr,laredoute.fr,inbound,150,Europe,0 +laterooms.com,laterooms.com,inbound,150,Europe,0.014746 +leboncoin.fr,leboncoin.fr,inbound,150,Europe,0 +leparisien.fr,leparisien.fr,inbound,150,Europe,0 +lexpress.fr,bp06.net,inbound,150,Europe,0 +libero.it,libero.it,inbound,150,Europe,0.00024 +libero.it,libero.it,outbound,150,Europe,0 +lifecooler.com,1-hostingservice.com,inbound,150,Europe,0 +listadventure.com,adlabsinc.com,inbound,150,Europe,0 +listjoe.com,adlabsinc.com,inbound,150,Europe,0 +litres.ru,litres.ru,inbound,150,Europe,1 +loccitane.com,neolane.net,inbound,150,Europe,0 +logentries.com,logentries.com,inbound,150,Europe,0 +loveplanet.ru,pochta.ru,inbound,150,Europe,0.640035 +lowcostholidays.co.uk,communicatoremail.com,inbound,150,Europe,0 +lua.org,pepperfish.net,inbound,150,Europe,1 +ludokados.com,ludokado.com,inbound,150,Europe,0 +mail-cdiscount.com,mail-cdiscount.com,inbound,150,Europe,0 +mail-mbank.pl,mail-mbank.pl,inbound,150,Europe,0 +mail.ru,mail.ru,inbound,150,Europe,0.991269 +mail.ru,mail.ru,outbound,150,Europe,0.006918 +mailer-service.de,mailer-service.de,inbound,150,Europe,4.9e-05 +mailersend.com,mailersend.com,inbound,150,Europe,0 +mailing-list.it,mailing-list.it,inbound,150,Europe,0 +mailjet.com,mailjet.com,inbound,150,Europe,0 +mailplus.nl,brightbase.net,inbound,150,Europe,1 +mailpv.net,pvmailer.net,inbound,150,Europe,1 +maisonsdumonde.com,bp06.net,inbound,150,Europe,0 +makro.nl,srv2.de,inbound,150,Europe,0.888692 +mapfre.com,emv5.com,inbound,150,Europe,0 +marktplaats.nl,marktplaats.nl,inbound,150,Europe,0 +matchwereld.nl,matchwereld.nl,inbound,150,Europe,0 +maxpark.com,gidepark.ru,inbound,150,Europe,1 +mdirector.com,mdrctr.com,inbound,150,Europe,0 +mecumauction.com,mecumauction.com,inbound,150,Europe,0 +meetic.com,meetic.com,inbound,150,Europe,0 +mequedouno.com,mequedouno.com,inbound,150,Europe,0 +metrodeal.com,fagms.de,inbound,150,Europe,0 +mightydeals.co.uk,mightydeals.co.uk,inbound,150,Europe,0 +mirtesen.ru,mtml.ru,inbound,150,Europe,0 +mitula.net,mitula.org,inbound,150,Europe,0 +mitula.org,mitula.org,inbound,150,Europe,0 +mixcloudmail.com,mixcloudmail.com,inbound,150,Europe,0.995482 +mlgns.com,mlgns.com,inbound,150,Europe,0 +mlgnserv.com,mlgnserv.com,inbound,150,Europe,0 +mmks.it,mail-maker.it,inbound,150,Europe,0 +mmsecure.nl,donenad.nl,inbound,150,Europe,0 +modnakasta.ua,emv5.com,inbound,150,Europe,0 +mooply.co,mailendo.com,inbound,150,Europe,0 +moviestarplanet.com,moviestarplanet.com,inbound,150,Europe,0 +mrc.org,msgfocus.com,inbound,150,Europe,0 +msdp1.com,msdp1.com,inbound,150,Europe,0 +msgfocus.com,msgfocus.com,inbound,150,Europe,0.011658 +mymms.com,fagms.de,inbound,150,Europe,0 +namorico.me,namorico.me,inbound,150,Europe,1 +nasza-klasa.pl,nasza-klasa.pl,inbound,150,Europe,0 +nationalexpress.com,nationalexpress.com,inbound,150,Europe,0 +neolane.net,neolane.net,inbound,150,Europe,0 +netopia.pt,netopia.pt,inbound,150,Europe,0.028429 +nhs.jobs,nhscareersjobs.co.uk,inbound,150,Europe,0 +nieuwsblad.be,vummail.be,inbound,150,Europe,0 +nmp1.com,nmp1.net,inbound,150,Europe,0 +nos.pt,netcabo.pt,inbound,150,Europe,6.3e-05 +noticiasaominuto.com,ccmdcampaigns.net,inbound,150,Europe,0 +noticiasaominuto.com,noticiasaominuto.com,inbound,150,Europe,0 +nrholding.net,nrholding.net,inbound,150,Europe,0 +odisseias.com,emv4.net,inbound,150,Europe,0 +odnoklassniki.ru,odnoklassniki.ru,inbound,150,Europe,0 +offerum.com,cccampaigns.com,inbound,150,Europe,0 +offerum.com,ccemails.com,inbound,150,Europe,0 +olx.pt,fixeads.com,inbound,150,Europe,0 +orange.fr,orange.fr,inbound,150,Europe,0 +orange.fr,orange.fr,outbound,150,Europe,0 +oroscopofree.com,adsender.us,inbound,150,Europe,0 +oroscopofree.com,oroscopofree.com,inbound,150,Europe,0 +orsay.com,emp-mail.de,inbound,150,Europe,0 +ovh.net,ovh.net,inbound,150,Europe,0.165188 +oxfam.org.uk,msgfocus.com,inbound,150,Europe,0 +payback.de,artegic.net,inbound,150,Europe,0.999986 +pccomponentes.com,pccomponentes.com,inbound,150,Europe,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,150,Europe,0 +peperoni.de,peperoni.de,inbound,150,Europe,1 +peytz.dk,peytz.dk,inbound,150,Europe,4e-06 +photobox.com,photobox.com,inbound,150,Europe,0 +photoprintit.com,photoprintit.com,inbound,150,Europe,0 +pixum.com,pixum.com,inbound,150,Europe,0 +placedestendances.com,placedestendances.com,inbound,150,Europe,0 +plaisio.gr,fagms.de,inbound,150,Europe,0 +planeo.com,planeo.com,inbound,150,Europe,0 +planeo.pt,planeo.pt,inbound,150,Europe,0 +playtika.com,emv8.com,inbound,150,Europe,0 +poinx.com,poinx.com,inbound,150,Europe,0 +pokerstars.com,pokerstars.eu,inbound,150,Europe,0 +pokerstars.eu,pokerstars.eu,inbound,150,Europe,0 +pontofrio.com.br,emv8.com,inbound,150,Europe,0 +postgresql.org,postgresql.org,inbound,150,Europe,1 +praca.pl,praca.pl,inbound,150,Europe,1 +pracuj.pl,pracuj.pl,inbound,150,Europe,0 +printvenue.com,fagms.de,inbound,150,Europe,0 +privalia.com,privalia.com,inbound,150,Europe,3.94913202027326e-07 +promod-news.fr,promod-news.com,inbound,150,Europe,0 +protopmail.com,protopmail.com,inbound,150,Europe,0 +pur3.net,pur3.net,inbound,150,Europe,0.001123 +python.org,python.org,inbound,150,Europe,1 +quality.net.ua,quality.net.ua,inbound,150,Europe,0 +r-project.org,ethz.ch,inbound,150,Europe,0.99999 +r51.it,musvc.com,inbound,150,Europe,0.092498 +r52.it,musvc.com,inbound,150,Europe,0.000221 +r57.it,musvc.com,inbound,150,Europe,0.139425 +r67.it,musvc.com,inbound,150,Europe,0.199845 +r70.it,musvc.com,inbound,150,Europe,0.311983 +rakuten.co.jp,shareee.jp,inbound,150,Europe,0 +rambler.ru,rambler.ru,inbound,150,Europe,0.08173 +ratedpeople.com,ratedpeople.com,inbound,150,Europe,0.000979 +rax.ru,rax.ru,inbound,150,Europe,0.000258 +regie11.net,odiso.net,inbound,150,Europe,0 +relax7.hu,gruppi.hu,inbound,150,Europe,0 +richersoundsvip.com,ibwmail.com,inbound,150,Europe,0 +rightmove.com,rightmove.com,inbound,150,Europe,0 +roulartamail.be,roulartamail.be,inbound,150,Europe,0 +rueducommerce.com,groupe-rueducommerce.fr,inbound,150,Europe,0 +runtastic.com,runtastic.com,inbound,150,Europe,0 +ryanairmail.com,ryanairmail.com,inbound,150,Europe,0 +rzone.de,rzone.de,inbound,150,Europe,1 +saimails.in,infimail.com,inbound,150,Europe,0 +sainsburys.co.uk,emv5.com,inbound,150,Europe,0 +salesmanago.pl,salesmanago.pl,inbound,150,Europe,0 +samsung.ru,samsung.ru,inbound,150,Europe,0 +sapnetworkmail.com,sap-ag.de,inbound,150,Europe,1 +sapo.pt,sapo.pt,inbound,150,Europe,0.242839 +sapo.pt,sapo.pt,outbound,150,Europe,0 +savingdeals.in,infimail.com,inbound,150,Europe,0 +scmp.com,emarsys.net,inbound,150,Europe,0 +scoop.it,scoop.it,inbound,150,Europe,0 +scoopon.com.au,inxserver.de,inbound,150,Europe,1 +screwfix.info,fwdto.net,inbound,150,Europe,0 +secureserver.net,secureserver.net,inbound,150,Europe,0 +selection-priceminister.com,selection-priceminister.com,inbound,150,Europe,0 +sender.lt,sritis.lt,inbound,150,Europe,0.000339 +sendsmaily.info,sendsmaily.info,inbound,150,Europe,0 +seniorplanet.fr,seniorplanet.fr,inbound,150,Europe,0 +seznam.cz,seznam.cz,inbound,150,Europe,0.001781 +seznam.cz,seznam.cz,outbound,150,Europe,0.001741 +sfr.fr,sfr.fr,inbound,150,Europe,0.236539 +sfr.fr,sfr.fr,outbound,150,Europe,0.004753 +shopto.net,shopto.net,inbound,150,Europe,1 +skype.com,skype.com,inbound,150,Europe,0 +solveerrors.com,infimail.com,inbound,150,Europe,0 +spareroom.co.uk,spareroom.co.uk,inbound,150,Europe,1 +sqlservercentral.com,sqlservercentral.com,inbound,150,Europe,0 +staples-pt.com,1-hostingservice.com,inbound,150,Europe,0 +stepstone.de,stepstone.com,inbound,150,Europe,0.001689 +studentbeans.com,emv8.com,inbound,150,Europe,0 +subito.it,subito.it,inbound,150,Europe,0 +subscribe.ru,subscribe.ru,inbound,150,Europe,0 +superdrug.com,superdrug.com,inbound,150,Europe,0 +superjob.ru,superjob.ru,inbound,150,Europe,0 +support-love.com,support-love.com,inbound,150,Europe,0 +sut1.co.uk,sut1.co.uk,inbound,150,Europe,0.001789 +sut5.co.uk,sut5.co.uk,inbound,150,Europe,0 +swanson-vitamins.com,emv5.com,inbound,150,Europe,0 +t-online.de,t-online.de,inbound,150,Europe,1 +t-online.de,t-online.de,outbound,150,Europe,0.999939 +tatrabanka.sk,tatrabanka.sk,inbound,150,Europe,0 +tchibo.de,srv2.de,inbound,150,Europe,0.980447 +teamo.ru,teamo.ru,inbound,150,Europe,0 +teamviewer.com,teamviewer.com,inbound,150,Europe,0 +teapartyinfo.org,teapartyinfo.org,inbound,150,Europe,0 +telenet.be,telenet-ops.be,inbound,150,Europe,1.5e-05 +teleportmyjob.com,clara.net,inbound,150,Europe,0 +theleadmagnet.com,your-server.de,inbound,150,Europe,1 +timeweb.ru,timeweb.ru,inbound,150,Europe,1 +tiscali.it,tiscali.it,outbound,150,Europe,0 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 +transversal.net,transversal.net,inbound,150,Europe,1 +travisperkins.co.uk,travisperkins.co.uk,inbound,150,Europe,0 +trovit.com,trovit.com,inbound,150,Europe,0 +tucasa.com,grupodtm.com,inbound,150,Europe,0 +twoomail.com,twoomail.com,inbound,150,Europe,0 +ubuntu.com,canonical.com,inbound,150,Europe,0 +ukr.net,fwdcdn.com,inbound,150,Europe,1 +ukr.net,ukr.net,outbound,150,Europe,0.999992 +ulteem.com,ulteem.com,inbound,150,Europe,0 +usndr.com,usndr.com,inbound,150,Europe,1 +venere.com,kiwari.com,inbound,150,Europe,0 +venteprivee.com,venteprivee.com,inbound,150,Europe,0 +virgilio.it,virgilio.net,inbound,150,Europe,0 +visualsoft.co.uk,visualsoft.co.uk,inbound,150,Europe,0 +vouchercloud.com,vouchercloud.com,inbound,150,Europe,0 +voyageprive.com,cccampaigns.net,inbound,150,Europe,0 +voyageprive.es,ccmdcampaigns.net,inbound,150,Europe,0 +voyageprive.it,ccmdcampaigns.net,inbound,150,Europe,0 +voyages-sncf.com,neolane.net,inbound,150,Europe,0 +vueling.com,vueling.com,inbound,150,Europe,0 +wanadoo.fr,orange.fr,inbound,150,Europe,0 +wanadoo.fr,orange.fr,outbound,150,Europe,0 +waves-audio.com,emv8.com,inbound,150,Europe,0 +web.de,web.de,inbound,150,Europe,0.999986 +web.de,web.de,outbound,150,Europe,1 +whereareyounow.com,wayn.net,inbound,150,Europe,0 +wiggle.com,wiggle.com,inbound,150,Europe,0 +william-reed.com,neolane.net,inbound,150,Europe,0 +williamhill.com,williamhill.com,inbound,150,Europe,0 +wldemail.com,emarsys.net,inbound,150,Europe,0 +wmtransfer.com,wmtransfer.com,inbound,150,Europe,0 +wnd.com,emv4.net,inbound,150,Europe,0 +wnd.com,worldnetdaily.com,inbound,150,Europe,0 +work.ua,work.ua,inbound,150,Europe,1 +workcircle.com,workcircle.net,inbound,150,Europe,0 +wp.pl,wp.pl,inbound,150,Europe,0.998027 +wp.pl,wp.pl,outbound,150,Europe,1 +xmailix.com,xmailix.com,inbound,150,Europe,0 +yandex.ru,yandex.net,inbound,150,Europe,0.999658 +yandex.ru,yandex.ru,outbound,150,Europe,1 +ymlpserver.net,ymlpserver.net,inbound,150,Europe,0 +ymlpsrv.net,ymlpsrv.net,inbound,150,Europe,0 +zalando.be,fagms.de,inbound,150,Europe,0 +zalando.dk,fagms.de,inbound,150,Europe,0 +zalando.fi,fagms.de,inbound,150,Europe,0 +zalando.it,fagms.de,inbound,150,Europe,0 +zalando.nl,fagms.de,inbound,150,Europe,0 +zalando.pl,fagms.de,inbound,150,Europe,0 +zumzi.com,neogen.ro,inbound,150,Europe,0 +zumzi.com,zumzi.com,inbound,150,Europe,0 +0bz.biz,hmts.jp,inbound,ZZ,Unknown Region,0 +104.com.tw,104.com.tw,inbound,ZZ,Unknown Region,0.091133 +1105info.com,1105info.com,inbound,ZZ,Unknown Region,0 +1111.com.tw,1111.com.tw,inbound,ZZ,Unknown Region,0 +160by2inbox.com,160by2inbox.com,inbound,ZZ,Unknown Region,0 +160by2invite.com,160by2invite.com,inbound,ZZ,Unknown Region,0 +160by2mail.com,160by2mail.com,inbound,ZZ,Unknown Region,0 +163.com,163.com,inbound,ZZ,Unknown Region,0.996685 +1800flowersinc.com,1800flowersinc.com,inbound,ZZ,Unknown Region,0 +1sale.com,1sale.com,inbound,ZZ,Unknown Region,0 +1v1y.com,euromsg.net,inbound,ZZ,Unknown Region,0 +4shared.com,4shared.com,inbound,ZZ,Unknown Region,1 +6pm.com,6pm.com,inbound,ZZ,Unknown Region,1 +6pm.com,zappos.com,inbound,ZZ,Unknown Region,0.782581 +a8.net,a8.net,inbound,ZZ,Unknown Region,0 +aaa.com,nextjump.com,inbound,ZZ,Unknown Region,0 +aaas-science.org,aaas-science.org,inbound,ZZ,Unknown Region,0 +aarp.org,aarp.org,inbound,ZZ,Unknown Region,0 +about.com,about.com,inbound,ZZ,Unknown Region,8.2e-05 +academy-enews.com,academy-enews.com,inbound,ZZ,Unknown Region,0 +accenture.com,outlook.com,inbound,ZZ,Unknown Region,1 +accountonline.com,accountonline.com,inbound,ZZ,Unknown Region,0.348991 +acehelpfulemails.com,teradatadmc.com,inbound,ZZ,Unknown Region,0 +actorsaccess.com,nonfatmedia.com,inbound,ZZ,Unknown Region,0 +adminforfree.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +adminforfree.net,adminforfree.com,inbound,ZZ,Unknown Region,1 +administrativejobinsider.com,administrativejobinsider.com,inbound,ZZ,Unknown Region,0 +adobe.com,obsmtp.com,inbound,ZZ,Unknown Region,1 +adobesystems.com,adobesystems.com,inbound,ZZ,Unknown Region,0 +adoreme.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +adp.com,adp.com,inbound,ZZ,Unknown Region,1 +adsender.us,adsender.us,inbound,ZZ,Unknown Region,0 +adsolutionline.com,adsolutionline.com,inbound,ZZ,Unknown Region,0 +adultfriendfinder.com,friendfinder.com,inbound,ZZ,Unknown Region,0 +advanceauto.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,ZZ,Unknown Region,0 +af.mil,af.mil,inbound,ZZ,Unknown Region,0.997561 +agoda-emails.com,agoda-emails.com,inbound,ZZ,Unknown Region,0 +agrupemonos.cl,agrupemonos.cl,inbound,ZZ,Unknown Region,1 +albertsonsemail.com,email4-mywebgrocer.com,inbound,ZZ,Unknown Region,0 +alibaba.com,alibaba.com,inbound,ZZ,Unknown Region,0 +alice.it,alice.it,inbound,ZZ,Unknown Region,0 +alice.it,aliceposta.it,outbound,ZZ,Unknown Region,0 +aliexpress.com,alibaba.com,inbound,ZZ,Unknown Region,0 +allegro.pl,allegro.pl,inbound,ZZ,Unknown Region,0 +allegrogroup.ua,allegrogroup.ua,inbound,ZZ,Unknown Region,0 +allsaints.com,allsaints.com,inbound,ZZ,Unknown Region,0 +ama-assn.org,elabs10.com,inbound,ZZ,Unknown Region,0 +amadeus.com,amadeus.net,inbound,ZZ,Unknown Region,0 +amazon.{...},amazon.{...},inbound,ZZ,Unknown Region,0.021685 +amazon.{...},amazonses.com,inbound,ZZ,Unknown Region,0.999971 +amazonses.com,amazonses.com,inbound,ZZ,Unknown Region,0.997919 +amazonses.com,postini.com,inbound,ZZ,Unknown Region,0.924067 +amctheatres.com,amctheatres.com,inbound,ZZ,Unknown Region,0 +americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,ZZ,Unknown Region,0 +ancestry.com,ancestry.com,inbound,ZZ,Unknown Region,0 +angieslist.com,angieslist.com,inbound,ZZ,Unknown Region,0 +anntaylor.com,anntaylor.com,inbound,ZZ,Unknown Region,0 +anpdm.com,anpdm.com,inbound,ZZ,Unknown Region,4e-06 +aol.com,aol.com,outbound,ZZ,Unknown Region,1 +apple.com,apple.com,inbound,ZZ,Unknown Region,0.915839 +argos.co.uk,argos.co.uk,inbound,ZZ,Unknown Region,1 +argos.co.uk,exacttarget.com,inbound,ZZ,Unknown Region,1 +artists-hub.com,artists-hub.com,inbound,ZZ,Unknown Region,0 +asda.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +ask.fm,ask.fm,inbound,ZZ,Unknown Region,1e-05 +askmen.com,askmen.com,inbound,ZZ,Unknown Region,0 +astrocenter.com,center.com,inbound,ZZ,Unknown Region,0 +athleta.com,athleta.com,inbound,ZZ,Unknown Region,0 +atlassian.net,uc-inf.net,inbound,ZZ,Unknown Region,1 +att.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999956 +auctionzip-email.com,email-auctionholdings.com,inbound,ZZ,Unknown Region,0 +auinmeio.com.br,fnac.com.br,inbound,ZZ,Unknown Region,0 +authorize.net,authorize.net,inbound,ZZ,Unknown Region,0 +authorize.net,visa.com,inbound,ZZ,Unknown Region,0.993555 +autoreply.com,autoreply.com,inbound,ZZ,Unknown Region,0 +avomail.com,avomail.com,inbound,ZZ,Unknown Region,0 +avon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 +avon.com,postdirect.com,inbound,ZZ,Unknown Region,0 +aweber.com,aweber.com,inbound,ZZ,Unknown Region,3e-06 +ayi.com,ayi.com,inbound,ZZ,Unknown Region,0 +backcountry.com,backcountry.com,inbound,ZZ,Unknown Region,0 +backlog.jp,backlog.jp,inbound,ZZ,Unknown Region,0 +bagitgetitmailer.in,emce2.in,inbound,ZZ,Unknown Region,0 +banamex.com,citi.com,inbound,ZZ,Unknown Region,0.999958 +bananarepublic.com,bananarepublic.com,inbound,ZZ,Unknown Region,3.48869418874254e-07 +bancochile.cl,bancochile.cl,inbound,ZZ,Unknown Region,0.999504 +bancofalabella.com,bancofalabella.com,inbound,ZZ,Unknown Region,0 +banesco.com,banesco.com,inbound,ZZ,Unknown Region,0 +bankofamerica.com,bankofamerica.com,inbound,ZZ,Unknown Region,0.971142 +banorte.com,gfnorte.com.mx,inbound,ZZ,Unknown Region,0.994999 +barclaycardus.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +basecamp.com,basecamp.com,inbound,ZZ,Unknown Region,1 +basecamphq.com,basecamphq.com,inbound,ZZ,Unknown Region,1 +baskinrobbins.com,baskinrobbins.com,inbound,ZZ,Unknown Region,0 +bazarchic-invitations.com,bazarchic-emstech.com,inbound,ZZ,Unknown Region,0 +bcbg.com,bcbg.com,inbound,ZZ,Unknown Region,0 +beatport-email.com,beatport-email.com,inbound,ZZ,Unknown Region,0 +beautylish.com,beautylish.com,inbound,ZZ,Unknown Region,1 +bebe.com,ed10.com,inbound,ZZ,Unknown Region,0 +befrugal.com,befrugal.com,inbound,ZZ,Unknown Region,0 +belkemail.com,belkemail.com,inbound,ZZ,Unknown Region,0 +bellsouth.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999951 +benihana-news.com,benihana-news.com,inbound,ZZ,Unknown Region,0 +bestbuy.com,bestbuy.com,inbound,ZZ,Unknown Region,0.003289 +beta.lt,mailersend3.com,inbound,ZZ,Unknown Region,0 +beyondtherack.com,beyondtherack.com,inbound,ZZ,Unknown Region,0 +bigfishgames.com,bigfishgames.com,inbound,ZZ,Unknown Region,0 +biglots.com,biglots.com,inbound,ZZ,Unknown Region,0 +bjsrestaurants.com,bjsrestaurants.com,inbound,ZZ,Unknown Region,0 +blackberry.com,blackberry.com,inbound,ZZ,Unknown Region,0 +blackboard.com,notification.com,inbound,ZZ,Unknown Region,0 +blackpeoplemeet.com,blackpeoplemeet.com,inbound,ZZ,Unknown Region,0 +bloglovin.com,bloglovin.com,inbound,ZZ,Unknown Region,0 +blogtrottr.com,blogtrottr.com,inbound,ZZ,Unknown Region,0 +bloomberg.com,bloomberg.com,inbound,ZZ,Unknown Region,0.001463 +bloomberg.net,bloomberg.net,inbound,ZZ,Unknown Region,1 +bluediamondhost3.com,web-hosting.com,inbound,ZZ,Unknown Region,1 +bluehornet.com,bluehornet.com,inbound,ZZ,Unknown Region,0 +bluenile.com,bluenile.com,inbound,ZZ,Unknown Region,0 +bluestatedigital.com,bluestatedigital.com,inbound,ZZ,Unknown Region,0 +bm05.net,bm05.net,inbound,ZZ,Unknown Region,0 +bmnt.jp,bmnt.jp,inbound,ZZ,Unknown Region,0 +bmsend.com,bmsend.com,inbound,ZZ,Unknown Region,0 +bn.com,bn.com,inbound,ZZ,Unknown Region,0 +bncollegemail.com,bncollegemail.com,inbound,ZZ,Unknown Region,0 +booking.com,booking.com,inbound,ZZ,Unknown Region,1 +bookmyshow.com,eccluster.com,inbound,ZZ,Unknown Region,0 +boots.com,boots.com,inbound,ZZ,Unknown Region,0 +box.com,box.com,inbound,ZZ,Unknown Region,0.955607 +brierleycrm.com,brierleycrm.com,inbound,ZZ,Unknown Region,0 +brooksbrothers.com,brooksbrothers.com,inbound,ZZ,Unknown Region,0 +btinternet.com,cpcloud.co.uk,inbound,ZZ,Unknown Region,0 +btinternet.com,cpcloud.co.uk,outbound,ZZ,Unknown Region,0 +btinternet.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +budgettravel.com,email-budgettravel.com,inbound,ZZ,Unknown Region,0 +buyinvite.com.au,buyinvite.com.au,inbound,ZZ,Unknown Region,0 +bv.com.br,bv.com.br,inbound,ZZ,Unknown Region,0 +byway.it,byway.it,inbound,ZZ,Unknown Region,0 +cabelas.com,cabelas.com,inbound,ZZ,Unknown Region,0 +californiajobdepartment.com,californiajobdepartment.com,inbound,ZZ,Unknown Region,0 +callcommand.com,callcommand.com,inbound,ZZ,Unknown Region,0 +calottery.com,calottery.com,inbound,ZZ,Unknown Region,1 +canadiantire.ca,canadiantire.ca,inbound,ZZ,Unknown Region,0 +canalplus.es,canalplus.es,inbound,ZZ,Unknown Region,0 +capitalone.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +capitaloneemail.com,capitaloneemail.com,inbound,ZZ,Unknown Region,0 +career-hub.net,career-hub.net,inbound,ZZ,Unknown Region,1 +careerbuilder-email.com,careerbuilder-email.com,inbound,ZZ,Unknown Region,0 +careerbuilder.com,careerbuilder.com,inbound,ZZ,Unknown Region,0 +careerflash.net,careerflash.net,inbound,ZZ,Unknown Region,1 +carrefour.fr,carrefour.fr,inbound,ZZ,Unknown Region,0 +carters.com,carters.com,inbound,ZZ,Unknown Region,2e-06 +caseyresearch.com,caseyresearch.com,inbound,ZZ,Unknown Region,1 +catchyfreebies.net,mmsend53.com,inbound,ZZ,Unknown Region,0 +cccampaigns.net,emv9.net,inbound,ZZ,Unknown Region,0 +cecentertainment.com,cecentertainment.com,inbound,ZZ,Unknown Region,0 +celebritycruises.com,celebritycruises.com,inbound,ZZ,Unknown Region,0 +centaur.co.uk,centaur.co.uk,inbound,ZZ,Unknown Region,0 +centauro.com.br,centauro.com.br,inbound,ZZ,Unknown Region,0 +centerparcs.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +cfmailer.com,elabs11.com,inbound,ZZ,Unknown Region,0 +change.org,change.org,inbound,ZZ,Unknown Region,1 +channel4.com,channel4.com,inbound,ZZ,Unknown Region,0 +chase.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +cheaperthandirt.com,cheaperthandirt.com,inbound,ZZ,Unknown Region,0 +chefscatalog.com,chefscatalog.com,inbound,ZZ,Unknown Region,0 +chemistdirect.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +chemistry.com,chemistry.com,inbound,ZZ,Unknown Region,0 +chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,ZZ,Unknown Region,0 +chopra.com,chopra.com,inbound,ZZ,Unknown Region,1 +christianmingle.com,postdirect.com,inbound,ZZ,Unknown Region,0 +cir.ca,cir.ca,inbound,ZZ,Unknown Region,1 +citi.com,citi.com,inbound,ZZ,Unknown Region,0.999941 +citibank.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +citibank.com,citi.com,inbound,ZZ,Unknown Region,0.999997 +citicorp.com,citi.com,inbound,ZZ,Unknown Region,0.999999 +citruslane.com,citruslane.com,inbound,ZZ,Unknown Region,5e-06 +clarisonic.com,clarisonic.com,inbound,ZZ,Unknown Region,0 +classmates.com,classmates.com,inbound,ZZ,Unknown Region,0 +clickon.com.ar,clickon.com.ar,inbound,ZZ,Unknown Region,0 +cmail1.com,createsend.com,inbound,ZZ,Unknown Region,0 +cmail2.com,createsend.com,inbound,ZZ,Unknown Region,0 +cmrfalabella.com,cmrfalabella.com,inbound,ZZ,Unknown Region,0 +codebreak.info,codebreak.info,inbound,ZZ,Unknown Region,1 +comcast.net,comcast.net,outbound,ZZ,Unknown Region,1 +confirmsignup.com,mmsend53.com,inbound,ZZ,Unknown Region,0 +constantcontact.com,postini.com,inbound,ZZ,Unknown Region,0.139551 +constantcontact.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999541 +contact-darty.com,mm-send.com,inbound,ZZ,Unknown Region,0 +contactlab.it,contactlab.it,inbound,ZZ,Unknown Region,0 +converse.com,converse.com,inbound,ZZ,Unknown Region,1 +cookingchanneltv.com,cookingchanneltv.com,inbound,ZZ,Unknown Region,0 +copernica.nl,picsrv.net,inbound,ZZ,Unknown Region,0.011507 +copernica.nl,vicinity.nl,inbound,ZZ,Unknown Region,0.011753 +coppel.com,coppel.com,inbound,ZZ,Unknown Region,0 +corporateperks.com,nextjump.com,inbound,ZZ,Unknown Region,0 +countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,ZZ,Unknown Region,0 +coupondunia.in,coupondunia.in,inbound,ZZ,Unknown Region,1 +crabtree-evelyn.com,crabtree-evelyn.com,inbound,ZZ,Unknown Region,0 +crainnewsalerts.com,crainnewsalerts.com,inbound,ZZ,Unknown Region,0 +crashlytics.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +cricut.com,elabs12.com,inbound,ZZ,Unknown Region,0 +criticalimpactinc.com,criticalimpactinc.com,inbound,ZZ,Unknown Region,0 +critsend.com,critsend.com,inbound,ZZ,Unknown Region,0 +crocs-email.com,crocs-email.com,inbound,ZZ,Unknown Region,0 +crunchyroll.com,crunchyroll.com,inbound,ZZ,Unknown Region,0 +cudo.com.au,exacttarget.com,inbound,ZZ,Unknown Region,0 +cuenote.jp,cuenote.jp,inbound,ZZ,Unknown Region,0 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,ZZ,Unknown Region,1 +cuppon.pl,cuppon.pl,inbound,ZZ,Unknown Region,0 +curriculum.com.br,curriculum.com.br,inbound,ZZ,Unknown Region,0 +currys.co.uk,currys.co.uk,inbound,ZZ,Unknown Region,0 +custom-emailing.com,elabs12.com,inbound,ZZ,Unknown Region,0 +cvent-planner.com,cvent-planner.com,inbound,ZZ,Unknown Region,0 +cyberdiet.com.br,allinmedia.com.br,inbound,ZZ,Unknown Region,0 +dabmail.com,iaires.com,inbound,ZZ,Unknown Region,0 +dailyom.com,dailyom.com,inbound,ZZ,Unknown Region,1 +dairyqueen.com,dairyqueen.com,inbound,ZZ,Unknown Region,0 +datadrivenemail.com,datadrivenemail.com,inbound,ZZ,Unknown Region,0 +daveramsey.com,daveramsey.com,inbound,ZZ,Unknown Region,0 +ddc-emails.com,ddc-emails.com,inbound,ZZ,Unknown Region,0 +dealchicken.com,dealchicken.com,inbound,ZZ,Unknown Region,0 +dealchicken.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +dealfind.com,dealfind.com,inbound,ZZ,Unknown Region,0 +dealnews.com,dealnews.com,inbound,ZZ,Unknown Region,0 +dealsdirect.com.au,dealsdirect.com.au,inbound,ZZ,Unknown Region,0 +dealspl.us,dealspl.us,inbound,ZZ,Unknown Region,0 +delta.com,delta.com,inbound,ZZ,Unknown Region,0.040177 +dermstore.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +dhl.com,dhl.com,inbound,ZZ,Unknown Region,0.994107 +dietaesaude.com.br,dietaesaude.com.br,inbound,ZZ,Unknown Region,0 +dinda.com.br,dinda.com.br,inbound,ZZ,Unknown Region,0 +directvla.com,directvla.com,inbound,ZZ,Unknown Region,0 +discover.com,discover.com,inbound,ZZ,Unknown Region,0 +disneydestinations.com,disneyparks.com,inbound,ZZ,Unknown Region,0 +disneydestinations.com,disneyworld.com,inbound,ZZ,Unknown Region,0 +diynetwork.com,diynetwork.com,inbound,ZZ,Unknown Region,0 +doctoroz.com,email-sharecare2.com,inbound,ZZ,Unknown Region,0 +dollartree.com,email-dollartree.com,inbound,ZZ,Unknown Region,0 +dominos.com,dominos.com,inbound,ZZ,Unknown Region,0.011861 +dominos.com.au,dominos.com.au,inbound,ZZ,Unknown Region,0 +donuts.ne.jp,dnuts.jp,inbound,ZZ,Unknown Region,0 +dotz.com.br,dotz.com.br,inbound,ZZ,Unknown Region,0 +doubletakeoffers.com,doubletakeoffers.com,inbound,ZZ,Unknown Region,0 +downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 +dptagent.net,dptagent.net,inbound,ZZ,Unknown Region,0 +draftkings.com,draftkings.com,inbound,ZZ,Unknown Region,0 +dreamhost.com,dreamhost.com,inbound,ZZ,Unknown Region,0 +driftem.com,emce2.in,inbound,ZZ,Unknown Region,0 +drjays-mail.com,drjays-mail.com,inbound,ZZ,Unknown Region,0 +dromadaire-news.com,ecmcluster.com,inbound,ZZ,Unknown Region,0 +dukecareers.com,dukecareers.com,inbound,ZZ,Unknown Region,1 +duluthtradingemail.com,email-duluthtrading.com,inbound,ZZ,Unknown Region,0 +dynect-mailer.net,dynect.net,inbound,ZZ,Unknown Region,0 +dynect-mailer.net,sendlabs.com,inbound,ZZ,Unknown Region,0 +e-bodyc.com,email-bodycentral.com,inbound,ZZ,Unknown Region,0 +e-jobs-ville.com,e-jobs-ville.com,inbound,ZZ,Unknown Region,1 +e-leclerc.com,e-leclerc.com,inbound,ZZ,Unknown Region,0 +easycanvasprints.com,easycanvasprints.com,inbound,ZZ,Unknown Region,0 +easyhits4u.com,easyhits4u.com,inbound,ZZ,Unknown Region,1 +easyhits4u.com,relmax.net,inbound,ZZ,Unknown Region,0 +ebags.com,ebags.com,inbound,ZZ,Unknown Region,0 +ebay-kleinanzeigen.de,mobile.de,inbound,ZZ,Unknown Region,1 +ebay.{...},ebay.{...},inbound,ZZ,Unknown Region,0.999648 +ebay.{...},emarsys.net,inbound,ZZ,Unknown Region,0 +ebay.{...},postdirect.com,inbound,ZZ,Unknown Region,0 +ebizac3.com,ebizac3.com,inbound,ZZ,Unknown Region,0 +ebuildabear.com,ebuildabear.com,inbound,ZZ,Unknown Region,8e-06 +ed10.net,ed10.com,inbound,ZZ,Unknown Region,0 +eddiebauer.com,eddiebauer.com,inbound,ZZ,Unknown Region,0 +eduk.com,eduk.com,inbound,ZZ,Unknown Region,0 +efamilydollar.com,efamilydollar.com,inbound,ZZ,Unknown Region,0 +elabs10.com,elabs10.com,inbound,ZZ,Unknown Region,0 +elabs12.com,elabs12.com,inbound,ZZ,Unknown Region,0 +elistas.net,elistas.net,inbound,ZZ,Unknown Region,0 +elkjop.no,ec-cluster.com,inbound,ZZ,Unknown Region,0 +elkjop.no,eccluster.com,inbound,ZZ,Unknown Region,0 +elo7.com.br,elo7.com.br,inbound,ZZ,Unknown Region,0 +email-1800contacts.com,email-1800contacts.com,inbound,ZZ,Unknown Region,0 +email-aaa.com,email-aaa.com,inbound,ZZ,Unknown Region,0 +email-aeriagames.com,email-aeriagames.com,inbound,ZZ,Unknown Region,0 +email-dressbarn.com,email-dressbarn.com,inbound,ZZ,Unknown Region,0 +email-firestone.com,reminder-firestone.com,inbound,ZZ,Unknown Region,0 +email-honest.com,email-honest.com,inbound,ZZ,Unknown Region,0 +email-od.com,email-od.com,inbound,ZZ,Unknown Region,0.999752 +email-petsmart.com,email-petsmart.com,inbound,ZZ,Unknown Region,0 +email-sportchalet.com,email-sportchalet.com,inbound,ZZ,Unknown Region,0 +email-telekom.de,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +email-totalwine.com,email-totalwine.com,inbound,ZZ,Unknown Region,0 +email-wildstar-online.com,email-carbine.com,inbound,ZZ,Unknown Region,0 +email2-beyond.com,messagebus.com,inbound,ZZ,Unknown Region,0 +email360api.com,email360api.com,inbound,ZZ,Unknown Region,0 +email365inc.com,email365inc.com,inbound,ZZ,Unknown Region,0 +email3m.com,email3m.com,inbound,ZZ,Unknown Region,0 +emailrestaurant.com,emailrestaurant.com,inbound,ZZ,Unknown Region,0 +emailtoryburch.com,emailtoryburch.com,inbound,ZZ,Unknown Region,0 +emarsys.net,emarsys.net,inbound,ZZ,Unknown Region,0.265197 +embarqmail.com,centurylink.net,inbound,ZZ,Unknown Region,0.999918 +emergencyemail.org,emergencyemail.org,inbound,ZZ,Unknown Region,0 +eminentinc.com,eminentinc.com,inbound,ZZ,Unknown Region,0 +emktviajarbarato.com.br,splio.com.br,inbound,ZZ,Unknown Region,0.875902 +employboard.com,employboard.com,inbound,ZZ,Unknown Region,1 +emsecure.net,emsecure.net,inbound,ZZ,Unknown Region,0 +emsmtp.com,emsmtp.com,inbound,ZZ,Unknown Region,0.44284 +en25.com,kqed.org,inbound,ZZ,Unknown Region,0 +enewscartes.net,bp06.net,inbound,ZZ,Unknown Region,0 +enewsletter.pl,mydeal.pl,inbound,ZZ,Unknown Region,0 +enewsletter.pl,sare25.com,inbound,ZZ,Unknown Region,0 +enplenitud.com,enplenitud.com,inbound,ZZ,Unknown Region,0 +entrepreneur.com,entrepreneur.com,inbound,ZZ,Unknown Region,0 +ethingsremembered.com,ethingsremembered.com,inbound,ZZ,Unknown Region,0 +etrade.com,etrade.com,inbound,ZZ,Unknown Region,0 +etransmail.com,etransmail.com,inbound,ZZ,Unknown Region,0 +etransmail.com,ptransmail.com,inbound,ZZ,Unknown Region,0 +etrmailbox.com,etrmailbox.com,inbound,ZZ,Unknown Region,0 +etsy.com,etsy.com,inbound,ZZ,Unknown Region,0.020849 +evernote.com,evernote.com,inbound,ZZ,Unknown Region,1 +everydayfamily.com,everydayfamily.com,inbound,ZZ,Unknown Region,0 +everytown.org,everytown.org,inbound,ZZ,Unknown Region,1 +exacttarget.com,bazaarvoice.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,booksamillion.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,exacttarget.com,inbound,ZZ,Unknown Region,0.000325 +exacttarget.com,msg.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,redboxinstant.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,skylinetechnologies.com,inbound,ZZ,Unknown Region,0 +expediamail.com,airasiago.com,inbound,ZZ,Unknown Region,1 +expediamail.com,exacttarget.com,inbound,ZZ,Unknown Region,1 +expediamail.com,expediamail.com,inbound,ZZ,Unknown Region,0.839662 +expediamail.com,quotitmail.com,inbound,ZZ,Unknown Region,0 +experteer.com,experteer.com,inbound,ZZ,Unknown Region,0 +express.com,expressfashion.com,inbound,ZZ,Unknown Region,0 +facebook.com,facebook.com,inbound,ZZ,Unknown Region,0 +facebook.com,facebook.com,outbound,ZZ,Unknown Region,1 +facebookmail.com,postini.com,inbound,ZZ,Unknown Region,0.918705 +facebookmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999846 +falabella.com,falabella.com,inbound,ZZ,Unknown Region,0 +fanatics.com,fanatics.com,inbound,ZZ,Unknown Region,0 +fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,ZZ,Unknown Region,0 +fanbridge.com,fanbridge.com,inbound,ZZ,Unknown Region,0 +fanofannas.com,fanofannas.com,inbound,ZZ,Unknown Region,0 +fansedge.com,fansedge.com,inbound,ZZ,Unknown Region,0 +farmers.com,farmers.com,inbound,ZZ,Unknown Region,0.999984 +fastcompany.com,fastcompany.com,inbound,ZZ,Unknown Region,0 +fedex.com,fedex.com,inbound,ZZ,Unknown Region,0.919932 +feld-ent.com,postdirect.com,inbound,ZZ,Unknown Region,0 +fellowshiponemail.com,fellowshiponemail.com,inbound,ZZ,Unknown Region,0 +fetlifemail.com,fetlifemail.com,inbound,ZZ,Unknown Region,0 +fidelity.com,fidelity.com,inbound,ZZ,Unknown Region,1 +financialfreedommail.com,financialfreedommail.com,inbound,ZZ,Unknown Region,0 +fingerhut.com,fingerhut.com,inbound,ZZ,Unknown Region,0 +flets.com,flets.com,inbound,ZZ,Unknown Region,0 +flirtlocal.com,flirtlocal.com,inbound,ZZ,Unknown Region,1 +flmsecure.com,fling.com,inbound,ZZ,Unknown Region,0 +flmsecure.com,flmsecure.com,inbound,ZZ,Unknown Region,0 +floridajobdepartment.com,floridajobdepartment.com,inbound,ZZ,Unknown Region,0 +fnac.com,fnac.com,inbound,ZZ,Unknown Region,0.025375 +fnb.co.za,fnb.co.za,inbound,ZZ,Unknown Region,0.325259 +foodnetwork.com,foodnetwork.com,inbound,ZZ,Unknown Region,0 +forcemail.in,iaires.com,inbound,ZZ,Unknown Region,0 +foreseegame.com,iaires.com,inbound,ZZ,Unknown Region,0 +fotffamily.com,fotffamily.com,inbound,ZZ,Unknown Region,0 +fotolivro.com.br,fotolivro.com.br,inbound,ZZ,Unknown Region,0 +fpmailerbr.com,fpmailerbr.com,inbound,ZZ,Unknown Region,0 +freecycle.org,freecycle.org,inbound,ZZ,Unknown Region,1 +freelancer.com,getafreelancer.com,inbound,ZZ,Unknown Region,0 +freesafelistmailer.com,waters-advertising.com,inbound,ZZ,Unknown Region,0 +freshmail.pl,freshmail.pl,inbound,ZZ,Unknown Region,0 +fridays.com,fridays.com,inbound,ZZ,Unknown Region,0 +frk.com,frk.com,inbound,ZZ,Unknown Region,1 +frontdoor.com,frontdoor.com,inbound,ZZ,Unknown Region,0 +frontgate-email.com,frontgate-email.com,inbound,ZZ,Unknown Region,0 +fuckbooknet.net,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +fundplaza.co.in,arrowsignindia.com,inbound,ZZ,Unknown Region,0 +fundplaza.in,fundplaza.in,inbound,ZZ,Unknown Region,0 +funonthenet.in,funonthenet.in,inbound,ZZ,Unknown Region,1 +futurmailer.pt,futurmailer.pt,inbound,ZZ,Unknown Region,0 +gabbar.info,gabbar.info,inbound,ZZ,Unknown Region,1 +gamefly.com,gamefly.com,inbound,ZZ,Unknown Region,0.014317 +gamingmails.com,gamingmails.com,inbound,ZZ,Unknown Region,0 +gap.com,gap.com,inbound,ZZ,Unknown Region,0 +gap.eu,gap.eu,inbound,ZZ,Unknown Region,0 +gapcanada.ca,gapcanada.ca,inbound,ZZ,Unknown Region,0 +garanti.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 +garnethill-email.com,garnethill-email.com,inbound,ZZ,Unknown Region,0 +gaylordalert.com,gaylordalert.com,inbound,ZZ,Unknown Region,0 +gcast.com.au,systemsserver.net,inbound,ZZ,Unknown Region,0 +gdtsuccess.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 +geico.com,geico.com,inbound,ZZ,Unknown Region,0.096795 +gene.com,roche.com,inbound,ZZ,Unknown Region,1 +geocaching.com,groundspeak.com,inbound,ZZ,Unknown Region,1 +getinbox.net,getinbox.net,inbound,ZZ,Unknown Region,0 +getkeepsafe.com,getkeepsafe.com,inbound,ZZ,Unknown Region,1 +getmein.com,getmein.com,inbound,ZZ,Unknown Region,0 +getresponse.com,getresponse.com,inbound,ZZ,Unknown Region,0 +gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,ZZ,Unknown Region,0 +gilt.com,gilt.com,inbound,ZZ,Unknown Region,1e-06 +github.com,github.net,inbound,ZZ,Unknown Region,1 +glassdoor.com,glassdoor.com,inbound,ZZ,Unknown Region,0 +globalmembersupport.com,globalmembersupport.com,inbound,ZZ,Unknown Region,0 +globalspec.com,globalspec.com,inbound,ZZ,Unknown Region,0 +gmail.com,blackberry.com,inbound,ZZ,Unknown Region,0.998326 +gmail.com,postini.com,inbound,ZZ,Unknown Region,0.891175 +gmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.998661 +go.com,starwave.com,inbound,ZZ,Unknown Region,0.006276 +godtubemail.com,godtubemail.com,inbound,ZZ,Unknown Region,0 +godvinemail.com,godvinemail.com,inbound,ZZ,Unknown Region,0 +gohappy.com.tw,gohappy.com.tw,inbound,ZZ,Unknown Region,0 +goldenbrands.gr,goldenbrands.gr,inbound,ZZ,Unknown Region,1 +golfnow.com,email-golfnow.com,inbound,ZZ,Unknown Region,0 +google.com,postini.com,inbound,ZZ,Unknown Region,0.81059 +gop.com,gop.com,inbound,ZZ,Unknown Region,0 +grabone-mail-ie.com,grabone-mail-ie.com,inbound,ZZ,Unknown Region,0 +grabone-mail.com,grabone-mail.com,inbound,ZZ,Unknown Region,0 +gratka.pl,gratka.pl,inbound,ZZ,Unknown Region,0 +grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,ZZ,Unknown Region,0 +groopdealz.com,groopdealz.com,inbound,ZZ,Unknown Region,1 +groupalia.es,groupalia.es,inbound,ZZ,Unknown Region,0 +groupalia.it,groupalia.it,inbound,ZZ,Unknown Region,0 +groupon.jp,data-hotel.net,inbound,ZZ,Unknown Region,0 +groupon.{...},groupon.{...},inbound,ZZ,Unknown Region,0.990176 +grouponmail.{...},grouponmail.{...},inbound,ZZ,Unknown Region,0 +grubhubmail.com,grubhubmail.com,inbound,ZZ,Unknown Region,0 +grupanya.com,euromsg.net,inbound,ZZ,Unknown Region,0 +guruin.info,guru.net.in,inbound,ZZ,Unknown Region,1 +habitaclia.com,splio.es,inbound,ZZ,Unknown Region,0.951406 +harborfreightemail.com,harborfreightemail.com,inbound,ZZ,Unknown Region,0 +hautelook.com,hautelook.com,inbound,ZZ,Unknown Region,0 +hepsiburada.com,euromsg.net,inbound,ZZ,Unknown Region,0 +herbalifemail.com,herbalifemail.com,inbound,ZZ,Unknown Region,0 +herculist.com,herculist.com,inbound,ZZ,Unknown Region,0 +hgtv.com,hgtv.com,inbound,ZZ,Unknown Region,0 +hhgreggemail.com,hhgreggemail.com,inbound,ZZ,Unknown Region,0 +hipmunk.com,hipmunk.com,inbound,ZZ,Unknown Region,0 +hln.be,persgroep-ops.net,inbound,ZZ,Unknown Region,0 +hm-f.jp,hm-f.jp,inbound,ZZ,Unknown Region,0 +hobsonsmail.com,hobsonsmail.com,inbound,ZZ,Unknown Region,0 +homebaselife.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +homechoice.co.za,homechoice.co.za,inbound,ZZ,Unknown Region,0 +homedepotemail.com,homedepotemail.com,inbound,ZZ,Unknown Region,0 +honto.jp,honto.jp,inbound,ZZ,Unknown Region,0 +horoscope.com,center.com,inbound,ZZ,Unknown Region,0 +hotels.com,hotels.com,inbound,ZZ,Unknown Region,0 +hotelurbano.com.br,allin.com.br,inbound,ZZ,Unknown Region,0 +hotmail.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999944 +hotmail.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 +hotspotmailer.com,hotspotmailer.com,inbound,ZZ,Unknown Region,1 +hp.com,hp.com,inbound,ZZ,Unknown Region,0.198696 +hsn.com,hsn.com,inbound,ZZ,Unknown Region,0 +htcampusmailer.com,eccluster.com,inbound,ZZ,Unknown Region,0 +hubspot.com,hubspot.com,inbound,ZZ,Unknown Region,1 +huinforma.com.br,huinforma.com.br,inbound,ZZ,Unknown Region,0 +hulumail.com,hulumail.com,inbound,ZZ,Unknown Region,0 +hungryhouse.co.uk,mxmfb.com,inbound,ZZ,Unknown Region,0 +huntington.com,huntington.com,inbound,ZZ,Unknown Region,0.986937 +i-say.com,ipsos-interactive.com,inbound,ZZ,Unknown Region,1 +icloud.com,apple.com,inbound,ZZ,Unknown Region,1 +icloud.com,icloud.com,outbound,ZZ,Unknown Region,1 +icloud.com,mac.com,inbound,ZZ,Unknown Region,1 +icloud.com,me.com,inbound,ZZ,Unknown Region,0.999995 +ifttt.com,ifttt.com,inbound,ZZ,Unknown Region,1 +ign.com,ign.com,inbound,ZZ,Unknown Region,0 +ignitionsender.com,ignitionsender.com,inbound,ZZ,Unknown Region,0 +iheart.com,iheart.com,inbound,ZZ,Unknown Region,0 +immobilienscout24.de,immobilienscout24.de,inbound,ZZ,Unknown Region,1 +in-boxpays.com,in-boxpays.com,inbound,ZZ,Unknown Region,0 +indiamart.com,indiamart.com,inbound,ZZ,Unknown Region,1 +indiatimes.com,speakingtree.in,inbound,ZZ,Unknown Region,0 +indiatimeshop.com,sendpal.in,inbound,ZZ,Unknown Region,0 +indieroyale.com,desura.com,inbound,ZZ,Unknown Region,1 +infibeam.com,eccluster.com,inbound,ZZ,Unknown Region,0 +infopanel.jp,mailds.jp,inbound,ZZ,Unknown Region,0 +infos-micromania.com,infos-micromania.com,inbound,ZZ,Unknown Region,0 +infosephora.com,splio.com,inbound,ZZ,Unknown Region,0.962167 +infoworld.com,infoworld.com,inbound,ZZ,Unknown Region,0 +infusionmail.com,infusionmail.com,inbound,ZZ,Unknown Region,0 +inmotionhosting.com,inmotionhosting.com,inbound,ZZ,Unknown Region,1 +ino.com,ino.com,inbound,ZZ,Unknown Region,0.999981 +interwell.gr,interwell.gr,inbound,ZZ,Unknown Region,0 +ipcmedia.co.uk,ipcmedia.co.uk,inbound,ZZ,Unknown Region,0 +itunes.com,apple.com,inbound,ZZ,Unknown Region,0.043012 +jackwills.com,jackwills.com,inbound,ZZ,Unknown Region,0 +jalag.de,jalag.de,inbound,ZZ,Unknown Region,1 +jane.com,jane.com,inbound,ZZ,Unknown Region,1 +jango.com,jango.com,inbound,ZZ,Unknown Region,0 +jared.com,jared.com,inbound,ZZ,Unknown Region,0 +jdate.com,postdirect.com,inbound,ZZ,Unknown Region,0 +jetprivilege.com,jetprivilege.com,inbound,ZZ,Unknown Region,0 +jeuxvideo.com,jeuxvideo.com,inbound,ZZ,Unknown Region,0.148921 +jeweloscoemail.com,email-mywebgrocer2.com,inbound,ZZ,Unknown Region,0 +joann-mail.com,joann-mail.com,inbound,ZZ,Unknown Region,0 +jobinsider.com,jobinsider.com,inbound,ZZ,Unknown Region,0 +jobisjob.com,jobisjob.com,inbound,ZZ,Unknown Region,0 +jobomas.com,jobomas.com,inbound,ZZ,Unknown Region,1 +jobstreet.com,jobstreet.com,inbound,ZZ,Unknown Region,0 +jpcycles.com,jpcycles.com,inbound,ZZ,Unknown Region,0.001716 +jungleerummy.com,jungleerummy.com,inbound,ZZ,Unknown Region,1 +jusbrasil.com.br,jusbrasil.com.br,inbound,ZZ,Unknown Region,0 +just-eat.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +justclick.ru,justclick.ru,inbound,ZZ,Unknown Region,0 +justdial.com,iaires.com,inbound,ZZ,Unknown Region,0 +k1speed.com,k1speed.com,inbound,ZZ,Unknown Region,1 +kaskusnetworks.com,kaskus.com,inbound,ZZ,Unknown Region,0 +kay.com,kay.com,inbound,ZZ,Unknown Region,0 +keek.com,keek.com,inbound,ZZ,Unknown Region,1 +kgbdeals.co.uk,email1-kgbdeals.com,inbound,ZZ,Unknown Region,0 +kliksa.net,euromsg.net,inbound,ZZ,Unknown Region,0 +kliktoday.com,kliktoday.com,inbound,ZZ,Unknown Region,0 +klm-mail.com,klm-mail.com,inbound,ZZ,Unknown Region,0 +kohls.com,kohls.com,inbound,ZZ,Unknown Region,0 +kongregate.com,kongregate.com,inbound,ZZ,Unknown Region,0 +krs.bz,tricorn.net,inbound,ZZ,Unknown Region,0 +la-meteo-mail.fr,splio.com,inbound,ZZ,Unknown Region,1 +laaptuemail.com,laaptuemail.com,inbound,ZZ,Unknown Region,0 +lakewoodchurch.com,lakewoodchurch.com,inbound,ZZ,Unknown Region,0 +landsend.com,email-landsend.com,inbound,ZZ,Unknown Region,0 +landsend.com,postdirect.com,inbound,ZZ,Unknown Region,0 +laptuinvite.com,laptuinvite.com,inbound,ZZ,Unknown Region,0 +lastminute.com,lastminute.com,inbound,ZZ,Unknown Region,0 +lazerhits.com,lazerhits.com,inbound,ZZ,Unknown Region,1 +leadercontato.com.br,leadercontato.com.br,inbound,ZZ,Unknown Region,0 +lefigaro.fr,splio.com,inbound,ZZ,Unknown Region,1 +lemonde.fr,lemonde.fr,inbound,ZZ,Unknown Region,0 +life360.com,life360.com,inbound,ZZ,Unknown Region,1 +lifecare-news.com,email-lifecare.com,inbound,ZZ,Unknown Region,0 +lifemiles.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +lifescript.com,ilinkmd.com,inbound,ZZ,Unknown Region,0 +line6.com,line6.com,inbound,ZZ,Unknown Region,0 +linkedin.com,linkedin.com,inbound,ZZ,Unknown Region,0.995201 +linkedin.com,postini.com,inbound,ZZ,Unknown Region,0.940251 +liquidation.com,liquidation.com,inbound,ZZ,Unknown Region,0 +listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,ZZ,Unknown Region,0.00023 +live.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999921 +live.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 +livejournal.com,livejournal.com,inbound,ZZ,Unknown Region,0 +livemailservice.com,livemailservice.com,inbound,ZZ,Unknown Region,0 +livenation.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +lmlmgv.com.br,gvarev.com.br,inbound,ZZ,Unknown Region,0 +loccitane.com,neolane.net,inbound,ZZ,Unknown Region,0 +loft.com,anntaylor.com,inbound,ZZ,Unknown Region,0 +lombardipublishing.com,lombardipublishing.com,inbound,ZZ,Unknown Region,0 +lookout.com,lookout.com,inbound,ZZ,Unknown Region,1 +lsi.com,postini.com,inbound,ZZ,Unknown Region,0.981115 +lynxmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +lyris.net,lyris.net,inbound,ZZ,Unknown Region,0 +m1e.net,m1e.net,inbound,ZZ,Unknown Region,0.000342 +mac.com,icloud.com,outbound,ZZ,Unknown Region,1 +mac.com,mac.com,inbound,ZZ,Unknown Region,1 +macromill.com,macromill.com,inbound,ZZ,Unknown Region,0 +macys.com,macys.com,inbound,ZZ,Unknown Region,0 +magix.net,magix.net,inbound,ZZ,Unknown Region,0.673176 +mail-backcountry.com,email-bcmarketing.com,inbound,ZZ,Unknown Region,0 +mail.ru,mail.ru,inbound,ZZ,Unknown Region,0.986526 +mail.ru,mail.ru,outbound,ZZ,Unknown Region,0.006561 +maileclipse.com,emce2.in,inbound,ZZ,Unknown Region,0 +mailengine1.com,mailengine1.com,inbound,ZZ,Unknown Region,0 +mailer4u.in,elabs10.com,inbound,ZZ,Unknown Region,0 +mailersend.com,mailersend.com,inbound,ZZ,Unknown Region,0 +mailjet.com,mailjet.com,inbound,ZZ,Unknown Region,0.803083 +mailmailmail.net,mailmailmail.net,inbound,ZZ,Unknown Region,0 +mailoct.in,tcmailer14.in,inbound,ZZ,Unknown Region,0 +mailoct1.in,mailoct1.in,inbound,ZZ,Unknown Region,0 +mailoct1.in,myntramail2.in,inbound,ZZ,Unknown Region,0 +mailorama.fr,mailorama.fr,inbound,ZZ,Unknown Region,0 +mailpost.in,iaires.com,inbound,ZZ,Unknown Region,0 +mailquant.com,iaires.com,inbound,ZZ,Unknown Region,0 +mandrillapp.com,backpage.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,mcsignup.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,myjobhelperalerts.com,inbound,ZZ,Unknown Region,1 +mango.com,emstechnology2.net,inbound,ZZ,Unknown Region,0 +manipal.edu,iaires.com,inbound,ZZ,Unknown Region,0 +manta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +mar0.net,mar0.net,inbound,ZZ,Unknown Region,0.976409 +markavip.com,markavip.com,inbound,ZZ,Unknown Region,0 +marketingstudio.com,marketingstudio.com,inbound,ZZ,Unknown Region,0 +marksandspencer.com,marksandspencer.com,inbound,ZZ,Unknown Region,0 +maropost.com,mp2200.com,inbound,ZZ,Unknown Region,0 +maropost.com,survivallife.com,inbound,ZZ,Unknown Region,0 +marykay.com,marykay.com,inbound,ZZ,Unknown Region,0 +masivapp.com,masivapp.com,inbound,ZZ,Unknown Region,1 +match.com,match.com,inbound,ZZ,Unknown Region,0 +mbga.jp,mbga.jp,inbound,ZZ,Unknown Region,0 +mbna.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +mcdlv.net,mcdlv.net,inbound,ZZ,Unknown Region,0 +mcsv.net,mcsv.net,inbound,ZZ,Unknown Region,0 +me.com,icloud.com,outbound,ZZ,Unknown Region,1 +me.com,mac.com,inbound,ZZ,Unknown Region,1 +medallia.com,medallia.com,inbound,ZZ,Unknown Region,1 +mediapost.com,mediapost.com,inbound,ZZ,Unknown Region,0 +medium.com,messagebus.com,inbound,ZZ,Unknown Region,0 +medscape.com,medscape.com,inbound,ZZ,Unknown Region,0 +meetup.com,meetup.com,inbound,ZZ,Unknown Region,0 +melaleuca.com,melaleuca.com,inbound,ZZ,Unknown Region,0 +mercadojobs.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +mercadolibre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 +mercadolivre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 +merceworld.com,merceworld.com,inbound,ZZ,Unknown Region,0.996737 +merodea.me,sendgrid.net,inbound,ZZ,Unknown Region,1 +metro.co.in,srv2.de,inbound,ZZ,Unknown Region,0.966947 +mgmresorts.com,mgmresorts.com,inbound,ZZ,Unknown Region,0 +microsoft.com,msn.com,inbound,ZZ,Unknown Region,1 +microsoftemail.com,microsoftemail.com,inbound,ZZ,Unknown Region,0 +microsoftemail.com,microsoftstoreemail.com,inbound,ZZ,Unknown Region,0 +mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,ZZ,Unknown Region,0 +minhavida.com.br,minhavida.com.br,inbound,ZZ,Unknown Region,0 +mixi.jp,mixi.jp,inbound,ZZ,Unknown Region,0 +mjinn.com,mailurja.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktdns.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktomail.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktroute.com,inbound,ZZ,Unknown Region,0 +mlsend.com,mlsend.com,inbound,ZZ,Unknown Region,0 +mlsend2.com,mlsend2.com,inbound,ZZ,Unknown Region,0 +mlssoccer.com,mlssoccer.com,inbound,ZZ,Unknown Region,0 +mmaco.net,mmaco.net,inbound,ZZ,Unknown Region,0.999998 +mmorpg.com,mmorpg.com,inbound,ZZ,Unknown Region,0.002979 +mocospace.com,mocospace.com,inbound,ZZ,Unknown Region,0 +moneyforward.com,moneyforward.com,inbound,ZZ,Unknown Region,0 +moneysupermarketmail.com,moneysupermarketmail.com,inbound,ZZ,Unknown Region,0 +monografias.com,elistas.net,inbound,ZZ,Unknown Region,0 +monster.com,monster.com,inbound,ZZ,Unknown Region,0.000503 +monsterindia.com,monster.co.in,inbound,ZZ,Unknown Region,0 +mooply.co,mailendo.com,inbound,ZZ,Unknown Region,0 +morningstar.net,morningstar.net,inbound,ZZ,Unknown Region,0 +mothercaregroup.com,neolane.net,inbound,ZZ,Unknown Region,0 +mozilla.org,mozilla.com,inbound,ZZ,Unknown Region,0.633241 +mpse.jp,mpme.jp,inbound,ZZ,Unknown Region,0 +ms00.net,ms00.net,inbound,ZZ,Unknown Region,0 +msdp1.com,msdp1.com,inbound,ZZ,Unknown Region,0 +msn.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999946 +msn.com,hotmail.{...},outbound,ZZ,Unknown Region,1 +musiciansfriend.com,musiciansfriend.com,inbound,ZZ,Unknown Region,0 +mxmfb.com,mxmfb.com,inbound,ZZ,Unknown Region,0 +mycolorscreen.com,mta4.net,inbound,ZZ,Unknown Region,0 +myfitnesspal.com,messagebus.com,inbound,ZZ,Unknown Region,0 +mygroupon.co.th,grouponmail.{...},inbound,ZZ,Unknown Region,0 +myheritage.com,myheritage.com,inbound,ZZ,Unknown Region,0 +myideeli.com,myideeli.com,inbound,ZZ,Unknown Region,0 +myntramail.com,iaires.com,inbound,ZZ,Unknown Region,0 +myntramail.com,myntramail.com,inbound,ZZ,Unknown Region,0 +myntramails.in,icubes.in,inbound,ZZ,Unknown Region,0 +myoutlets.in,trustmailer.com,inbound,ZZ,Unknown Region,0 +mypoints.com,mypoints.com,inbound,ZZ,Unknown Region,0 +mysale.my,mysale.my,inbound,ZZ,Unknown Region,0 +mysale.ph,mysale.ph,inbound,ZZ,Unknown Region,0 +mysupermarket.co.uk,mysupermarket.co.uk,inbound,ZZ,Unknown Region,0 +mysurvey.com,mysurvey.com,inbound,ZZ,Unknown Region,0 +mysurvey.eu,mysurvey.com,inbound,ZZ,Unknown Region,0 +myvegas.com,myvegas.com,inbound,ZZ,Unknown Region,1 +naaptoldeals.com,eccluster.com,inbound,ZZ,Unknown Region,0 +nanomail.com.br,araie.com.br,inbound,ZZ,Unknown Region,1 +nascar.com,nascar.com,inbound,ZZ,Unknown Region,0 +nationbuilder.com,nationbuilder.com,inbound,ZZ,Unknown Region,1 +nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,ZZ,Unknown Region,0 +naukri.com,naukri.com,inbound,ZZ,Unknown Region,0 +navy.mil,navy.mil,inbound,ZZ,Unknown Region,0.000243 +nend.net,postini.com,inbound,ZZ,Unknown Region,0 +netatlantic.com,netatlantic.com,inbound,ZZ,Unknown Region,0.001085 +netflix.com,amazonses.com,inbound,ZZ,Unknown Region,0.999999 +netflix.com,netflix.com,inbound,ZZ,Unknown Region,1 +netlogmail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 +netprosoftmail.com,netprosoftmail.com,inbound,ZZ,Unknown Region,0 +netshoes.com.br,netshoes.com.br,inbound,ZZ,Unknown Region,0 +newegg.com,newegg.com,inbound,ZZ,Unknown Region,2e-06 +newmarkethealth.com,newmarkethealth.com,inbound,ZZ,Unknown Region,0 +news-h5g.com,news-h5g.com,inbound,ZZ,Unknown Region,0 +newsletter-verychic.com,splio.es,inbound,ZZ,Unknown Region,0.9804 +newsmax.com,newsmax.com,inbound,ZZ,Unknown Region,0 +nflshop.com,nflshop.com,inbound,ZZ,Unknown Region,0 +nieuwsblad.be,vummail.be,inbound,ZZ,Unknown Region,0 +nike.com,nike.com,inbound,ZZ,Unknown Region,0 +ninewestmail.com,ninewestmail.com,inbound,ZZ,Unknown Region,0 +nokia.com,nokia.com,inbound,ZZ,Unknown Region,0.001256 +npr.org,npr.org,inbound,ZZ,Unknown Region,0 +ns.nl,tripolis.com,inbound,ZZ,Unknown Region,0 +nsandi.com,mxmfb.com,inbound,ZZ,Unknown Region,0 +nytimes.com,nytimes.com,inbound,ZZ,Unknown Region,0 +nzsale.co.nz,nzsale.co.nz,inbound,ZZ,Unknown Region,0 +oakley.com,oakley.com,inbound,ZZ,Unknown Region,0 +ocmail1.in,tcmail.in,inbound,ZZ,Unknown Region,0 +ocmail14.in,tcmailer5.in,inbound,ZZ,Unknown Region,0 +ocmail22.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 +ocmail22.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 +ocmail40.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 +ocmail40.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 +ocnmail.in,ocmail6.in,inbound,ZZ,Unknown Region,0 +ocnmail.in,tcmail3.in,inbound,ZZ,Unknown Region,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,ZZ,Unknown Region,0 +ofertix.com,ofertix.com,inbound,ZZ,Unknown Region,0 +offers.com,offers.com,inbound,ZZ,Unknown Region,1 +officedepot.com,officedepot.com,inbound,ZZ,Unknown Region,0.019269 +officemax.com,officemax.com,inbound,ZZ,Unknown Region,0 +officemax.com,officemaxworkplace.com,inbound,ZZ,Unknown Region,0 +ofsys.com,bulletin-metro.ca,inbound,ZZ,Unknown Region,0 +oknotify2.com,oknotify2.com,inbound,ZZ,Unknown Region,0 +oldnavy.ca,oldnavy.ca,inbound,ZZ,Unknown Region,0 +oldnavy.com,oldnavy.com,inbound,ZZ,Unknown Region,0 +omahasteaks.com,omahasteaks.com,inbound,ZZ,Unknown Region,0 +oneindia.in,mailurja.com,inbound,ZZ,Unknown Region,0 +onekingslane.com,onekingslane.com,inbound,ZZ,Unknown Region,0 +onlive.com,ipost.com,inbound,ZZ,Unknown Region,0 +onmicrosoft.com,outlook.com,inbound,ZZ,Unknown Region,1 +optimusmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +orderscatalog.com,orderscatalog.com,inbound,ZZ,Unknown Region,0 +oroscopofree.com,adsender.us,inbound,ZZ,Unknown Region,0 +os-email.com,os-email.com,inbound,ZZ,Unknown Region,0 +oshkoshbgosh.com,oshkoshbgosh.com,inbound,ZZ,Unknown Region,6e-06 +osu.edu,outlook.com,inbound,ZZ,Unknown Region,1 +otto.de,eccluster.com,inbound,ZZ,Unknown Region,1 +ouffer.com,ouffer.com,inbound,ZZ,Unknown Region,0 +ourtime.com,seniorpeoplemeet.com,inbound,ZZ,Unknown Region,0 +outback.com,outback.com,inbound,ZZ,Unknown Region,0 +outlook.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999843 +outlook.com,hotmail.{...},outbound,ZZ,Unknown Region,1 +outspot.be,teneo.be,inbound,ZZ,Unknown Region,0 +outspot.nl,teneo.be,inbound,ZZ,Unknown Region,0 +ovenmail.com,iaires.com,inbound,ZZ,Unknown Region,0 +overstock.com,overstock.com,inbound,ZZ,Unknown Region,0 +ovh.net,ovh.net,inbound,ZZ,Unknown Region,0.352531 +ozsale.com.au,ozsale.com.au,inbound,ZZ,Unknown Region,0.000192 +panelplace.com,smtp.com,inbound,ZZ,Unknown Region,0 +panerabreadnews.com,panerabreadnews.com,inbound,ZZ,Unknown Region,0 +pantaloondirect.net,iaires.com,inbound,ZZ,Unknown Region,0 +papajohns-specials.com,papajohns-specials.com,inbound,ZZ,Unknown Region,0 +paradisepublishers.com,paradisepublishers.com,inbound,ZZ,Unknown Region,0.99957 +path.com,path.com,inbound,ZZ,Unknown Region,1 +payback.in,eccluster.com,inbound,ZZ,Unknown Region,0 +payback.in,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +payback.in,ecmcluster.com,inbound,ZZ,Unknown Region,0 +paypal.co.uk,paypal.com,inbound,ZZ,Unknown Region,1 +paypal.com,paypal.com,inbound,ZZ,Unknown Region,0.608834 +paypal.com.au,paypal.com,inbound,ZZ,Unknown Region,1 +paypal.de,paypal.com,inbound,ZZ,Unknown Region,1 +pd25.com,pd25.com,inbound,ZZ,Unknown Region,1 +peanuthome.info,adopterc.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,aguitytr.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bevest.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bluester.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,burror.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bursion.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cantexi.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,caserhi.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,celect.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,chintone.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,citery.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cleathal.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,colicom.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,complec.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,deciarc.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,declaws.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,dewest.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,epconce.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,fnotec.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,folkswor.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,forepert.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,gurgaro.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,heallyps.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,holeph.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,homewor.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,hydroni.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,ingenbu.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,unglibshudder.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,adcrent.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,addmiel.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,agilhe.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,allegap.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,andvore.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,angogl.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,animass.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,arettery.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,aribank.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,arkefoc.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,avenog.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,barrave.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bindowmo.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bitravit.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,borsand.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,branti.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,breaserp.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bredogly.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,briantra.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bridea.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,carial.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,castac.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,chedoner.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,chiquent.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cinchoi.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cliate.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cognn.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,abjibbin.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,audiette.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,blancer.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,bluester.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,burror.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,bursion.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,caserhi.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,celect.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,cleathal.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,complec.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,condost.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,deciarc.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,declaws.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,epconce.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ferrayer.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,fnotec.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,folkswor.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,forepert.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,gurgaro.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,holeph.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,homewor.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,hydroni.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ingenbu.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,ZZ,Unknown Region,0 +pennwell.com,pennwell.com,inbound,ZZ,Unknown Region,0 +perfectworld.com,perfectworld.com,inbound,ZZ,Unknown Region,0 +personare.com.br,personare.com.br,inbound,ZZ,Unknown Region,0 +pge.com,pge.com,inbound,ZZ,Unknown Region,0.149792 +philosophy.com,philosophy.com,inbound,ZZ,Unknown Region,0 +phpclasses.org,phpclasses.org,inbound,ZZ,Unknown Region,0 +pinger.com,pinger.com,inbound,ZZ,Unknown Region,0 +pinterest.com,pinterest.com,inbound,ZZ,Unknown Region,1 +pizzahutoffers.com,pizzahutoffers.com,inbound,ZZ,Unknown Region,0 +playstationmail.net,playstationmail.net,inbound,ZZ,Unknown Region,0 +plumdistrict.com,plumdistrict.com,inbound,ZZ,Unknown Region,1 +politicoemail.com,politicoemail.com,inbound,ZZ,Unknown Region,0 +polyvore.com,polyvore.com,inbound,ZZ,Unknown Region,1 +popsugar.com,popsugar.com,inbound,ZZ,Unknown Region,0 +priceline.com,priceline.com,inbound,ZZ,Unknown Region,1 +princess.com,princess.com,inbound,ZZ,Unknown Region,0 +privoscite.si,privoscite.si,inbound,ZZ,Unknown Region,0 +profitcenteronline.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 +progressive.com,progressive.com,inbound,ZZ,Unknown Region,0.814003 +publix.com,publix.com,inbound,ZZ,Unknown Region,0 +puritan.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 +qoinpro.com,qoinpro.com,inbound,ZZ,Unknown Region,1 +qoo10.jp,qoo10.jp,inbound,ZZ,Unknown Region,4e-06 +qoo10.sg,qoo10.co.id,inbound,ZZ,Unknown Region,1.9e-05 +qoo10.sg,qoo10.com,inbound,ZZ,Unknown Region,0 +qoo10.sg,qoo10.my,inbound,ZZ,Unknown Region,2.2e-05 +qoo10.sg,qoo10.sg,inbound,ZZ,Unknown Region,8e-06 +quinstreet.com,neoquin.com,inbound,ZZ,Unknown Region,0 +quora.com,quora.com,inbound,ZZ,Unknown Region,1 +rabota.ua,rabota.ua,inbound,ZZ,Unknown Region,0 +rackroom-email.com,rackroom-email.com,inbound,ZZ,Unknown Region,0 +railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,ZZ,Unknown Region,0 +rakuten.co.jp,rakuten.co.jp,inbound,ZZ,Unknown Region,0 +rakuten.com,rakuten.com,inbound,ZZ,Unknown Region,0 +reactiveadz.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 +realage-mail.com,postdirect.com,inbound,ZZ,Unknown Region,0 +redbox.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +redcross.org.uk,redcross.org.uk,inbound,ZZ,Unknown Region,0 +rediffmail.com,rediffmail.com,inbound,ZZ,Unknown Region,0 +reebonz.com,reebonz.com,inbound,ZZ,Unknown Region,0.759458 +reed.co.uk,reed.co.uk,inbound,ZZ,Unknown Region,0 +regie11.net,odiso.net,inbound,ZZ,Unknown Region,0 +regionalhelpwanted.com,regionalhelpwanted.com,inbound,ZZ,Unknown Region,1 +registrar-servers.com,registrar-servers.com,inbound,ZZ,Unknown Region,0.05163 +repica.jp,kakaku.com,inbound,ZZ,Unknown Region,0 +repica.jp,repica.jp,inbound,ZZ,Unknown Region,0 +republicwireless.com,republicwireless.com,inbound,ZZ,Unknown Region,0.033564 +retailjobinsider.com,retailjobinsider.com,inbound,ZZ,Unknown Region,0 +retailmenot.com,retailmenot.com,inbound,ZZ,Unknown Region,4.68294583517529e-07 +reverbnation.com,reverbnation.com,inbound,ZZ,Unknown Region,0 +revolutiongolf.com,revolutiongolf.com,inbound,ZZ,Unknown Region,0 +reyrey.net,reyrey.net,inbound,ZZ,Unknown Region,0.999904 +richyrichmailer.com,maddog-productions.info,inbound,ZZ,Unknown Region,1 +rikunabi.com,rikunabi.com,inbound,ZZ,Unknown Region,0 +ringcentral.com,ringcentral.com,inbound,ZZ,Unknown Region,1 +rmtr.de,rapidmail.de,inbound,ZZ,Unknown Region,0 +rnmk.com,rnmk.com,inbound,ZZ,Unknown Region,0 +rockpath.info,rockpath.info,inbound,ZZ,Unknown Region,1 +rogers.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +rogers.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +rookiestewsemails.com,rookiestewsemails.com,inbound,ZZ,Unknown Region,0 +royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,ZZ,Unknown Region,0 +rpo9usa.email,rpo9usa.email,inbound,ZZ,Unknown Region,0 +rr.com,rr.com,inbound,ZZ,Unknown Region,2e-06 +rr.com,rr.com,outbound,ZZ,Unknown Region,0 +rsgsv.net,rsgsv.net,inbound,ZZ,Unknown Region,0 +rummycirclemails.com,eccluster.com,inbound,ZZ,Unknown Region,0 +runkeeper.com,runkeeper.com,inbound,ZZ,Unknown Region,0 +s3s-br1.net,splio.com.br,inbound,ZZ,Unknown Region,0.908187 +s3s-main.net,splio.com,inbound,ZZ,Unknown Region,0.970428 +s4s-pl1.pl,splio.com,inbound,ZZ,Unknown Region,0.939032 +safe-sender.net,safe-sender.net,inbound,ZZ,Unknown Region,0 +safeway.com,safeway.com,inbound,ZZ,Unknown Region,0.000382 +salesforce.com,postini.com,inbound,ZZ,Unknown Region,0.915635 +salesforce.com,salesforce.com,inbound,ZZ,Unknown Region,0.952721 +samsungusa.com,samsungusa.com,inbound,ZZ,Unknown Region,0 +sanmina-sci.com,postini.com,inbound,ZZ,Unknown Region,0.999991 +sanmina.com,postini.com,inbound,ZZ,Unknown Region,0.999828 +saturday.com,saturday.com,inbound,ZZ,Unknown Region,0 +sbcglobal.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999985 +sears.ca,sears.ca,inbound,ZZ,Unknown Region,0.005447 +searscard.com,searscard.com,inbound,ZZ,Unknown Region,1 +secretescapes.com,secretescapes.com,inbound,ZZ,Unknown Region,0 +secure.ne.jp,secure.ne.jp,inbound,ZZ,Unknown Region,0.000618 +secureserver.net,secureserver.net,inbound,ZZ,Unknown Region,0 +seek.com.au,seek.com.au,inbound,ZZ,Unknown Region,0 +selectacast.net,selectacast.net,inbound,ZZ,Unknown Region,0.000561 +semana.com,semana.com,inbound,ZZ,Unknown Region,0 +sendgrid.info,sendgrid.net,inbound,ZZ,Unknown Region,0.999896 +sendgrid.me,sendgrid.net,inbound,ZZ,Unknown Region,1 +sendpal.in,sendpal.in,inbound,ZZ,Unknown Region,0 +serviciobancomer.com,serviciobancomer.com,inbound,ZZ,Unknown Region,0 +sfid01.com,sfid01.com,inbound,ZZ,Unknown Region,0 +shaadi.com,shaadi.com,inbound,ZZ,Unknown Region,0 +shop2gether.com.br,shop2gether.com.br,inbound,ZZ,Unknown Region,0 +shopjustice.com,shopjustice.com,inbound,ZZ,Unknown Region,0 +shopnineteenmails.in,iaires.com,inbound,ZZ,Unknown Region,0 +shoppersstop.com,shoppersstop.com,inbound,ZZ,Unknown Region,0 +shoprite-email.com,email-mywebgrocer.com,inbound,ZZ,Unknown Region,0 +showingtime.com,showingtime.com,inbound,ZZ,Unknown Region,0 +showroomprive.com,showroomprive.be,inbound,ZZ,Unknown Region,0 +showroomprive.com,showroomprive.nl,inbound,ZZ,Unknown Region,0 +showroomprive.es,showroomprive.es,inbound,ZZ,Unknown Region,0 +showroomprive.it,showroomprive.pt,inbound,ZZ,Unknown Region,0 +showroomprive.pt,showroomprive.co.uk,inbound,ZZ,Unknown Region,0 +shtyle.fm,shtyle.fm,inbound,ZZ,Unknown Region,0 +simplesafelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +simpletextadz.com,web-hosting.com,inbound,ZZ,Unknown Region,1 +singsale.com.sg,singsale.com.sg,inbound,ZZ,Unknown Region,0 +skillpages-mailer.com,dynect.net,inbound,ZZ,Unknown Region,0 +skymall.com,skymall.com,inbound,ZZ,Unknown Region,0 +skynet.be,belgacom.be,inbound,ZZ,Unknown Region,0 +skype.com,skype.com,inbound,ZZ,Unknown Region,0 +skyscanner.net,skyscanner.net,inbound,ZZ,Unknown Region,1 +slickdeals.net,slickdeals.net,inbound,ZZ,Unknown Region,0 +slidesharemail.com,slideshare.net,inbound,ZZ,Unknown Region,1 +smartresponder.ru,smartresponder.ru,inbound,ZZ,Unknown Region,1 +snagajob-email.com,snagajob-email.com,inbound,ZZ,Unknown Region,0 +snapdeal.com,snapdeal.com,inbound,ZZ,Unknown Region,0 +socialsex.biz,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +softbank.jp,softbank.jp,outbound,ZZ,Unknown Region,0 +softbank.ne.jp,softbank.ne.jp,inbound,ZZ,Unknown Region,0 +softbank.ne.jp,softbank.ne.jp,outbound,ZZ,Unknown Region,0 +sony.com,sony.com,inbound,ZZ,Unknown Region,0 +sonyrewards.com,sonyrewards.com,inbound,ZZ,Unknown Region,0 +soundcloudmail.com,soundcloudmail.com,inbound,ZZ,Unknown Region,0.999996 +spanishdict.com,spanishdict.com,inbound,ZZ,Unknown Region,0 +sparklist.com,sparklist.com,inbound,ZZ,Unknown Region,0 +spartoo.com,spartoo.com,inbound,ZZ,Unknown Region,0 +speedyrewards-email.com,speedyrewards-email.com,inbound,ZZ,Unknown Region,0 +spotifymail.com,spotifymail.com,inbound,ZZ,Unknown Region,1 +ssgadm.com,ssg.com,inbound,ZZ,Unknown Region,0 +staffeazymailers.com,iaires.com,inbound,ZZ,Unknown Region,0 +stakemail.com,iaires.com,inbound,ZZ,Unknown Region,0 +stampmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +standaard.be,vummail.be,inbound,ZZ,Unknown Region,0 +stansberryresearch.com,stansberry-re.net,inbound,ZZ,Unknown Region,0 +stansberryresearch.com,stansberryresearch.com,inbound,ZZ,Unknown Region,0 +staples.co.uk,ncrwebhost.de,inbound,ZZ,Unknown Region,0 +starsports.com,eccluster.com,inbound,ZZ,Unknown Region,0 +startwire.com,jobsreport.com,inbound,ZZ,Unknown Region,1 +starwoodhotels.com,outlook.com,inbound,ZZ,Unknown Region,1 +state-of-the-art-mailer.com,futurebanners.net,inbound,ZZ,Unknown Region,0 +stayfriends.de,stayfriends.de,inbound,ZZ,Unknown Region,0 +steampowered.com,steampowered.com,inbound,ZZ,Unknown Region,1 +stelladot.com,stelladot.com,inbound,ZZ,Unknown Region,0 +stjobs.sg,st701.com,inbound,ZZ,Unknown Region,1 +stjude.org,stjude.org,inbound,ZZ,Unknown Region,0.006723 +strava.com,strava.com,inbound,ZZ,Unknown Region,1 +streeteasy.com,streeteasy.com,inbound,ZZ,Unknown Region,0 +stylecareers.com,stylecareers.com,inbound,ZZ,Unknown Region,0 +subtend.info,subtend.info,inbound,ZZ,Unknown Region,1 +subway.com,subway.com,inbound,ZZ,Unknown Region,0 +surveyspot.com,ssisurveys.com,inbound,ZZ,Unknown Region,0 +sweepstakesalerts.com,sweepstakesalerts.com,inbound,ZZ,Unknown Region,0 +swimoutlet.com,isport.com,inbound,ZZ,Unknown Region,0 +sympatico.ca,hotmail.{...},inbound,ZZ,Unknown Region,1 +synchronyfinancial.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tadtopmails.com,tadtopmails.com,inbound,ZZ,Unknown Region,0 +taishinbank.com.tw,taishinbank.com.tw,inbound,ZZ,Unknown Region,0 +take2games.com,take2games.com,inbound,ZZ,Unknown Region,0 +talkmatch.com,talkmatch.com,inbound,ZZ,Unknown Region,0 +tanga.com,tanga.com,inbound,ZZ,Unknown Region,0 +tanningmail.com,tanningmail.com,inbound,ZZ,Unknown Region,0 +tappingsolutionemail.com,tappingsolutionemail.com,inbound,ZZ,Unknown Region,0 +target.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tasteofhome.com,tasteofhome.com,inbound,ZZ,Unknown Region,0 +tchibo.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 +teambuymail.com,teambuymail.com,inbound,ZZ,Unknown Region,0 +teamsnap.com,teamsnap.com,inbound,ZZ,Unknown Region,1 +technolutions.net,technolutions.net,inbound,ZZ,Unknown Region,1 +telegraph.co.uk,telegraph.co.uk,inbound,ZZ,Unknown Region,0 +telus.com,telus.com,inbound,ZZ,Unknown Region,0 +templeandwebster.com.au,templeandwebster.com.au,inbound,ZZ,Unknown Region,0 +texasjobdepartment.com,texasjobdepartment.com,inbound,ZZ,Unknown Region,0 +thebodyshop-usa.com,email-bodyshop.com,inbound,ZZ,Unknown Region,0 +thebodyshop-usa.com,postdirect.com,inbound,ZZ,Unknown Region,0 +thecarousell.com,thecarousell.com,inbound,ZZ,Unknown Region,1 +theguardian.com,theguardian.com,inbound,ZZ,Unknown Region,0 +thepamperedchef.com,thepamperedchef.com,inbound,ZZ,Unknown Region,0 +thephonehouse.es,splio.com,inbound,ZZ,Unknown Region,0.983578 +thephonehouse.es,splio.es,inbound,ZZ,Unknown Region,0.983266 +therealreal.com,email-realreal.com,inbound,ZZ,Unknown Region,0 +thesource.ca,thesource.ca,inbound,ZZ,Unknown Region,0 +thewarehouse.co.nz,thewarehouse.co.nz,inbound,ZZ,Unknown Region,0 +thinkgeek.com,thinkgeek.com,inbound,ZZ,Unknown Region,1e-06 +thirtyonegifts.com,thirtyonegifts.com,inbound,ZZ,Unknown Region,0 +thomascook.com,eccluster.com,inbound,ZZ,Unknown Region,0 +ticketmaster.com,ticketmaster.com,inbound,ZZ,Unknown Region,0.001001 +ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,ZZ,Unknown Region,0 +timehop.com,timehop.com,inbound,ZZ,Unknown Region,1 +timeout.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +timewarnercable.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tinyletterapp.com,tinyletterapp.com,inbound,ZZ,Unknown Region,0 +tobi.com,messagebus.com,inbound,ZZ,Unknown Region,0 +topface.com,topface.com,inbound,ZZ,Unknown Region,1e-06 +topica.com,topica-silver-y.com,inbound,ZZ,Unknown Region,0 +topqpon.si,topqpon.si,inbound,ZZ,Unknown Region,0 +touchbase2.com,mailurja.com,inbound,ZZ,Unknown Region,0 +townnews-mail.com,townnews-mail.com,inbound,ZZ,Unknown Region,0 +trabajar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999997 +trabalhar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999994 +tradeloop.com,tradeloop.com,inbound,ZZ,Unknown Region,1 +trafficwave.net,trafficwave.net,inbound,ZZ,Unknown Region,0 +transittraveljobinsider.com,transittraveljobinsider.com,inbound,ZZ,Unknown Region,0 +transportexchangegroup.com,transportexchangegroup.com,inbound,ZZ,Unknown Region,0.998825 +travelchannel.com,travelchannel.com,inbound,ZZ,Unknown Region,0 +travelocity.com,travelocity.com,inbound,ZZ,Unknown Region,0.000195 +travelzoo.com,travelzoo.com,inbound,ZZ,Unknown Region,0 +trclient.com,trclient.com,inbound,ZZ,Unknown Region,0 +trello.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 +triongames.com,triongames.com,inbound,ZZ,Unknown Region,0.085718 +tripadvisor.com,tripadvisor.com,inbound,ZZ,Unknown Region,0.000588 +tripolis.com,tripolis.com,inbound,ZZ,Unknown Region,0 +trulia.com,trulia.com,inbound,ZZ,Unknown Region,0 +tsmmail.com,tsmmail.com,inbound,ZZ,Unknown Region,0 +tumblr.com,tumblr.com,inbound,ZZ,Unknown Region,1 +turbine.com,turbine.com,inbound,ZZ,Unknown Region,0 +turner.com,cnn.com,inbound,ZZ,Unknown Region,0 +twe-safelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +twitter.com,twitter.com,inbound,ZZ,Unknown Region,0.999969 +twoomail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 +ubivox.com,ubivox.com,inbound,ZZ,Unknown Region,0.964085 +uga.edu,outlook.com,inbound,ZZ,Unknown Region,1 +uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,ZZ,Unknown Region,0 +ulta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +ulta.com,ulta.com,inbound,ZZ,Unknown Region,0 +uniqlo-usa.com,uniqlo-usa.com,inbound,ZZ,Unknown Region,0 +unitedrepublic.org,unitedrepublic.org,inbound,ZZ,Unknown Region,1 +unosinsidersclub.com,unosinsidersclub.com,inbound,ZZ,Unknown Region,0 +urx.com.br,urx.com.br,inbound,ZZ,Unknown Region,0 +usaa.com,usaa.com,inbound,ZZ,Unknown Region,0.999997 +usahockey-email.com,usahockey-email.com,inbound,ZZ,Unknown Region,0 +usndr.com,usndr.com,inbound,ZZ,Unknown Region,1 +usx.com.br,uqx.com.br,inbound,ZZ,Unknown Region,0 +usx.com.br,usx.com.br,inbound,ZZ,Unknown Region,0 +usx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 +utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,ZZ,Unknown Region,0 +utx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 +uvarosa.com.br,uvarosa.com.br,inbound,ZZ,Unknown Region,0 +vakifbank.com.tr,vakifbank.com.tr,inbound,ZZ,Unknown Region,1 +vd.nl,emsecure.net,inbound,ZZ,Unknown Region,0 +venca.es,eccluster.com,inbound,ZZ,Unknown Region,0 +verizon.com,verizon.com,inbound,ZZ,Unknown Region,0.999816 +vfoutletvip.com,vfoutletvip.com,inbound,ZZ,Unknown Region,0 +vicinity.nl,picsrv.net,inbound,ZZ,Unknown Region,0.022107 +vietcombank.com.vn,vietcombank.com.vn,inbound,ZZ,Unknown Region,1 +viralsender.com,viralsender.com,inbound,ZZ,Unknown Region,0 +vistaprint.com,vistaprint.com,inbound,ZZ,Unknown Region,0 +vistaprint.com.au,vistaprint.com.au,inbound,ZZ,Unknown Region,0 +vitaminworld.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 +vk.com,vkontakte.ru,inbound,ZZ,Unknown Region,0 +vocus.com,vocus.com,inbound,ZZ,Unknown Region,0 +vovici.com,vovici.com,inbound,ZZ,Unknown Region,0 +vresp.com,verticalresponse.com,inbound,ZZ,Unknown Region,0 +vudu.com,vudu.com,inbound,ZZ,Unknown Region,0 +walmart.ca,walmart.ca,inbound,ZZ,Unknown Region,0 +walmart.com,walmart.com,inbound,ZZ,Unknown Region,0.315059 +warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,ZZ,Unknown Region,0 +way2sms.biz,way2sms.biz,inbound,ZZ,Unknown Region,0 +way2sms.in,way2sms.in,inbound,ZZ,Unknown Region,0 +way2smsemail.com,way2smsemail.com,inbound,ZZ,Unknown Region,0 +way2smsemails.com,way2smsemails.com,inbound,ZZ,Unknown Region,0 +way2smsmail.in,way2smsmail.in,inbound,ZZ,Unknown Region,0 +way2smsmails.com,way2smsmails.com,inbound,ZZ,Unknown Region,0 +wealthyaffiliate.com,wealthyaffiliate.com,inbound,ZZ,Unknown Region,1 +webmd.com,webmd.com,inbound,ZZ,Unknown Region,0 +wegottickets.com,wegottickets.com,inbound,ZZ,Unknown Region,0 +weheartit.com,weheartit.com,inbound,ZZ,Unknown Region,1 +wehkamp.nl,wehkamp.nl,inbound,ZZ,Unknown Region,0 +wellsfargo.com,wellsfargo.com,inbound,ZZ,Unknown Region,1 +wemakeprice.com,wemakeprice.com,inbound,ZZ,Unknown Region,0 +westwing.com.br,cust-cluster.com,inbound,ZZ,Unknown Region,0 +westwing.es,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +westwing.ru,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +wgbh.org,wgbh.org,inbound,ZZ,Unknown Region,0 +whaakky.com,whaakky.com,inbound,ZZ,Unknown Region,0 +whereareyounow.com,wayn.net,inbound,ZZ,Unknown Region,0 +whitehouse.gov,whitehouse.gov,inbound,ZZ,Unknown Region,0 +whitelabelpros.com,whitelabelpros.com,inbound,ZZ,Unknown Region,0 +wikia.com,wikia.com,inbound,ZZ,Unknown Region,0.289222 +wisdomitservices.com,infimail.com,inbound,ZZ,Unknown Region,0 +wolfmedia.us,wolfmedia.us,inbound,ZZ,Unknown Region,0 +wordfly.com,wordfly.com,inbound,ZZ,Unknown Region,0 +workhunter.net,workhunter.net,inbound,ZZ,Unknown Region,1 +worldwinner.com,worldwinner.com,inbound,ZZ,Unknown Region,0 +wowcher.co.uk,wowcher.co.uk,inbound,ZZ,Unknown Region,0 +wp.com,wordpress.com,inbound,ZZ,Unknown Region,0 +wpengine.com,wpengine.com,inbound,ZZ,Unknown Region,1 +writers-community.com,writers-community.com,inbound,ZZ,Unknown Region,0 +writersstore.com,writersstore.com,inbound,ZZ,Unknown Region,0 +wsjemail.com,wsjemail.com,inbound,ZZ,Unknown Region,0 +wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,ZZ,Unknown Region,0 +xbox.com,xbox.com,inbound,ZZ,Unknown Region,0 +xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,ZZ,Unknown Region,0 +xing.com,xing.com,inbound,ZZ,Unknown Region,0 +xxxconnect.com,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +yahoo-inc.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +yahoo.{...},yahoo.{...},inbound,ZZ,Unknown Region,0.999989 +yahoo.{...},yahoodns.net,outbound,ZZ,Unknown Region,1 +yahoogroups.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +yammer.com,yammer.com,inbound,ZZ,Unknown Region,1 +yapikredi.com.tr,yapikredi.com.tr,inbound,ZZ,Unknown Region,1 +yapstone.com,yapstone.com,inbound,ZZ,Unknown Region,0 +yesbank.in,yesbank.in,inbound,ZZ,Unknown Region,0 +yipit.com,yipit.com,inbound,ZZ,Unknown Region,1 +ymail.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +ymail.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +youravon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 +yournewsletters.net,everydayhealth.com,inbound,ZZ,Unknown Region,0 +youversion.com,youversion.com,inbound,ZZ,Unknown Region,1 +zappos.com,zappos.com,inbound,ZZ,Unknown Region,0.625377 +zattoo.com,sendnode.com,inbound,ZZ,Unknown Region,0 +zelonews.com.br,zelonews.com.br,inbound,ZZ,Unknown Region,0 +zendesk.com,zdsys.com,inbound,ZZ,Unknown Region,1 +zibmail.info,zibmail.info,inbound,ZZ,Unknown Region,0 +zillow.com,zillow.com,inbound,ZZ,Unknown Region,2.82543102656668e-07 +zinio.net,zinio.com,inbound,ZZ,Unknown Region,1 +zipalerts.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +zipalerts.com,zipalerts.com,inbound,ZZ,Unknown Region,1 +zlavadna.sk,zlavadna.sk,inbound,ZZ,Unknown Region,0 +zoom.com.br,zoom.com.br,inbound,ZZ,Unknown Region,0 +zoominternet.net,synacor.com,inbound,ZZ,Unknown Region,0 +zoosk.com,zoosk.com,inbound,ZZ,Unknown Region,0 +zorpia.com,zorpia.com,inbound,ZZ,Unknown Region,0.520147 +zovifashion.com,eccluster.com,inbound,ZZ,Unknown Region,0 +zyngamail.com,zyngamail.com,inbound,ZZ,Unknown Region,0 \ No newline at end of file From 6a1aa8e6b640e0c6504cdbb9635f88ec9929cd7d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 17:34:21 -0400 Subject: [PATCH 057/631] Update to latest config format. --- CheckSTARTTLS.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 205a4a779..9e5117b84 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -7,6 +7,7 @@ import socket import subprocess import re import json +import collections import dns.resolver from M2Crypto import X509 @@ -152,12 +153,8 @@ if __name__ == '__main__': if len(sys.argv) == 1: print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") - config = { - "address-domains": { - }, - "mx-domains": { - } - } + config = collections.defaultdict(dict) + for domain in open(sys.argv[1]).readlines(): domain = domain.strip() if not os.path.exists(domain): @@ -168,10 +165,10 @@ if __name__ == '__main__': min_version = min_tls_version(domain) if suffix != "": suffix_match = "." + suffix - config["address-domains"][domain] = { + config["acceptable-mxs"][domain] = { "accept-mx-domains": [suffix_match] } - config["mx-domains"][suffix_match] = { + config["tls-policies"][suffix_match] = { "require-tls": True, "min-tls-version": min_version } From 749c4e39e0fa539a36cfa717a07554153ff97406 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 17:34:40 -0400 Subject: [PATCH 058/631] Update meta-config with latest domains. --- starttls-everywhere.json | 93 +++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 5dd487f9a..db76745ef 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,15 @@ { - "address-domains": { + "acceptable-mxs": { + "163.com": { + "accept-mx-domains": [ + ".163.com" + ] + }, + "aol.com": { + "accept-mx-domains": [ + ".aol.com" + ] + }, "craigslist.org": { "accept-mx-domains": [ ".craigslist.org" @@ -10,19 +20,49 @@ ".google.com" ] }, - "interia.pl": { + "hotmail.com": { "accept-mx-domains": [ - ".interia.pl" + ".outlook.com" ] }, - "marktplaats.nl": { + "icloud.com": { "accept-mx-domains": [ - ".marktplaats.nl" + ".icloud.com" ] }, - "rambler.ru": { + "live.com": { "accept-mx-domains": [ - ".rambler.ru" + ".outlook.com" + ] + }, + "mac.com": { + "accept-mx-domains": [ + ".icloud.com" + ] + }, + "me.com": { + "accept-mx-domains": [ + ".icloud.com" + ] + }, + "msn.com": { + "accept-mx-domains": [ + ".outlook.com" + ] + }, + "naver.com": { + "accept-mx-domains": [ + ".naver.com" + ] + }, + "outlook.com": { + "accept-mx-domains": [ + ".outlook.com" + ] + }, + "qq.com": { + "accept-mx-domains": [ + ".qq.com" ] }, "rocketmail.com": { @@ -45,9 +85,14 @@ ".yahoo.com" ] }, - "sompo-japan.co.jp": { + "shaw.ca": { "accept-mx-domains": [ - ".psmtp.com" + ".shaw.ca" + ] + }, + "sympatico.ca": { + "accept-mx-domains": [ + ".outlook.com" ] }, "t-online.de": { @@ -60,12 +105,12 @@ ".wp.pl" ] }, - "yahoo.co.uk": { + "yahoo.com": { "accept-mx-domains": [ ".yahoo.com" ] }, - "yahoo.com": { + "yahoogroups.com": { "accept-mx-domains": [ ".yahoo.com" ] @@ -81,7 +126,15 @@ ] } }, - "mx-domains": { + "tls-policies": { + ".163.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".aol.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, ".craigslist.org": { "min-tls-version": "TLSv1.1", "require-tls": true @@ -90,11 +143,15 @@ "min-tls-version": "TLSv1.1", "require-tls": true }, - ".interia.pl": { + ".icloud.com": { "min-tls-version": "TLSv1", "require-tls": true }, - ".marktplaats.nl": { + ".naver.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".outlook.com": { "min-tls-version": "TLSv1.1", "require-tls": true }, @@ -102,8 +159,12 @@ "min-tls-version": "TLSv1", "require-tls": true }, - ".rambler.ru": { - "min-tls-version": "TLSv1.1", + ".qq.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".shaw.ca": { + "min-tls-version": "TLSv1", "require-tls": true }, ".t-online.de": { From cebc6f9a205696d7f849bba6ebbd6af549849043 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 8 Aug 2014 13:28:01 -0400 Subject: [PATCH 059/631] ProcessGoogleSTARTTLSDomains -> latest CSV format. Also output in a more useful format, require >= 99% encrypted output in the CSV, hande .{...} domains, and manually add gmail.com. --- ProcessGoogleSTARTTLSDomains.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ProcessGoogleSTARTTLSDomains.py b/ProcessGoogleSTARTTLSDomains.py index abb2b3495..3078bd93a 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/ProcessGoogleSTARTTLSDomains.py @@ -15,8 +15,15 @@ from collections import defaultdict csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') d = defaultdict(set) -for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in csvreader: +# Google's report doesn't include gmail.com because it's local delivery, but we +# know they support STARTTLS, so manually include them. +d["gmail.com"] = set([1]) +for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader: if direction == "outbound": + # Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}. + # We're tryingto get a solid list of the relevant TLDs, but in the meantime + # just use .com. + address_suffix = address_suffix.replace("{...}", "com") try: d[address_suffix].add(float(fraction_encrypted)) except ValueError: @@ -24,4 +31,4 @@ for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in for address_suffix, fraction_encrypted in d.iteritems(): if min(fraction_encrypted) >= 0.99: - print min(fraction_encrypted), address_suffix + print address_suffix From 21ff3acf932c74455e2d50ac4679afbd5cfb27a4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:26:59 -0700 Subject: [PATCH 060/631] Set/list comprehensions are a bit more readable than lambdas --- CheckSTARTTLS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 9e5117b84..ffff05dce 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -24,7 +24,7 @@ def mkdirp(path): else: raise def extract_names(pem): - """Return a list of DNS subject names from PEM-encoded leaf cert.""" + """Return a set of DNS subject names from PEM-encoded leaf cert.""" leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) subj = leaf.get_subject() @@ -93,7 +93,7 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) - new_names = map(lambda n: public_suffix_list.get_public_suffix(n), new_names) + new_names = set(public_suffix_list.get_public_suffix(n) for in new_names) names.update(new_names) if len(names) >= 1: # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. From ff5810d78f0cdc4e46de03470aa685744b05fa91 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:36:04 -0700 Subject: [PATCH 061/631] Don't accept files on the command line that don't do anything --- CheckSTARTTLS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index ffff05dce..eda1761f0 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -150,7 +150,7 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) == 1: + if len(sys.argv) != 2: # XXX or accept multiple files as input print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = collections.defaultdict(dict) From 0bd8134e5fde7930de557c041adf69199ef2cd5a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:57:01 -0700 Subject: [PATCH 062/631] Comments (and code review in comment form) --- CheckSTARTTLS.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index eda1761f0..a9d8de9de 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -86,6 +86,8 @@ def valid_cert(filename): return False def check_certs(mail_domain): + # Return "" if any certs for any mx domains pointed to by mail_domain + # were invalid, and a public suffix for one if they were all valid names = set() for mx_hostname in os.listdir(mail_domain): filename = os.path.join(mail_domain, mx_hostname) @@ -141,6 +143,8 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): + # XXX comment this function and explain why we're using the + # filesystem rather than internal data structures for plumbing here print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') From 30ba7e930587ac40104c9bf9d6173e2f8706eb34 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:01:04 -0700 Subject: [PATCH 063/631] Deduplicate stray comment line --- MTAConfigGenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 4d2f3432a..061b9445f 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -6,7 +6,6 @@ import os, os.path def parse_line(line_data): """ - Return the left and right hand sides of stripped, non-comment postfix Return the (line number, left hand side, right hand side) of a stripped postfix config line. From 8c6d28ce9583a56fa07d342fdbaf9b4c72eddf0f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:01:33 -0700 Subject: [PATCH 064/631] Comment with some sample postfix log lines (both those we support already, and those we may want to in the future...) --- PostfixLogSummary.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index daec93db0..566651cc2 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -5,10 +5,26 @@ import collections import ConfigParser +# XXX There's more to be learned from postfix logs! Here's one sample +# observed during failures from the sender vagrant vm: + +# Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] +# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1] +# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1] +# Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix +# Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from= +# Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com> +# Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=, size=576, nrcpt=1 (queue active) +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) +# def get_counts(input, config): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) + # Typical line looks like: + # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) From 9cafcf1caf14ca8bb81bf61f9597014f6c060b71 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:15:15 -0700 Subject: [PATCH 065/631] Comment regexp --- PostfixLogSummary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 566651cc2..db3653a6f 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -25,6 +25,7 @@ def get_counts(input, config): counts = collections.defaultdict(lambda: collections.defaultdict(int)) # Typical line looks like: # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) + # ([^[]*) <--- any group of characters that is not "[" r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) From 78a55c3823b1794c9ff6d7e268e78e785d817b93 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:16:40 -0700 Subject: [PATCH 066/631] Question about postfix logparsing output --- PostfixLogSummary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index db3653a6f..1c9d64610 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -40,6 +40,7 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 if not seen_trusted: + # XXX aren't these outbound? How can the admin install certs? print "Didn't see any trusted connections. Need to install some certs?" return counts From e0edc1b7ec0867d2624ef56d33da79765d776554 Mon Sep 17 00:00:00 2001 From: jsha Date: Tue, 12 Aug 2014 12:18:32 -0400 Subject: [PATCH 067/631] Add link to mailing list. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e50630ab8..3f5818255 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ Jacob Hoffman-Andrews , Peter Eckersley +## Mailing List + +starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere + ## Background Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. From ad40618897ee3905a7e651443ba0fb624eebbfdf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 13 Aug 2014 10:49:50 -0400 Subject: [PATCH 068/631] Respond to pde's comments --- CheckSTARTTLS.py | 52 ++++++++++++++++++++++++-------------------- PostfixLogSummary.py | 7 +++--- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index a9d8de9de..7c9c94e0f 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -86,8 +86,10 @@ def valid_cert(filename): return False def check_certs(mail_domain): - # Return "" if any certs for any mx domains pointed to by mail_domain - # were invalid, and a public suffix for one if they were all valid + """ + Return "" if any certs for any mx domains pointed to by mail_domain + were invalid, and a public suffix for one if they were all valid + """ names = set() for mx_hostname in os.listdir(mail_domain): filename = os.path.join(mail_domain, mx_hostname) @@ -95,7 +97,7 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) - new_names = set(public_suffix_list.get_public_suffix(n) for in new_names) + new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names) names.update(new_names) if len(names) >= 1: # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. @@ -143,8 +145,11 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): - # XXX comment this function and explain why we're using the - # filesystem rather than internal data structures for plumbing here + """ + Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS. + Store the output in a directory with the same name as mail_domain to make + subsequent analysis faster. + """ print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') @@ -154,27 +159,28 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) != 2: # XXX or accept multiple files as input + if len(sys.argv) < 2: print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = collections.defaultdict(dict) - for domain in open(sys.argv[1]).readlines(): - domain = domain.strip() - if not os.path.exists(domain): - collect(domain) - if len(os.listdir(domain)) == 0: - continue - suffix = check_certs(domain) - min_version = min_tls_version(domain) - if suffix != "": - suffix_match = "." + suffix - config["acceptable-mxs"][domain] = { - "accept-mx-domains": [suffix_match] - } - config["tls-policies"][suffix_match] = { - "require-tls": True, - "min-tls-version": min_version - } + for input in sys.argv[1:]: + for domain in open(input).readlines(): + domain = domain.strip() + if not os.path.exists(domain): + collect(domain) + if len(os.listdir(domain)) == 0: + continue + suffix = check_certs(domain) + min_version = min_tls_version(domain) + if suffix != "": + suffix_match = "." + suffix + config["acceptable-mxs"][domain] = { + "accept-mx-domains": [suffix_match] + } + config["tls-policies"][suffix_match] = { + "require-tls": True, + "min-tls-version": min_version + } print json.dumps(config, indent=2, sort_keys=True) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 1c9d64610..b68c7f6c6 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -5,7 +5,7 @@ import collections import ConfigParser -# XXX There's more to be learned from postfix logs! Here's one sample +# TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: # Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] @@ -40,8 +40,9 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 if not seen_trusted: - # XXX aren't these outbound? How can the admin install certs? - print "Didn't see any trusted connections. Need to install some certs?" + # Postfix will only emit 'Trusted' if the certificate validates according to + # the set of trust roots (CA certs) configured in smtp_tls_CAfile. + print "Didn't see any trusted connections. Need to install some trust roots?" return counts def print_summary(counts): From 31e320d0a7832ca8d25032eeb321cce75f48b51b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 13 Aug 2014 15:59:50 -0400 Subject: [PATCH 069/631] Collect certs in a subdir. --- CheckSTARTTLS.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 7c9c94e0f..e8adc010e 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -14,6 +14,7 @@ from M2Crypto import X509 from publicsuffix import PublicSuffixList public_suffix_list = PublicSuffixList() +CERTS_OBSERVED = 'certs-observed' def mkdirp(path): try: @@ -63,7 +64,7 @@ def tls_connect(mx_host, mail_domain): return # Save a copy of the certificate for later analysis - with open(os.path.join(mail_domain, mx_host), "w") as f: + with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f: f.write(output) def valid_cert(filename): @@ -90,9 +91,12 @@ def check_certs(mail_domain): Return "" if any certs for any mx domains pointed to by mail_domain were invalid, and a public suffix for one if they were all valid """ + dir = os.path.join(CERTS_OBSERVED, mail_domain) + if not os.path.exists(dir): + collect(mail_domain) names = set() - for mx_hostname in os.listdir(mail_domain): - filename = os.path.join(mail_domain, mx_hostname) + for mx_hostname in os.listdir(dir): + filename = os.path.join(dir, mx_hostname) if not valid_cert(filename): return "" else: @@ -137,8 +141,8 @@ def supports_starttls(mx_host): def min_tls_version(mail_domain): protocols = [] - for mx_hostname in os.listdir(mail_domain): - filename = os.path.join(mail_domain, mx_hostname) + for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)): + filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname) contents = open(filename).read() protocol = re.findall("Protocol : (.*)", contents)[0] protocols.append(protocol) @@ -151,7 +155,7 @@ def collect(mail_domain): subsequent analysis faster. """ print "Checking domain %s" % mail_domain - mkdirp(mail_domain) + mkdirp(os.path.join(CERTS_OBSERVED, mail_domain)) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: mx_host = str(rdata.exchange).rstrip(".") @@ -167,10 +171,6 @@ if __name__ == '__main__': for input in sys.argv[1:]: for domain in open(input).readlines(): domain = domain.strip() - if not os.path.exists(domain): - collect(domain) - if len(os.listdir(domain)) == 0: - continue suffix = check_certs(domain) min_version = min_tls_version(domain) if suffix != "": From 42c63cb6ddb4864f155a3a844f512a1add61b50f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 8 Sep 2014 16:25:40 -0400 Subject: [PATCH 070/631] More informative message for RDNS fail. --- CheckSTARTTLS.py | 11 ++++++++--- requirements.txt | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index e8adc010e..e5c5a4323 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -135,8 +135,13 @@ def supports_starttls(mx_host): except socket.error as e: print "Connection to %s failed: %s" % (mx_host, e.strerror) return False - except smtplib.SMTPException: - print "No STARTTLS support on %s" % mx_host + except smtplib.SMTPException, e: + # In order to talk to some hosts, you need to run this from a host that has a + # reverse DNS entry. AWS instances all have reverse DNS, as an example. + if e[0] == 554: + print e[1] + else: + print "No STARTTLS support on %s" % mx_host, e[0] return False def min_tls_version(mail_domain): @@ -172,8 +177,8 @@ if __name__ == '__main__': for domain in open(input).readlines(): domain = domain.strip() suffix = check_certs(domain) - min_version = min_tls_version(domain) if suffix != "": + min_version = min_tls_version(domain) suffix_match = "." + suffix config["acceptable-mxs"][domain] = { "accept-mx-domains": [suffix_match] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..891e5809d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dnspython +publicsuffix +m2crypto From 622fc72dc13d4d9de2c95875e4ef98fff6895b05 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 10 Sep 2014 17:36:46 -0400 Subject: [PATCH 071/631] Treat min-tls-version as a minimum. Fixes #5. --- MTAConfigGenerator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 061b9445f..a733ab27a 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -128,7 +128,14 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.tls_policies[mx_domain] entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: - entry += " protocols=" + mx_policy["min-tls-version"] + if mx_policy["min-tls-version"].lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy["min-tls-version"].lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy["min-tls-version"].lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy["min-tls-version"] self.policy_lines.append(entry) f = open(self.policy_file, "w") From a1d016d0312c6f4a9e97ed772c0df90031bda7b5 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 13 Oct 2014 15:57:46 -0400 Subject: [PATCH 072/631] Add motivating examples to README --- PostfixLogSummary.py | 25 +++++++++++++++++++------ README.md | 8 ++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index b68c7f6c6..0348432b0 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -18,18 +18,27 @@ import ConfigParser # Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 # Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: # Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) -# +# +# Also: +# Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) def get_counts(input, config): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) + tls_deferred = collections.defaultdict(int) # Typical line looks like: # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) + # indicate a problem that should be alerted on. # ([^[]*) <--- any group of characters that is not "[" - r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + # Log lines for when a message is deferred for a TLS-related reason. These + deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS") + # Log lines for when a TLS connection was successfully established. These can + # indicate the difference between Untrusted, Trusted, and Verified certs. + connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: - result = r.search(line) - if result: + deferred = deferred_re.search(line) + connected = connected_re.search(line) + if connected: validation = result.group(1) mx_hostname = result.group(2).lower() if validation == "Trusted" or validation == "Verified": @@ -39,11 +48,14 @@ def get_counts(input, config): for d in address_domains: counts[d][validation] += 1 counts[d]["all"] += 1 + elif deferred: + mx_hostname = result.group(1).lower() + tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to # the set of trust roots (CA certs) configured in smtp_tls_CAfile. print "Didn't see any trusted connections. Need to install some trust roots?" - return counts + return (counts, tls_deferred) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -54,5 +66,6 @@ def print_summary(counts): if __name__ == "__main__": config = ConfigParser.Config("starttls-everywhere.json") - counts = get_counts(sys.stdin, config) + (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) + print tls_deferred diff --git a/README.md b/README.md index 3f5818255..8d6160c9a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d * Develop a fully-decentralized solution. * Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. +## Motivating examples + +* [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to + prevent negotiating an encrypted connection] + (https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) +* [Unknown party removes STARTTLS flag from all SMTP connections leaving + Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) + ## Threat model Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. From 726afb8b95f7759724c487f1163d582119333332 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:32 -0400 Subject: [PATCH 073/631] Install CAfile on config generation. --- MTAConfigGenerator.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index a733ab27a..051d9676d 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,11 +28,16 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() + if not os.access(self.postfix_cf_file, os.W_OK): + raise Exception("Can't write to %s, please re-run as root." + % self.postfix_cf_file) + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") + MTAConfigGenerator.__init__(self, policy_config) self.wrangle_existing_config() self.set_domainwise_tls_policies() + self.update_CAfile() os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): @@ -49,7 +54,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): values = map(parse_line, l) if len(set(values)) > 1: if self.fixup: - #print "Scheduling deletions:" + `values` conflicting_lines = [num for num,_var,val in values] self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) @@ -57,7 +61,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): raise ExistingConfigError, "Conflicting existing config values " + `l` val = values[0][2] if val not in acceptable: - #print "Scheduling deletions:" + `values` if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) @@ -86,6 +89,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): policy_cf_entry = "texthash:" + self.policy_file self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) self.maybe_add_config_lines() @@ -109,7 +113,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += line self.new_cf += sep + new_cf_lines - #print self.new_cf f = open(self.fn, "w") f.write(self.new_cf) f.close() @@ -142,6 +145,10 @@ class PostfixConfigGenerator(MTAConfigGenerator): f.write("\n".join(self.policy_lines) + "\n") f.close() + def update_CAfile(self): + os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + + self.ca_file) + if __name__ == "__main__": import ConfigParser if len(sys.argv) != 3: @@ -150,3 +157,4 @@ if __name__ == "__main__": c = ConfigParser.Config(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + print "Done." From 1d47acddfdffe93f75c0d07ae0f127dd1693d984 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:44 -0400 Subject: [PATCH 074/631] Remove extraneous print of config. --- ConfigParser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ConfigParser.py b/ConfigParser.py index 2d7c88ada..d1c413f74 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -90,7 +90,6 @@ class Config: # XXX is it ever permissible to have a domain with an acceptable-mx # that does not point to a TLS security policy? If not, check/warn/fail # here - print self.tls_policies def get_address_domains(self, mx_hostname): labels = mx_hostname.split(".") From 828c00b758063172990073bd4b667b61c1a30699 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:49 -0400 Subject: [PATCH 075/631] Find deferred lines in log summary. Also add a cron mode that only emits output if there's something wrong. --- PostfixLogSummary.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 0348432b0..9b2f9c3b1 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -2,6 +2,7 @@ import re import sys import collections +import argparse import ConfigParser @@ -39,8 +40,8 @@ def get_counts(input, config): deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: - validation = result.group(1) - mx_hostname = result.group(2).lower() + validation = connected.group(1) + mx_hostname = connected.group(2).lower() if validation == "Trusted" or validation == "Verified": seen_trusted = True address_domains = config.get_address_domains(mx_hostname) @@ -49,13 +50,13 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 elif deferred: - mx_hostname = result.group(1).lower() + mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to # the set of trust roots (CA certs) configured in smtp_tls_CAfile. print "Didn't see any trusted connections. Need to install some trust roots?" - return (counts, tls_deferred) + return (counts, tls_deferred, seen_trusted) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -65,7 +66,19 @@ def print_summary(counts): print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": + arg_parser = argparse.ArgumentParser(description='This is a PyMOTW sample program') + arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) + args = arg_parser.parse_args() + config = ConfigParser.Config("starttls-everywhere.json") - (counts, tls_deferred) = get_counts(sys.stdin, config) - print_summary(counts) - print tls_deferred + (counts, tls_deferred, seen_trusted) = get_counts(sys.stdin, config) + + # If not running in cron, print an overall summary of log lines seen from known hosts. + if not args.cron: + print_summary(counts) + if not seen_trusted: + print 'No Trusted connections seen! Probably need to install a CAfile.' + + if len(tls_deferred) > 0: + print "Some mail was deferred due to TLS problems:" + print tls_deferred From f04e8259a9aef0e6547fe22884268581461512e2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 20 Oct 2014 09:29:22 -0400 Subject: [PATCH 076/631] Protocols separated by colons, not commas. --- MTAConfigGenerator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 051d9676d..5ca8f5aee 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -132,11 +132,11 @@ class PostfixConfigGenerator(MTAConfigGenerator): entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" + entry += " protocols=!SSLv2:!SSLv3" elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1" elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: print mx_policy["min-tls-version"] self.policy_lines.append(entry) From 613a8f5e8877878f3d0b169a045b0f6b2b31adec Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 20 Oct 2014 16:15:13 -0400 Subject: [PATCH 077/631] Improve cronjob operation to only process diffs. --- PostfixLogSummary.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 9b2f9c3b1..be130d958 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -1,11 +1,15 @@ #!/usr/bin/python2.7 +import argparse +import collections +import os import re import sys -import collections -import argparse +import time import ConfigParser +TIME_FORMAT = "%b %d %H:%M:%S" + # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -22,7 +26,7 @@ import ConfigParser # # Also: # Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) -def get_counts(input, config): +def get_counts(input, config, earliest_timestamp): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) @@ -32,11 +36,15 @@ def get_counts(input, config): # indicate a problem that should be alerted on. # ([^[]*) <--- any group of characters that is not "[" # Log lines for when a message is deferred for a TLS-related reason. These - deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS") + deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") # Log lines for when a TLS connection was successfully established. These can # indicate the difference between Untrusted, Trusted, and Verified certs. connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + timestamp = 0 for line in sys.stdin: + timestamp = time.strptime(line[0:15], TIME_FORMAT) + if timestamp < earliest_timestamp: + continue deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: @@ -52,11 +60,7 @@ def get_counts(input, config): elif deferred: mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 - if not seen_trusted: - # Postfix will only emit 'Trusted' if the certificate validates according to - # the set of trust roots (CA certs) configured in smtp_tls_CAfile. - print "Didn't see any trusted connections. Need to install some trust roots?" - return (counts, tls_deferred, seen_trusted) + return (counts, tls_deferred, seen_trusted, timestamp) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -71,7 +75,14 @@ if __name__ == "__main__": args = arg_parser.parse_args() config = ConfigParser.Config("starttls-everywhere.json") - (counts, tls_deferred, seen_trusted) = get_counts(sys.stdin, config) + + last_timestamp_processed = 0 + timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' + if os.path.isfile(timestamp_file): + last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) + (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) + with open(timestamp_file, "w") as f: + f.write(time.strftime(TIME_FORMAT, latest_timestamp)) # If not running in cron, print an overall summary of log lines seen from known hosts. if not args.cron: @@ -81,4 +92,5 @@ if __name__ == "__main__": if len(tls_deferred) > 0: print "Some mail was deferred due to TLS problems:" - print tls_deferred + for (k, v) in tls_deferred.iteritems(): + print "%s: %s" % (k, v) From fe17c873c0e7bd0f1e6debdb5cc1e6df11bcb0a3 Mon Sep 17 00:00:00 2001 From: pypoet Date: Tue, 13 Oct 2015 17:09:03 -0400 Subject: [PATCH 078/631] Initial re-vamp of the Config object to centralize validation and lay the basis for making compositions of configs and overrides. Lots of TODOs, be warned. --- Config.py | 315 ++++++++++++++++++++++++++++++++++++++++ bigger_test_config.json | 36 +++++ 2 files changed, 351 insertions(+) create mode 100644 Config.py create mode 100644 bigger_test_config.json diff --git a/Config.py b/Config.py new file mode 100644 index 000000000..157c2c2e4 --- /dev/null +++ b/Config.py @@ -0,0 +1,315 @@ +from datetime import datetime +import json + + +"""Idea here being to start with something that is decomposed so it's easier to +make do json in *and* out, differences between configs and config extension. +""" + +def parse_bool_from_json(value, attr_name): + if value in ('true', '1', 1, 'yes'): + bool_value = True + elif value in ('false', '0', 0, 'no'): + bool_value = False + elif value in (True, False): + bool_value = value + else: + raise ValueError('Config value %s is an invalid boolean value.' % attr_name) + return bool_value + + +def parse_timestamp(value, attr_name): + #TODO support full extended timestamp "2014-06-06T14:30:16+00:00" as well + if isinstance(value, datetime): + dt = value + else: + try: + ts = int(value) + dt = datetime.fromtimestamp(ts) + except: + raise ValueError('Config value %s is an invalid timestamp integer.' % attr_name) + return dt + + +def verify_member_of(value, member_list, attr_name): + if value not in member_list: + raise ValueError('Config value "%s" must be one of (%s)' % ( + attr_name, ', '.join(member_list)) + ) + return value + + +def verify_string(value, attr_name, max_length=200): + if not isinstance(value, (str, unicode)): + raise TypeError('Config value %s must be a string.' % attr_name) + if len(value) > max_length: + raise ValueError('Config value %s is too long.' % attr_name) + return value + + +class Config(object): + """Config container for StartTLS Everywhere configuration. + + Intended as a simple container that unifies where validatation occurs, + and is capable of comparing configs to warn of things like changing + certificate fingerprints from one scan to the next. + + There is a one to one mapping of the object attributes to the JSON + object keys, albeit with dashes replaced with underscores. + """ + + def __init__(self): + # container for validated properties with JSON names + self._data = {} + + self.tls_policies = [] + self.acceptable_mxs = [] + + def __add__(self, other_config): + """Allow addition but not really of *full* configs, need to flesh that out.""" + #TODO add this + raise NotImplemented + + def __repr__(self): + #TODO fix this generically, and maybe put it in the inheritence tree + s = '' % (self._data.iteritems()) + return s + + def update(self, other_config): + """Update properties of config from a 'newer' config and force verification.""" + #TODO add this + raise NotImplemented + + def load_from_json_file(self, json_filename, f_open=open): + #TODO add robust catching and checking + # try: + with f_open(json_filename, 'r') as f: + json_str = f.read() + json_dict = json.loads(json_str) + # except oserr + # except json parse err + self.from_json_dict(json_dict) + + def from_json_dict(self, json_dict): + """Assign JSON data to Config properties and declare sub-objects. + + Let's property verification methods do the heavy lifting and mostly + maps between the JSON config names and attributes. Keeps track of + unused variables and warns about them. + """ + for key, val in json_dict.iteritems(): + if key == 'author': + self.author = val + elif key == 'comment': + self.comment = val + elif key == 'expires': + self.expires = val + elif key == 'timestamp': + self.timestamp = val + elif key == 'tls-policies': + self.tls_policies = self.make_tls_policy_dict(val) + elif key == 'acceptable-mxs': + self.acceptable_mxs = self.make_acceptable_mxs_dict(val) + else: + #TODO log warning + print 'Unknown attribute "%s", skipping' % key + + def to_json(self): + #TODO implement output and make sure it can be re-input with identical results + raise NotImplemented + + @property + def author(self): + return self._data.get('author') + + @author.setter + def author(self, value): + self._data['author'] = verify_string(value, 'author') + + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + + @property + def expires(self): + return self._data.get('expires') + + @expires.setter + def expires(self, value): + self._data['expires'] = parse_timestamp(value, 'expires') + + @property + def timestamp(self): + return self._data.get('timestamp') + + @timestamp.setter + def timestamp(self, value): + self._data['timestamp'] = parse_timestamp(value, 'timestamp') + + def make_tls_policy_dict(self, policy_dict): + tls_policy_dict = {} + for domain_suffix, settings in policy_dict.iteritems(): + new_domain_policy = TLSPolicy(domain_suffix) + #TODO define config errs and use + #try + new_domain_policy.from_json_dict(settings) + #except config err + tls_policy_dict[domain_suffix] = new_domain_policy + return tls_policy_dict + + def make_acceptable_mxs_dict(self, mxs_dict): + acceptable_mxs_dict = {} + for domain, settings in mxs_dict.iteritems(): + new_domain_policy = AcceptableMX(domain) + #TODO define config errs and use + #try + new_domain_policy.from_json_dict(settings) + #except config err + acceptable_mxs_dict[domain] = new_domain_policy + return acceptable_mxs_dict + + def is_valid(self): + #TODO implement with checks to make sure domains don't overlap + # and every acceptable mx has a tls policy, etc. + raise NotImplemented + + +class TLSPolicy(object): + + ENFORCE_MODES = ('enforce', 'log-only') + TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') + + def __init__(self, domain_suffix): + # container for validated properties with JSON names + self._data = {} + self.domain_suffix = domain_suffix + + #TODO add me + self.accept_spki_hashs = None + #TODO add me + self.error_notification = None + + def from_json_dict(self, json_dict): + for key, val in json_dict.iteritems(): + if key == 'comment': + self.comment = val + elif key == 'enforce-mode': + self.enforce_mode = val + elif key == 'min-tls-version': + self.min_tls_version = val + elif key == 'require-tls': + self.require_tls = val + elif key == 'require-valid-certificate': + self.require_valid_certificate = val + else: + #TODO wat, log this instead + print 'Unknown key %s' % key + + def is_valid(self): + """Do simple check that config contains all required values. + + Should find a way to expose easily which config values + are required, at least place in error messages such that + incomplete configs will expose it. + """ + required_attrs = ('enforce-mode', 'min-tls-version', + 'require-tls') + values_set = [self._data.get(attr) for attr in required_attrs] + if not all(values_set): + return False + else: + return True + + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + + @property + def enforce_mode(self): + return self._data.get('enforce-mode') + + @enforce_mode.setter + def enforce_mode(self, value): + self._data['enforce-mode'] = verify_member_of(value, self.ENFORCE_MODES, 'enforce-mode') + + @property + def min_tls_version(self): + return self._data.get('min-tls-version') + + @min_tls_version.setter + def min_tls_version(self, value): + """Should this be dealing only with strings processed by map ... lower()?""" + tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] + tls_versions.extend(self.TLS_VERSIONS) + self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') + + @property + def require_tls(self): + return self._data.get('require-tls') + + @require_tls.setter + def require_tls(self, value): + self._data['require-tls'] = parse_bool_from_json(value, 'require-tls') + + @property + def require_valid_certificate(self): + return self._data.get('require-valid-certificate') + + @require_valid_certificate.setter + def require_valid_certificate(self, value): + self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') + + +class AcceptableMX(object): + """Holds acceptable MX domain suffixes for a single mail serving domain. + + Such as for gmail.com that single mail serving suffix domain is: + gmail-smtp-in.l.google.com. + + Configuration of the acceptable MX suffix domains must match up with TLS policies + for the suffix domains. + """ + def __init__(self, domain): + self.domain = domain + # container for validated properties with JSON names + self._data = {} + self._data['accept-mx-domains'] = [] + + def add_acceptable_mx(self, domain_suffix): + unique_domain_suffixes = set(self._data['accept-mx-domains']) + unique_domain_suffixes.add(domain_suffix) + self._data['accept-mx-domains'] = list(unique_domain_suffixes) + + def is_valid(self): + """Check to make sure there is one acceptable domain suffix. + + This will need to be updated once we can actually test and support + for more than one acceptable domain suffix. + + TODO: could make this object double check the data it is given with + DNS queries. + """ + if len(self._data['accept-mx-domains']) != 1: + return False + else: + return True + + def from_json_dict(self, json_dict): + for key, val in json_dict.iteritems(): + if key == 'accept-mx-domains': + if isinstance(val, list): + for domain_suffix in val: + self.add_acceptable_mx(domain_suffix) + else: + self.add_acceptable_mx(val) + else: + #TODO add logging for this + print 'warning: unknown key %s' % key diff --git a/bigger_test_config.json b/bigger_test_config.json new file mode 100644 index 000000000..e0697fc85 --- /dev/null +++ b/bigger_test_config.json @@ -0,0 +1,36 @@ +{ + "timestamp": 1401414363, + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1404242424, + "tls-policies": { + ".yahoodns.net": { + "require-valid-certificate": true + }, + ".eff.org": { + "require-tls": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "enforce", + "accept-spki-hashes": [ + "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + ] + }, + ".google.com": { + "require-valid-certificate": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "log-only", + "error-notification": "https://google.com/post/reports/here" + } + }, + "acceptable-mxs": { + "yahoo.com": { + "accept-mx-domains": [".yahoodns.net"] + }, + "gmail.com": { + "accept-mx-domains": [".google.com"] + }, + "eff.org": { + "accept-mx-domains": [".eff.org"] + } + } +} From 147f58bdbc5cc08c7abe43d304df6efd2ad86fe2 Mon Sep 17 00:00:00 2001 From: pypoet Date: Wed, 14 Oct 2015 02:49:34 -0400 Subject: [PATCH 079/631] Rounds out missing features and is now on par with ConfigParser.py. Still missing logging, composibility and a couple of attributes. --- Config.py | 165 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 64 deletions(-) diff --git a/Config.py b/Config.py index 157c2c2e4..93504dd5a 100644 --- a/Config.py +++ b/Config.py @@ -1,11 +1,14 @@ from datetime import datetime +from dateutil import parser import json +import pprint """Idea here being to start with something that is decomposed so it's easier to make do json in *and* out, differences between configs and config extension. """ + def parse_bool_from_json(value, attr_name): if value in ('true', '1', 1, 'yes'): bool_value = True @@ -14,26 +17,27 @@ def parse_bool_from_json(value, attr_name): elif value in (True, False): bool_value = value else: - raise ValueError('Config value %s is an invalid boolean value.' % attr_name) + raise ConfigError('Config value %s is an invalid boolean value.' % attr_name) return bool_value def parse_timestamp(value, attr_name): - #TODO support full extended timestamp "2014-06-06T14:30:16+00:00" as well if isinstance(value, datetime): - dt = value - else: - try: - ts = int(value) - dt = datetime.fromtimestamp(ts) - except: - raise ValueError('Config value %s is an invalid timestamp integer.' % attr_name) - return dt + return value + try: + ts = int(value) + return datetime.fromtimestamp(ts) + except (TypeError, ValueError): + pass + try: + return parser.parse(value) + except (TypeError, ValueError): + raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) def verify_member_of(value, member_list, attr_name): if value not in member_list: - raise ValueError('Config value "%s" must be one of (%s)' % ( + raise ConfigError('Config value "%s" must be one of (%s)' % ( attr_name, ', '.join(member_list)) ) return value @@ -41,13 +45,67 @@ def verify_member_of(value, member_list, attr_name): def verify_string(value, attr_name, max_length=200): if not isinstance(value, (str, unicode)): - raise TypeError('Config value %s must be a string.' % attr_name) + raise ConfigError('Config value %s must be a string.' % attr_name) if len(value) > max_length: - raise ValueError('Config value %s is too long.' % attr_name) + raise ConfigError('Config value %s is too long.' % attr_name) return value -class Config(object): +def to_dict(config_dict): + """Cleans up BaseConfig children to be serialized.""" + d = {} + for key, val in config_dict.iteritems(): + if isinstance(val, BaseConfig): + d[key] = to_dict(val._data) + elif isinstance(val, datetime): + d[key] = val.strftime('%Y-%m-%dT%H:%M:%S%z') + elif isinstance(val, dict): + d[key] = to_dict(val) + else: + d[key] = val + return d + + +class BaseConfig(object): + """Top level config class for common methods.""" + + def __init__(self): + # container for validated properties with JSON names + self._data = {} + + def __repr__(self): + s = '< %s %s >' % (self.__class__.__name__, + pprint.pformat(self._data)) + return s + + def to_json(self): + d = to_dict(self._data) + return json.dumps(d) + + def write_to_json_file(self, json_filename, f_open=open): + data = self.to_json() + try: + with f_open(json_filename, 'w') as f: + f.write(data) + except IOError: + raise + + def load_from_json_file(self, json_filename, f_open=open): + try: + with f_open(json_filename, 'r') as f: + json_str = f.read() + json_dict = json.loads(json_str) + except IOError: + raise + except ValueError: + raise ConfigError('No valid JSON found in file: %s' % json_filename) + self.from_json_dict(json_dict) + + def from_json_dict(self, json_dict): + raise NotImplmented('BaseConfig should not be populated.') + + +class Config(BaseConfig): """Config container for StartTLS Everywhere configuration. Intended as a simple container that unifies where validatation occurs, @@ -59,37 +117,20 @@ class Config(object): """ def __init__(self): - # container for validated properties with JSON names - self._data = {} - - self.tls_policies = [] - self.acceptable_mxs = [] + super(self.__class__, self).__init__() + self._data['tls-policies'] = {} + self._data['acceptable-mxs'] = {} def __add__(self, other_config): """Allow addition but not really of *full* configs, need to flesh that out.""" #TODO add this raise NotImplemented - def __repr__(self): - #TODO fix this generically, and maybe put it in the inheritence tree - s = '' % (self._data.iteritems()) - return s - def update(self, other_config): """Update properties of config from a 'newer' config and force verification.""" #TODO add this raise NotImplemented - def load_from_json_file(self, json_filename, f_open=open): - #TODO add robust catching and checking - # try: - with f_open(json_filename, 'r') as f: - json_str = f.read() - json_dict = json.loads(json_str) - # except oserr - # except json parse err - self.from_json_dict(json_dict) - def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -107,17 +148,13 @@ class Config(object): elif key == 'timestamp': self.timestamp = val elif key == 'tls-policies': - self.tls_policies = self.make_tls_policy_dict(val) + self.make_tls_policy_dict(val) elif key == 'acceptable-mxs': - self.acceptable_mxs = self.make_acceptable_mxs_dict(val) + self.make_acceptable_mxs_dict(val) else: #TODO log warning print 'Unknown attribute "%s", skipping' % key - def to_json(self): - #TODO implement output and make sure it can be re-input with identical results - raise NotImplemented - @property def author(self): return self._data.get('author') @@ -151,47 +188,42 @@ class Config(object): self._data['timestamp'] = parse_timestamp(value, 'timestamp') def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = {} + tls_policy_dict = self._data['tls-policies'] for domain_suffix, settings in policy_dict.iteritems(): new_domain_policy = TLSPolicy(domain_suffix) - #TODO define config errs and use - #try - new_domain_policy.from_json_dict(settings) - #except config err + try: + new_domain_policy.from_json_dict(settings) + except ConfigError as e: + raise tls_policy_dict[domain_suffix] = new_domain_policy - return tls_policy_dict def make_acceptable_mxs_dict(self, mxs_dict): - acceptable_mxs_dict = {} + acceptable_mxs_dict = self._data['acceptable-mxs'] for domain, settings in mxs_dict.iteritems(): new_domain_policy = AcceptableMX(domain) - #TODO define config errs and use - #try - new_domain_policy.from_json_dict(settings) - #except config err + try: + new_domain_policy.from_json_dict(settings) + except ConfigError as e: + raise acceptable_mxs_dict[domain] = new_domain_policy - return acceptable_mxs_dict def is_valid(self): #TODO implement with checks to make sure domains don't overlap # and every acceptable mx has a tls policy, etc. raise NotImplemented + - -class TLSPolicy(object): +class TLSPolicy(BaseConfig): ENFORCE_MODES = ('enforce', 'log-only') TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') def __init__(self, domain_suffix): - # container for validated properties with JSON names - self._data = {} + super(self.__class__, self).__init__() self.domain_suffix = domain_suffix - - #TODO add me - self.accept_spki_hashs = None - #TODO add me - self.error_notification = None + #TODO add support for two designed but yet unsupported attrs + # self._data['accept-spki-hashs'] = None + # self._data['error-notification'] = None def from_json_dict(self, json_dict): for key, val in json_dict.iteritems(): @@ -268,7 +300,7 @@ class TLSPolicy(object): self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') -class AcceptableMX(object): +class AcceptableMX(BaseConfig): """Holds acceptable MX domain suffixes for a single mail serving domain. Such as for gmail.com that single mail serving suffix domain is: @@ -278,9 +310,8 @@ class AcceptableMX(object): for the suffix domains. """ def __init__(self, domain): + super(self.__class__, self).__init__() self.domain = domain - # container for validated properties with JSON names - self._data = {} self._data['accept-mx-domains'] = [] def add_acceptable_mx(self, domain_suffix): @@ -313,3 +344,9 @@ class AcceptableMX(object): else: #TODO add logging for this print 'warning: unknown key %s' % key + + +class ConfigError(ValueError): + def __init__(self, message): + super(self.__class__, self).__init__(message) + From 6da5de6b19d3247c49290631dd4557a05f1e230b Mon Sep 17 00:00:00 2001 From: pypoet Date: Fri, 16 Oct 2015 00:57:42 -0400 Subject: [PATCH 080/631] Beginnings of generic config composibility in place. --- Config.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/Config.py b/Config.py index 93504dd5a..740de35c0 100644 --- a/Config.py +++ b/Config.py @@ -67,7 +67,18 @@ def to_dict(config_dict): class BaseConfig(object): - """Top level config class for common methods.""" + """Top level config class for common methods. + + Requirements for using class: + - list all properties with getters *and* setters in class + variable 'config_properties' + - __init__ of child classes must be callable with *only* + keyword arguments to allow method calls to update to create + a new config + ... more ... + """ + + config_properties = [] def __init__(self): # container for validated properties with JSON names @@ -78,6 +89,27 @@ class BaseConfig(object): pprint.pformat(self._data)) return s + def update(self, newer_config, merge=False, **kwargs): + fresh_config = self.__class__(**kwargs) + if not isinstance(newer_config, self.__class__): + raise ConfigError('Attempting to update a %s with a %s' % ( + self.__class__, + newer_config.__class__)) + for prop_name in self.config_properties: + prop = self.__class__.__dict__.get(prop_name) + assert prop + new_value = prop.fget(newer_config) + old_value = prop.fget(self) + if new_value is not None: + prop.fset(fresh_config, new_value) + elif merge and old_value is not None: + prop.fset(fresh_config, old_value) + return fresh_config + + def merge(self, newer_config, **kwargs): + kwargs['merge'] = True + return self.update(newer_config, **kwargs) + def to_json(self): d = to_dict(self._data) return json.dumps(d) @@ -129,7 +161,10 @@ class Config(BaseConfig): def update(self, other_config): """Update properties of config from a 'newer' config and force verification.""" #TODO add this + new_config = Config() raise NotImplemented + + def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -187,6 +222,14 @@ class Config(BaseConfig): def timestamp(self, value): self._data['timestamp'] = parse_timestamp(value, 'timestamp') + @property + def tls_policies(self): + return self._data.get('tls-policies') + + @property + def acceptable_mxs(self): + return self._data.get('acceptable-mxs') + def make_tls_policy_dict(self, policy_dict): tls_policy_dict = self._data['tls-policies'] for domain_suffix, settings in policy_dict.iteritems(): @@ -208,9 +251,19 @@ class Config(BaseConfig): acceptable_mxs_dict[domain] = new_domain_policy def is_valid(self): - #TODO implement with checks to make sure domains don't overlap - # and every acceptable mx has a tls policy, etc. - raise NotImplemented + #TODO implement checks to make sure domains don't overlap + #TODO add debug logging for troubleshooting stake + for mx_config in self.acceptable_mxs.values(): + if not mx_config.is_valid(): + return False + for domain_suffix in mx_config.accept_mx_domains: + # check to make sure every accepted MX has a TLS policy + if not domain_suffix in self.tls_policies: + return False + for tls_config in self.tls_policies.values(): + if not tls_config.is_valid(): + return False + return True class TLSPolicy(BaseConfig): @@ -218,7 +271,10 @@ class TLSPolicy(BaseConfig): ENFORCE_MODES = ('enforce', 'log-only') TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') - def __init__(self, domain_suffix): + config_properties = ['comment', 'enforce_mode', 'min_tls_version', + 'require_tls', 'require_valid_certificate'] + + def __init__(self, domain_suffix=None): super(self.__class__, self).__init__() self.domain_suffix = domain_suffix #TODO add support for two designed but yet unsupported attrs @@ -256,6 +312,12 @@ class TLSPolicy(BaseConfig): else: return True + def update(self, newer_policy, **kwargs): + fresh_policy = super(self.__class__, self).update(newer_policy, + domain_suffix=self.domain_suffix) + fresh_policy.domain_suffix = self.domain_suffix + return fresh_policy + @property def comment(self): return self._data.get('comment') @@ -278,7 +340,7 @@ class TLSPolicy(BaseConfig): @min_tls_version.setter def min_tls_version(self, value): - """Should this be dealing only with strings processed by map ... lower()?""" + """TODO: Should this be dealing only with strings processed by map ... lower()?""" tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] tls_versions.extend(self.TLS_VERSIONS) self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') @@ -309,11 +371,15 @@ class AcceptableMX(BaseConfig): Configuration of the acceptable MX suffix domains must match up with TLS policies for the suffix domains. """ - def __init__(self, domain): + def __init__(self, domain=None): super(self.__class__, self).__init__() self.domain = domain self._data['accept-mx-domains'] = [] + @property + def accept_mx_domains(self): + return self._data.get('accept-mx-domains') + def add_acceptable_mx(self, domain_suffix): unique_domain_suffixes = set(self._data['accept-mx-domains']) unique_domain_suffixes.add(domain_suffix) From 9a71b18b851c58bd7cb7c6a3fdfdaee23bdc3d94 Mon Sep 17 00:00:00 2001 From: pypoet Date: Fri, 23 Oct 2015 18:26:26 -0700 Subject: [PATCH 081/631] Fix updates and merges, add testing to make sure they stay fixed. --- Config.py | 65 ++++++++++++++++++++++++++++++++++++++------- TestConfig.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 TestConfig.py diff --git a/Config.py b/Config.py index 740de35c0..9cfbad3f0 100644 --- a/Config.py +++ b/Config.py @@ -1,6 +1,7 @@ from datetime import datetime from dateutil import parser import json +import logging import pprint @@ -8,6 +9,10 @@ import pprint make do json in *and* out, differences between configs and config extension. """ +#TODO scope logging and handlers better, control verbosity by command line flags +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + def parse_bool_from_json(value, attr_name): if value in ('true', '1', 1, 'yes'): @@ -90,7 +95,11 @@ class BaseConfig(object): return s def update(self, newer_config, merge=False, **kwargs): + # removed 'merge' kw arg - and it was passed to constructor + # make a note to not do that, consume it on the param list fresh_config = self.__class__(**kwargs) + logger.debug('from parent update kwargs %s' % kwargs) + logger.debug('from parent update merge %s' % merge) if not isinstance(newer_config, self.__class__): raise ConfigError('Attempting to update a %s with a %s' % ( self.__class__, @@ -108,6 +117,7 @@ class BaseConfig(object): def merge(self, newer_config, **kwargs): kwargs['merge'] = True + logger.debug('from parent merge: %s' % kwargs) return self.update(newer_config, **kwargs) def to_json(self): @@ -163,8 +173,6 @@ class Config(BaseConfig): #TODO add this new_config = Config() raise NotImplemented - - def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -187,8 +195,7 @@ class Config(BaseConfig): elif key == 'acceptable-mxs': self.make_acceptable_mxs_dict(val) else: - #TODO log warning - print 'Unknown attribute "%s", skipping' % key + logger.warn('Unknown attribute "%s", skipping' % key) @property def author(self): @@ -294,8 +301,7 @@ class TLSPolicy(BaseConfig): elif key == 'require-valid-certificate': self.require_valid_certificate = val else: - #TODO wat, log this instead - print 'Unknown key %s' % key + logger.warn('Unknown key %s' % key) def is_valid(self): """Do simple check that config contains all required values. @@ -313,9 +319,17 @@ class TLSPolicy(BaseConfig): return True def update(self, newer_policy, **kwargs): + if not kwargs.get('domain_suffix'): + kwargs['domain_suffix'] = self.domain_suffix fresh_policy = super(self.__class__, self).update(newer_policy, - domain_suffix=self.domain_suffix) - fresh_policy.domain_suffix = self.domain_suffix + **kwargs) + logger.debug('from TLS child update %s' % kwargs) + return fresh_policy + + def merge(self, newer_policy, **kwargs): + logger.debug('from TLS child merge: %s' % kwargs) + fresh_policy = super(self.__class__, self).merge(newer_policy, + domain_suffix=self.domain_suffix) return fresh_policy @property @@ -385,6 +399,14 @@ class AcceptableMX(BaseConfig): unique_domain_suffixes.add(domain_suffix) self._data['accept-mx-domains'] = list(unique_domain_suffixes) + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + def is_valid(self): """Check to make sure there is one acceptable domain suffix. @@ -407,9 +429,32 @@ class AcceptableMX(BaseConfig): self.add_acceptable_mx(domain_suffix) else: self.add_acceptable_mx(val) + elif key == 'comment': + self.comment = val else: - #TODO add logging for this - print 'warning: unknown key %s' % key + logger.warn('warning: unknown key %s' % key) + + def update(self, newer_policy, **kwargs): + logger.debug('from MX child update got %s' % kwargs) + if not kwargs.get('domain'): + kwargs['domain'] = self.domain + fresh_policy = super(self.__class__, self).update(newer_policy, + **kwargs) + if kwargs.get('merge'): + new_accepted_mxs = set(self.accept_mx_domains) + new_accepted_mxs = new_accepted_mxs.union(newer_policy.accept_mx_domains) + else: + new_accepted_mxs = newer_policy.accept_mx_domains + for domain in new_accepted_mxs: + fresh_policy.add_acceptable_mx(domain) + + return fresh_policy + + def merge(self, newer_policy, **kwargs): + logger.debug('from MX child merge: %s' % kwargs) + fresh_policy = super(self.__class__, self).merge(newer_policy, + **kwargs) + return fresh_policy class ConfigError(ValueError): diff --git a/TestConfig.py b/TestConfig.py new file mode 100644 index 000000000..043eb2dca --- /dev/null +++ b/TestConfig.py @@ -0,0 +1,73 @@ +import copy +import logging +import unittest + +import Config + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + + +class TestTLSPolicy(unittest.TestCase): + + def setUp(self): + self.old_config = Config.TLSPolicy(domain_suffix='.eff.org') + self.old_config.comment = 'Testing EFF.org TLS policy' + self.old_config.require_tls = True + self.old_config.require_valid_certificate = False + self.old_config.min_tls_version = 'TLSv1' + self.old_config.enforce_mode = 'log-only' + + self.new_config = Config.TLSPolicy(domain_suffix='.eff.org') + self.new_config.require_valid_certificate = True + self.new_config.min_tls_version = 'TLSv1.2' + self.new_config.enforce_mode = 'enforce' + + def testUpdateDropsOldSettings(self): + logger.debug('old: %s' % self.old_config) + logger.debug('new: %s' % self.new_config) + tls_policy = self.old_config.update(self.new_config) + logger.debug('just generated: %s' % tls_policy) + self.assertFalse(any([tls_policy.require_tls, tls_policy.comment])) + + def testMergeKeepsOldSettings(self): + logger.debug('old: %s' % self.old_config) + logger.debug('new: %s' % self.new_config) + tls_policy = self.old_config.merge(self.new_config, merge=True) + logger.debug('just generated: %s' % tls_policy) + self.assertTrue(all([tls_policy.require_tls, tls_policy.comment])) + + def testUpdateGetsNameSet(self): + tls_policy = self.old_config.update(self.new_config) + self.assertEquals(tls_policy.domain_suffix, self.old_config.domain_suffix) + + +class TestAcceptableMX(unittest.TestCase): + + def setUp(self): + self.old_config = Config.AcceptableMX(domain='eff.org') + self.old_config.add_acceptable_mx('.eff.org') + + def testUpdateDropsOldMXs(self): + new_bogus_mx = '.testing.eff.org' + new_config = Config.AcceptableMX(domain='eff.org') + new_config.add_acceptable_mx(new_bogus_mx) + updated_config = self.old_config.update(new_config) + self.assertNotIn('.eff.org', updated_config.accept_mx_domains) + + def testMergeKeepsOldMXs(self): + new_bogus_mx = '.testing.eff.org' + new_config = Config.AcceptableMX(domain='eff.org') + new_config.add_acceptable_mx(new_bogus_mx) + updated_config = self.old_config.merge(new_config) + self.assertListEqual(sorted(['.eff.org', '.testing.eff.org']), + sorted(updated_config.accept_mx_domains)) + + def testUpdateGetsNameSet(self): + new_policy = Config.AcceptableMX(domain=self.old_config.domain) + mx_policy = self.old_config.update(new_policy) + self.assertEquals(mx_policy.domain, self.old_config.domain) + + +if __name__ == '__main__': + unittest.main() From c87b5d6a7834553379b8f1a105adb35685499b96 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Thu, 21 Jan 2016 00:56:29 -0800 Subject: [PATCH 082/631] Hook the MTA config generation into the new config container. --- Config.py | 77 ++++++++++++++++++++++++++++++++++++++--- MTAConfigGenerator.py | 31 +++++++++-------- PostfixLogSummary.py | 24 ++++++++----- TestConfig.py | 58 +++++++++++++++++++++++++++++++ bigger_test_config.json | 2 +- 5 files changed, 164 insertions(+), 28 deletions(-) diff --git a/Config.py b/Config.py index 9cfbad3f0..eb4f31dba 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,6 @@ from datetime import datetime from dateutil import parser +import collections import json import logging import pprint @@ -238,7 +239,7 @@ class Config(BaseConfig): return self._data.get('acceptable-mxs') def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = self._data['tls-policies'] + tls_policy_dict = self.tls_policies for domain_suffix, settings in policy_dict.iteritems(): new_domain_policy = TLSPolicy(domain_suffix) try: @@ -247,6 +248,9 @@ class Config(BaseConfig): raise tls_policy_dict[domain_suffix] = new_domain_policy + def get_tls_policy(self, mx_domain): + return self.tls_policies.get(mx_domain) + def make_acceptable_mxs_dict(self, mxs_dict): acceptable_mxs_dict = self._data['acceptable-mxs'] for domain, settings in mxs_dict.iteritems(): @@ -257,9 +261,71 @@ class Config(BaseConfig): raise acceptable_mxs_dict[domain] = new_domain_policy + def get_address_domains(self, mx_hostname, mx_to_domain_map): + """Do a fuzzy DNS host match on provided map to get lists of policies. + + Args: + mx_hostname (string): The hostname from an MX record. + mx_to_domain_map: Mapping from MX hosts to AcceptableMX + policies, the same AcceptableMX policy may occur more + than once. e.g. {'mx_host3': set(AcceptableMX, ...)} + The map can be generated by Config.get_mx_to_domain_policy_map. + + Returns: + The set containing all AcceptableMX policies that list the + provided MX host as viable. + """ + labels = mx_hostname.split(".") + for n in range(1, len(labels)): + parent = "." + ".".join(labels[n:]) + if parent in mx_to_domains_map: + return mx_to_domain_map[parent] + return None + + def get_mx_to_domain_policy_map(self): + """Create mapping of MX hostnames to sets of AcceptableMX policies. + + Generate a dictionary that is typically used in log analysis + (e.g. if your MTA logs interact with beta.innotech.com you use + this mapping to tell you it used the innotech.com AcceptableMX + policy or policies). There are of course complications. + """ + # create reverse mapping dictionary as well for auditing + # and reviewing logs + mx_to_domain_policy = collections.defaultdict(set) + + for mx_host, domain_policy in self.get_all_mx_items(): + existing_mx_policies = mx_to_domain_policy.get(mx_host) + if existing_mx_policies: + existing_domains = [ e.domain for e in existing_mx_policies ] + if domain_policy.domain not in existing_domains: + #TODO plenty of room to enforce a security policy here + # this is also the case of google apps personal domains + msg = ('Attempting to add domain policy (%s) for MX host but MX' + ' host already has a domain policy (%s), appending...') + logger.debug(msg % (domain_policy.domain, + ', '.join(existing_domains))) + mx_to_domain_policy[mx_host].add(domain_policy) + return mx_to_domain_policy + + def get_all_mx_items(self): + """Iterate over (mx_host, mx_policy) - be sure to dedup!""" + all_mx_items = [] + for policy in self.acceptable_mxs.values(): + accepted_mxs = policy.accept_mx_domains + all_mx_items.extend([(mx_host, policy) + for mx_host in accepted_mxs]) + return all_mx_items + + def get_all_mx_hosts(self): + all_mx_hosts = [] + [ all_mx_hosts.extend(domain_policy.acceptable_mxs) + for domain_policy in self.acceptable_mxs.values() ] + return all_mx_hosts + def is_valid(self): #TODO implement checks to make sure domains don't overlap - #TODO add debug logging for troubleshooting stake + #TODO add debug logging for troubleshooting sake for mx_config in self.acceptable_mxs.values(): if not mx_config.is_valid(): return False @@ -267,9 +333,13 @@ class Config(BaseConfig): # check to make sure every accepted MX has a TLS policy if not domain_suffix in self.tls_policies: return False - for tls_config in self.tls_policies.values(): + all_mx_hosts = self.get_all_mx_hosts() + for domain_suffix, tls_config in self.tls_policies.iteritems(): if not tls_config.is_valid(): return False + # make sure no unclaimed TLS policies have made their way in + if domain_suffix not in all_mx_hosts: + return False return True @@ -460,4 +530,3 @@ class AcceptableMX(BaseConfig): class ConfigError(ValueError): def __init__(self, message): super(self.__class__, self).__init__(message) - diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index a733ab27a..f83d98d93 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -33,6 +33,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + #TODO make this optional for testing, etc. os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): @@ -120,33 +121,35 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for address_domain, properties in self.policy_config.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] + all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + for address_domain, properties in all_acceptable_mxs.items(): + mx_list = properties.accept_mx_domains if len(mx_list) > 1: print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain mx_domain = mx_list[0] - mx_policy = self.policy_config.tls_policies[mx_domain] + mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" - if "min-tls-version" in mx_policy: - if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" - elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" - elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" - else: - print mx_policy["min-tls-version"] + if mx_policy.min_tls_version.lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy.min_tls_version.lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy.min_tls_version.lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy.min_tls_version self.policy_lines.append(entry) f = open(self.policy_file, "w") f.write("\n".join(self.policy_lines) + "\n") f.close() + if __name__ == "__main__": - import ConfigParser + import Config as config if len(sys.argv) != 3: print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" sys.exit(1) - c = ConfigParser.Config(sys.argv[1]) + c = config.Config() + c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 0348432b0..c08e953ae 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -3,7 +3,7 @@ import re import sys import collections -import ConfigParser +import Config as config # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -35,21 +35,23 @@ def get_counts(input, config): # Log lines for when a TLS connection was successfully established. These can # indicate the difference between Untrusted, Trusted, and Verified certs. connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + mx_to_domain_mapping = config.get_mx_to_domain_policy_map() + for line in sys.stdin: deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: - validation = result.group(1) - mx_hostname = result.group(2).lower() + validation = connected.group(1) + mx_hostname = connected.group(2).lower() if validation == "Trusted" or validation == "Verified": seen_trusted = True - address_domains = config.get_address_domains(mx_hostname) + address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) if address_domains: - for d in address_domains: - counts[d][validation] += 1 - counts[d]["all"] += 1 + d = ', '.join(address_domains) + counts[d][validation] += 1 + counts[d]["all"] += 1 elif deferred: - mx_hostname = result.group(1).lower() + mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to @@ -65,7 +67,11 @@ def print_summary(counts): print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": - config = ConfigParser.Config("starttls-everywhere.json") + if len(sys.argv) != 2: + print "Usage: %s starttls-everywhere.json" % sys.argv[0] + sys.exit(1) + c = config.Config() + c.load_from_json_file(sys.argv[1]) (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) print tls_deferred diff --git a/TestConfig.py b/TestConfig.py index 043eb2dca..f323554c0 100644 --- a/TestConfig.py +++ b/TestConfig.py @@ -1,4 +1,5 @@ import copy +import itertools import logging import unittest @@ -69,5 +70,62 @@ class TestAcceptableMX(unittest.TestCase): self.assertEquals(mx_policy.domain, self.old_config.domain) +class TestConfig(unittest.TestCase): + """Test entire configuration. + + Currently lower coverage is being obtained since string sets are + being compared rather than returned objects. Comparison logic for + the config objects isn't clear yet and proof that they function is enough. + """ + + def setUp(self): + self.config = Config.Config() + domain_policies = self.config._data['acceptable-mxs'] + self.mail_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', '123.cn', 'qq.com'] + for domain in self.mail_domains: + new = Config.AcceptableMX(domain=domain) + new.add_acceptable_mx('.' + domain) + domain_policies[domain] = new + + def testGetAllMxItems(self): + """Make sure the basic use case of get_all_mx_items functions.""" + # [ ('.gmail.com', 'gmail.com'), ('.yahoo.com', 'yahoo.com'), ... ] + control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] + test_data = [ (mx, p.domain) for mx, p in self.config.get_all_mx_items() ] + self.assertListEqual(sorted(test_data), sorted(control_data)) + + def testGetAllMxItemsMultiMX(self): + config = copy.deepcopy(self.config) + domain_policy = config.acceptable_mxs.get('gmail.com') + # deal with reality, mail.google.com + domain_policy.add_acceptable_mx('.mail.google.com') + control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] + control_data.append(('.mail.google.com', 'gmail.com')) + test_data = [ (mx, p.domain) for mx, p in config.get_all_mx_items() ] + self.assertListEqual(sorted(test_data), sorted(control_data)) + + def testGetMXtoDomainPolicy(self): + control_data = dict([ ('.' + domain, set([domain])) + for domain in self.mail_domains ]) + test_data = {} + for mx, pset in self.config.get_mx_to_domain_policy_map().items(): + policy_list = [ p.domain for p in pset ] + test_data[mx] = set(policy_list) + self.assertDictEqual(test_data, control_data) + + def testGetMXtoDomainPolicyMultiMX(self): + config = copy.deepcopy(self.config) + domain_policy = config.acceptable_mxs.get('gmail.com') + domain_policy.add_acceptable_mx('.mail.google.com') + control_data = dict([ ('.' + domain, set([domain])) + for domain in self.mail_domains ]) + control_data['.mail.google.com'] = set(['gmail.com']) + test_data = {} + for mx, pset in config.get_mx_to_domain_policy_map().items(): + policy_list = [ p.domain for p in pset ] + test_data[mx] = set(policy_list) + self.assertDictEqual(test_data, control_data) + + if __name__ == '__main__': unittest.main() diff --git a/bigger_test_config.json b/bigger_test_config.json index e0697fc85..c3c23c455 100644 --- a/bigger_test_config.json +++ b/bigger_test_config.json @@ -1,7 +1,7 @@ { "timestamp": 1401414363, "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404242424, + "expires": "2015-08-01T12:00:00+08:00", "tls-policies": { ".yahoodns.net": { "require-valid-certificate": true From 7c6c3efb0f94a6954c99cc8e90b9f0821c1222a8 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Thu, 21 Jan 2016 01:46:30 -0800 Subject: [PATCH 083/631] Confirmed Postfix log parsing is working again. --- Config.py | 2 +- PostfixLogSummary.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Config.py b/Config.py index eb4f31dba..256a17b04 100644 --- a/Config.py +++ b/Config.py @@ -278,7 +278,7 @@ class Config(BaseConfig): labels = mx_hostname.split(".") for n in range(1, len(labels)): parent = "." + ".".join(labels[n:]) - if parent in mx_to_domains_map: + if parent in mx_to_domain_map: return mx_to_domain_map[parent] return None diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index c08e953ae..f9e717f66 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -3,7 +3,7 @@ import re import sys import collections -import Config as config +import Config # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -47,7 +47,8 @@ def get_counts(input, config): seen_trusted = True address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) if address_domains: - d = ', '.join(address_domains) + domains_str = [ a.domain for a in address_domains ] + d = ', '.join(domains_str) counts[d][validation] += 1 counts[d]["all"] += 1 elif deferred: @@ -70,8 +71,8 @@ if __name__ == "__main__": if len(sys.argv) != 2: print "Usage: %s starttls-everywhere.json" % sys.argv[0] sys.exit(1) - c = config.Config() - c.load_from_json_file(sys.argv[1]) + config = Config.Config() + config.load_from_json_file(sys.argv[1]) (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) print tls_deferred From 904dc11b03a9883952572b6b3294edcd040fc64a Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 09:45:37 -0800 Subject: [PATCH 084/631] Add docstrings for Config objects update/merge methods. --- Config.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Config.py b/Config.py index 256a17b04..dc402529a 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,5 @@ from datetime import datetime -from dateutil import parser +from dateutil import dateutil_parser import collections import json import logging @@ -36,7 +36,7 @@ def parse_timestamp(value, attr_name): except (TypeError, ValueError): pass try: - return parser.parse(value) + return dateutil_parser.parse(value) except (TypeError, ValueError): raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) @@ -96,6 +96,30 @@ class BaseConfig(object): return s def update(self, newer_config, merge=False, **kwargs): + """Create a fresh config combining the new and old configs. + + It does this by iterating over the 'config_properties' class + attribute which contains names of property attributes for the config. + + Two methods of combining configs are possible, an 'update' and + a 'merge', the latter set by the keyword argument 'merge=True'. + + An update overrides older values with new values -- even if those + new values are None. Update will remove values that are present in + the old config if they are not present in the new config. + + A merge by comparison will allow old values to persist if they are + not specified in the new config. This can be used for end-user + customizations to override specific settings without having to re-create + large portions of a config to override it. + + Arguments: + newer_config: A config object to combine with the current config. + merge: Allows old values not overridden to survive into the fresh config. + + Returns: + A config object of the same sort as called upon. + """ # removed 'merge' kw arg - and it was passed to constructor # make a note to not do that, consume it on the param list fresh_config = self.__class__(**kwargs) @@ -117,6 +141,17 @@ class BaseConfig(object): return fresh_config def merge(self, newer_config, **kwargs): + """Combines configs and keeps old values if they are not overridden. + + See docstring for 'update' method for more details. + + Arguments: + newer_config: A config object to combine with the current config. + merge: Allows old values not overridden to survive into the fresh config. + + Returns: + A config object of the same sort as called upon. + """ kwargs['merge'] = True logger.debug('from parent merge: %s' % kwargs) return self.update(newer_config, **kwargs) From 146fce3878f7b75ae3e7a9733f79cfae666de448 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 09:51:37 -0800 Subject: [PATCH 085/631] Add comment about magic hat trick with class properties. --- Config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Config.py b/Config.py index dc402529a..402fb4953 100644 --- a/Config.py +++ b/Config.py @@ -130,6 +130,7 @@ class BaseConfig(object): self.__class__, newer_config.__class__)) for prop_name in self.config_properties: + # get the specified property off of the current class prop = self.__class__.__dict__.get(prop_name) assert prop new_value = prop.fget(newer_config) From 9abef4c0bdf33fd4e2c874103def6c9bcac74462 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 10:20:56 -0800 Subject: [PATCH 086/631] Log MX records that will not be configured. --- MTAConfigGenerator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index f83d98d93..1c273cf94 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -125,7 +125,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + print "Lists of multiple accept-mx-domains not yet supported." + print "Using MX %s for %s" % (mx_list[0], address_domain) + print "Ignoring: %s" % (', '.join(mx_list[1:])) mx_domain = mx_list[0] mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" From 39a01190d50514666a9bc6f147fc9ba6cce20227 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:19:02 -0800 Subject: [PATCH 087/631] Use 4 space indents For greater compatibility with LE codebases --- MTAConfigGenerator.py | 262 +++++++++++++++++++++--------------------- 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 1c273cf94..bb3d5e5db 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -5,153 +5,153 @@ import string import os, os.path def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. + """ + Return the (line number, left hand side, right hand side) of a stripped + postfix config line. - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + num,line = line_data + left, sep, right = line.partition("=") + if not sep: + return None + return (num, left.strip(), right.strip()) class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config + def __init__(self, policy_config): + self.policy_config = policy_config class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) - self.postfix_cf_file = self.find_postfix_cf() - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - #TODO make this optional for testing, etc. - os.system("sudo service postfix reload") + def __init__(self, policy_config, postfix_dir, fixup=False): + self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) + self.postfix_cf_file = self.find_postfix_cf() + self.wrangle_existing_config() + self.set_domainwise_tls_policies() + #TODO make this optional for testing, etc. + os.system("sudo service postfix reload") - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - """ - acceptable = [ideal] + also_acceptable + def ensure_cf_var(self, var, ideal, also_acceptable): + """ + Ensure that existing postfix config @var is in the list of @acceptable + values; if not, set it to the ideal value. + """ + acceptable = [ideal] + also_acceptable - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - #print "Scheduling deletions:" + `values` - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) + l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] + if not any(l): + self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Conflicting existing config values " + `l` - val = values[0][2] - if val not in acceptable: - #print "Scheduling deletions:" + `values` + values = map(parse_line, l) + if len(set(values)) > 1: + if self.fixup: + #print "Scheduling deletions:" + `values` + conflicting_lines = [num for num,_var,val in values] + self.deletions.extend(conflicting_lines) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Conflicting existing config values " + `l` + val = values[0][2] + if val not in acceptable: + #print "Scheduling deletions:" + `values` + if self.fixup: + self.deletions.append(values[0][0]) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + + def wrangle_existing_config(self): + """ + Try to ensure/mutate that the config file is in a sane state. + Fixup means we'll delete existing lines if necessary to get there. + """ + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + + # Check we're currently accepting inbound STARTTLS sensibly + self.ensure_cf_var("smtpd_use_tls", "yes", []) + # Ideally we use it opportunistically in the outbound direction + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) + # Maximum verbosity lets us collect failure information + self.ensure_cf_var("smtp_tls_loglevel", "1", []) + # Inject a reference to our per-domain policy map + policy_cf_entry = "texthash:" + self.policy_file + + self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + + self.maybe_add_config_lines() + + def maybe_add_config_lines(self): + if not self.additions: + return if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + print "Deleting lines:", self.deletions + self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + new_cf_lines = "\n".join(self.additions) + "\n" + print "Adding to %s:" % self.fn + print new_cf_lines + if self.raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] + self.new_cf = "" + for num, line in enumerate(self.raw_cf): + if self.fixup and num in self.deletions: + self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line + else: + self.new_cf += line + self.new_cf += sep + new_cf_lines - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file + #print self.new_cf + f = open(self.fn, "w") + f.write(self.new_cf) + f.close() - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.postfix_dir, "main.cf") - self.maybe_add_config_lines() + def set_domainwise_tls_policies(self): + self.policy_lines = [] + all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + for address_domain, properties in all_acceptable_mxs.items(): + mx_list = properties.accept_mx_domains + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported." + print "Using MX %s for %s" % (mx_list[0], address_domain) + print "Ignoring: %s" % (', '.join(mx_list[1:])) + mx_domain = mx_list[0] + mx_policy = self.policy_config.get_tls_policy(mx_domain) + entry = address_domain + " encrypt" + if mx_policy.min_tls_version.lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy.min_tls_version.lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy.min_tls_version.lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy.min_tls_version + self.policy_lines.append(entry) - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - self.new_cf = "" - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - #print self.new_cf - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() - - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - - def set_domainwise_tls_policies(self): - self.policy_lines = [] - all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() - for address_domain, properties in all_acceptable_mxs.items(): - mx_list = properties.accept_mx_domains - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported." - print "Using MX %s for %s" % (mx_list[0], address_domain) - print "Ignoring: %s" % (', '.join(mx_list[1:])) - mx_domain = mx_list[0] - mx_policy = self.policy_config.get_tls_policy(mx_domain) - entry = address_domain + " encrypt" - if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" - elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" - elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" - else: - print mx_policy.min_tls_version - self.policy_lines.append(entry) - - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() + f = open(self.policy_file, "w") + f.write("\n".join(self.policy_lines) + "\n") + f.close() if __name__ == "__main__": - import Config as config - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) - c = config.Config() - c.load_from_json_file(sys.argv[1]) - postfix_dir = sys.argv[2] - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + import Config as config + if len(sys.argv) != 3: + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" + sys.exit(1) + c = config.Config() + c.load_from_json_file(sys.argv[1]) + postfix_dir = sys.argv[2] + pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) From 965027ce528358cd838cffd6f3a8228630643637 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:36:38 -0800 Subject: [PATCH 088/631] Start metamorphising to use LE's IInstaller interface --- MTAConfigGenerator.py | 137 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index bb3d5e5db..ae74892eb 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,13 +28,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) - self.postfix_cf_file = self.find_postfix_cf() - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - #TODO make this optional for testing, etc. - os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -145,6 +138,133 @@ class PostfixConfigGenerator(MTAConfigGenerator): f.write("\n".join(self.policy_lines) + "\n") f.close() + ### Let's Encrypt client IPlugin ### + + def prepare(): + """Prepare the plugin. + Finish up any additional initialization. + :raises .PluginError: + when full initialization cannot be completed. + :raises .MisconfigurationError: + when full initialization cannot be completed. Plugin will + be displayed on a list of available plugins. + :raises .NoInstallationError: + when the necessary programs/files cannot be located. Plugin + will NOT be displayed on a list of available plugins. + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + """ + # XXX ensure we raise the right kinds of exceptions + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) + self.postfix_cf_file = self.find_postfix_cf() + + + def more_info(): + """Human-readable string to help the user. + Should describe the steps taken and any relevant info to help the user + decide which plugin to use. + :rtype str: + """ + + + ### Let's Encrypt client IInstaller ### + + def get_all_names(): + """Returns all names that may be authenticated. + :rtype: `list` of `str` + """ + + def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path): + """Deploy certificate. + :param str domain: domain to deploy certificate file + :param str cert_path: absolute path to the certificate file + :param str key_path: absolute path to the private key file + :param str chain_path: absolute path to the certificate chain file + :param str fullchain_path: absolute path to the certificate fullchain + file (cert plus chain) + :raises .PluginError: when cert cannot be deployed + """ + + self.wrangle_existing_config() + self.set_domainwise_tls_policies() + + def enhance(domain, enhancement, options=None): + """Perform a configuration enhancement. + :param str domain: domain for which to provide enhancement + :param str enhancement: An enhancement as defined in + :const:`~letsencrypt.constants.ENHANCEMENTS` + :param options: Flexible options parameter for enhancement. + Check documentation of + :const:`~letsencrypt.constants.ENHANCEMENTS` + for expected options for each enhancement. + :raises .PluginError: If Enhancement is not supported, or if + an error occurs during the enhancement. + """ + + def supported_enhancements(): + """Returns a list of supported enhancements. + :returns: supported enhancements which should be a subset of + :const:`~letsencrypt.constants.ENHANCEMENTS` + :rtype: :class:`list` of :class:`str` + """ + + def get_all_certs_keys(): + """Retrieve all certs and keys set in configuration. + :returns: tuples with form `[(cert, key, path)]`, where: + - `cert` - str path to certificate file + - `key` - str path to associated key file + - `path` - file path to configuration file + :rtype: list + """ + + def save(title=None, temporary=False): + """Saves all changes to the configuration files. + Both title and temporary are needed because a save may be + intended to be permanent, but the save is not ready to be a full + checkpoint. If an exception is raised, it is assumed a new + checkpoint was not created. + :param str title: The title of the save. If a title is given, the + configuration will be saved as a new checkpoint and put in a + timestamped directory. `title` has no effect if temporary is true. + :param bool temporary: Indicates whether the changes made will + be quickly reversed in the future (challenges) + :raises .PluginError: when save is unsuccessful + """ + + def rollback_checkpoints(rollback=1): + """Revert `rollback` number of configuration checkpoints. + :raises .PluginError: when configuration cannot be fully reverted + """ + + def recovery_routine(): + """Revert configuration to most recent finalized checkpoint. + Remove all changes (temporary and permanent) that have not been + finalized. This is useful to protect against crashes and other + execution interruptions. + :raises .errors.PluginError: If unable to recover the configuration + """ + + def view_config_changes(): + """Display all of the LE config changes. + :raises .PluginError: when config changes cannot be parsed + """ + + def config_test(): + """Make sure the configuration is valid. + :raises .MisconfigurationError: when the config is not in a usable state + """ + + def restart(): + """Restart or refresh the server content. + :raises .PluginError: when server cannot be restarted + """ + if os.geteuid() != 0: + os.system("sudo service postfix reload") + else: + os.system("service postfix reload") + if __name__ == "__main__": import Config as config @@ -155,3 +275,6 @@ if __name__ == "__main__": c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + pcgen.prepare() + pcgen.deploy_cert() # XXX add cert args! + pcgen.restart() From fedf97028427203ff8ea9e3b2e63e24826982c71 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 11:37:28 -0800 Subject: [PATCH 089/631] Fix bad import to be import *as*... as it should be. --- Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Config.py b/Config.py index 402fb4953..ac5202d0c 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,5 @@ from datetime import datetime -from dateutil import dateutil_parser +from dateutil import parser as dateutil_parser import collections import json import logging From 47a5b7e3ba6fd6708e2a776f5730602029f12c00 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:56:41 -0800 Subject: [PATCH 090/631] Start implementing cert installation --- MTAConfigGenerator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index ae74892eb..74a53a345 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -81,7 +81,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.maybe_add_config_lines() def maybe_add_config_lines(self): if not self.additions: @@ -186,8 +185,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): file (cert plus chain) :raises .PluginError: when cert cannot be deployed """ - self.wrangle_existing_config() + self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) + self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() def enhance(domain, enhancement, options=None): @@ -233,6 +233,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): :raises .PluginError: when save is unsuccessful """ + self.maybe_add_config_lines() + def rollback_checkpoints(rollback=1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted @@ -277,4 +279,5 @@ if __name__ == "__main__": pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() pcgen.deploy_cert() # XXX add cert args! + pcgen.save() pcgen.restart() From 8f5b8558d28f817077d663854d89b5128d1b8cbe Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:09:25 -0800 Subject: [PATCH 091/631] Actually deploy a cert? - Also add missing selves to interface methods --- MTAConfigGenerator.py | 48 ++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 74a53a345..3a2a653d6 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,6 +28,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -139,7 +141,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): ### Let's Encrypt client IPlugin ### - def prepare(): + def prepare(self): """Prepare the plugin. Finish up any additional initialization. :raises .PluginError: @@ -155,12 +157,10 @@ class PostfixConfigGenerator(MTAConfigGenerator): currently supported. """ # XXX ensure we raise the right kinds of exceptions - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() - def more_info(): + def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user decide which plugin to use. @@ -170,12 +170,12 @@ class PostfixConfigGenerator(MTAConfigGenerator): ### Let's Encrypt client IInstaller ### - def get_all_names(): + def get_all_names(self): """Returns all names that may be authenticated. :rtype: `list` of `str` """ - def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path): + def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. :param str domain: domain to deploy certificate file :param str cert_path: absolute path to the certificate file @@ -190,7 +190,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() - def enhance(domain, enhancement, options=None): + def enhance(self, domain, enhancement, options=None): """Perform a configuration enhancement. :param str domain: domain for which to provide enhancement :param str enhancement: An enhancement as defined in @@ -203,14 +203,14 @@ class PostfixConfigGenerator(MTAConfigGenerator): an error occurs during the enhancement. """ - def supported_enhancements(): + def supported_enhancements(self): """Returns a list of supported enhancements. :returns: supported enhancements which should be a subset of :const:`~letsencrypt.constants.ENHANCEMENTS` :rtype: :class:`list` of :class:`str` """ - def get_all_certs_keys(): + def get_all_certs_keys(self): """Retrieve all certs and keys set in configuration. :returns: tuples with form `[(cert, key, path)]`, where: - `cert` - str path to certificate file @@ -219,7 +219,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): :rtype: list """ - def save(title=None, temporary=False): + def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be intended to be permanent, but the save is not ready to be a full @@ -235,12 +235,12 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.maybe_add_config_lines() - def rollback_checkpoints(rollback=1): + def rollback_checkpoints(self, rollback=1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted """ - def recovery_routine(): + def recovery_routine(self): """Revert configuration to most recent finalized checkpoint. Remove all changes (temporary and permanent) that have not been finalized. This is useful to protect against crashes and other @@ -248,17 +248,17 @@ class PostfixConfigGenerator(MTAConfigGenerator): :raises .errors.PluginError: If unable to recover the configuration """ - def view_config_changes(): + def view_config_changes(self): """Display all of the LE config changes. :raises .PluginError: when config changes cannot be parsed """ - def config_test(): + def config_test(self): """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ - def restart(): + def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ @@ -268,16 +268,26 @@ class PostfixConfigGenerator(MTAConfigGenerator): os.system("service postfix reload") +def usage(): + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" + sys.exit(1) + if __name__ == "__main__": import Config as config - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) + if len(sys.argv) != 4: + usage() c = config.Config() c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] + le_lineage = sys.argv[3] + pieces = [os.path.join(le_lineage, f) for f in ("cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] + if not os.isdir(le_lineage) or not all(os.isfile(p) for p in pieces) : + print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" + print + usage() + cert, key, chain, fullchain = pieces pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() - pcgen.deploy_cert() # XXX add cert args! + pcgen.deploy_cert(cert, key, chain, fullchain) pcgen.save() pcgen.restart() From 3aeb62cf7e3d0eacedaafeca386afddf4a85a39f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:13:28 -0800 Subject: [PATCH 092/631] bugfixes, cleanups --- MTAConfigGenerator.py | 8 +++++--- requirements.txt | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3a2a653d6..3c676700e 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -269,7 +269,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def usage(): - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" + print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % + sys.argv[0]) sys.exit(1) if __name__ == "__main__": @@ -280,8 +281,9 @@ if __name__ == "__main__": c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] le_lineage = sys.argv[3] - pieces = [os.path.join(le_lineage, f) for f in ("cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] - if not os.isdir(le_lineage) or not all(os.isfile(p) for p in pieces) : + pieces = [os.path.join(le_lineage, f) for f in ( + "cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] + if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) : print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" print usage() diff --git a/requirements.txt b/requirements.txt index 891e5809d..5334ba03a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ dnspython publicsuffix m2crypto +dateutils From 074fef773bcf3097a6c6532740b12535e3f3334d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:14:24 -0800 Subject: [PATCH 093/631] Make up a domain --- MTAConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3c676700e..9de48ca26 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -290,6 +290,6 @@ if __name__ == "__main__": cert, key, chain, fullchain = pieces pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() - pcgen.deploy_cert(cert, key, chain, fullchain) + pcgen.deploy_cert("example.com", cert, key, chain, fullchain) pcgen.save() pcgen.restart() From 28bb0eb6acf7c63c763189963803a93f0b2691fd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:20:26 -0800 Subject: [PATCH 094/631] Obtain acceptable_mxs the right way --- MTAConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 9de48ca26..43dad5f37 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -115,7 +115,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: From 9e42f6ed08e27f022568a4d651d24ddcf38aeffd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 18 Feb 2016 18:57:41 -0800 Subject: [PATCH 095/631] Clarify project status --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e50630ab8..49db2aeda 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # STARTTLS Everywhere +NOTE: this is a pre-alpha codebase. Do not run it on non-experimental systems +yet! + ## Authors -Jacob Hoffman-Andrews , Peter Eckersley +Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox ## Background From 4d14423a2146bc39f6d4f24c0266e26e262d71f2 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Wed, 24 Feb 2016 21:38:31 +0100 Subject: [PATCH 096/631] re-structured project folder.. * Removed `ConfigParser.py` (ACK by Daniel). * Removed MTAConfigGenerator-stub and renamed to `PostfixConfigGenerator.py` * Moved all text/csv processing scripts to `tools/`. * Moved all configuration files into dedicated `examples/` directory. * unified all shebangs to `#!/usr/bin/env python` (default system python). * Moved domain CSV and text-files to `share/`. --- ConfigParser.py | 115 ------------------ .../bigger_test_config.json | 0 config.json => examples/config.json | 0 .../starttls-everywhere.json | 0 Config.py => letsencrypt-postfix/Config.py | 1 + .../PostfixConfigGenerator.py | 15 +-- .../PostfixLogSummary.py | 2 +- .../TestConfig.py | 1 + .../golden-domains.txt | 0 .../google-starttls-domains.csv | 0 CheckSTARTTLS.py => tools/CheckSTARTTLS.py | 2 +- .../ProcessGoogleSTARTTLSDomains.py | 2 +- 12 files changed, 10 insertions(+), 128 deletions(-) delete mode 100755 ConfigParser.py rename bigger_test_config.json => examples/bigger_test_config.json (100%) rename config.json => examples/config.json (100%) rename starttls-everywhere.json => examples/starttls-everywhere.json (100%) rename Config.py => letsencrypt-postfix/Config.py (99%) rename MTAConfigGenerator.py => letsencrypt-postfix/PostfixConfigGenerator.py (96%) rename PostfixLogSummary.py => letsencrypt-postfix/PostfixLogSummary.py (99%) rename TestConfig.py => letsencrypt-postfix/TestConfig.py (99%) mode change 100644 => 100755 rename golden-domains.txt => share/golden-domains.txt (100%) rename google-starttls-domains.csv => share/google-starttls-domains.csv (100%) rename CheckSTARTTLS.py => tools/CheckSTARTTLS.py (99%) rename ProcessGoogleSTARTTLSDomains.py => tools/ProcessGoogleSTARTTLSDomains.py (98%) diff --git a/ConfigParser.py b/ConfigParser.py deleted file mode 100755 index 2d7c88ada..000000000 --- a/ConfigParser.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python - -import sys -import json -from datetime import datetime -import string -import collections - -def parse_timestamp(ts): - try: - int(ts) - dt = datetime.fromtimestamp(ts) - return dt - except: - raise ValueError, "Invalid timestamp integer: " + `ts` - -legal = string.letters + string.digits + ".-" -known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list -def looks_like_a_domain(s): - "Return true if string looks like a domain, as best we can tell..." - global known_tlds - try: - domain = s.lower() - assert domain[0].islower() - assert all([c in legal for c in domain]) - tld = s.split(".")[-1] - if tld not in known_tlds: - # XXX perform DNS query to determine that this TLD exists - pass - return True - except: - return False - -class Config: - def __init__(self, cfg_file_name = "config.json"): - f = open(cfg_file_name) - self.cfg = json.loads(f.read()) - self.tls_policies = {} - self.mx_map = {} - for atr, val in self.cfg.items(): - # Verify each attribute of the structure - if atr.startswith("comment"): - continue - if atr == "author": - if type(val) not in [str, unicode]: - raise TypeError, "Author must be a string: " + `val` - elif atr == "timestamp": - self.timestamp = parse_timestamp(val) - elif atr == "expires": - self.expires = parse_timestamp(val) - elif atr == "tls-policies": - for domain, policies in self.check_tls_policy_domains(val): - if type(policies) != dict: - raise TypeError, domain + "'s policies should be a dict: " + `policies` - self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, v in policies.items(): - value = str(v).lower() - if policy == "require-tls": - if value in ("true", "1", "yes"): - self.tls_policies[domain]["required"] = True - elif value in ("false", "0", "no"): - self.tls_policies[domain]["required"] = False - else: - raise ValueError, "Unknown require-tls value " + `value` - elif policy == "min-tls-version": - reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] - reasonable = map(string.lower, reasonable) - if not value in reasonable: - raise ValueError, "Not a valid TLS version string: " + `value` - self.tls_policies[domain]["min-tls-version"] = str(value) - elif policy == "enforce-mode": - if value == "enforce": - self.tls_policies[domain]["enforce"] = True - elif value == "log-only": - self.tls_policies[domain]["enforce"] = False - else: - raise ValueError, "Not a known enoforcement policy " + `value` - elif atr == "acceptable-mxs": - self.acceptable_mxs = val - self.mx_domain_to_address_domains = collections.defaultdict(set) - for address_domain, properties in self.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - self.mx_domain_to_address_domains[mx_domain].add(address_domain) - pass - else: - sys.stderr.write("Unknown attribute: " + `atr` + "\n") - # XXX is it ever permissible to have a domain with an acceptable-mx - # that does not point to a TLS security policy? If not, check/warn/fail - # here - print self.tls_policies - - def get_address_domains(self, mx_hostname): - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in self.mx_domain_to_address_domains: - return self.mx_domain_to_address_domains[parent] - return None - - def check_tls_policy_domains(self, val): - if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` - for domain, policies in val.items(): - try: - assert type(domain) == unicode - d = str(domain) # convert from unicode - except: - raise TypeError, "tls-policy domain not a string" + `domain` - yield (d, policies) - -if __name__ == "__main__": - c = Config() diff --git a/bigger_test_config.json b/examples/bigger_test_config.json similarity index 100% rename from bigger_test_config.json rename to examples/bigger_test_config.json diff --git a/config.json b/examples/config.json similarity index 100% rename from config.json rename to examples/config.json diff --git a/starttls-everywhere.json b/examples/starttls-everywhere.json similarity index 100% rename from starttls-everywhere.json rename to examples/starttls-everywhere.json diff --git a/Config.py b/letsencrypt-postfix/Config.py similarity index 99% rename from Config.py rename to letsencrypt-postfix/Config.py index ac5202d0c..cc0df00d1 100644 --- a/Config.py +++ b/letsencrypt-postfix/Config.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python from datetime import datetime from dateutil import parser as dateutil_parser import collections diff --git a/MTAConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py similarity index 96% rename from MTAConfigGenerator.py rename to letsencrypt-postfix/PostfixConfigGenerator.py index 43dad5f37..af1953208 100755 --- a/MTAConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import sys import string import os, os.path @@ -18,18 +17,14 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) -class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config - class ExistingConfigError(ValueError): pass -class PostfixConfigGenerator(MTAConfigGenerator): +class PostfixConfigGenerator: def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) + self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_config = policy_config + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") def ensure_cf_var(self, var, ideal, also_acceptable): """ diff --git a/PostfixLogSummary.py b/letsencrypt-postfix/PostfixLogSummary.py similarity index 99% rename from PostfixLogSummary.py rename to letsencrypt-postfix/PostfixLogSummary.py index f9e717f66..956a069eb 100755 --- a/PostfixLogSummary.py +++ b/letsencrypt-postfix/PostfixLogSummary.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.7 +#!/usr/bin/env python import re import sys import collections diff --git a/TestConfig.py b/letsencrypt-postfix/TestConfig.py old mode 100644 new mode 100755 similarity index 99% rename from TestConfig.py rename to letsencrypt-postfix/TestConfig.py index f323554c0..ca8e77654 --- a/TestConfig.py +++ b/letsencrypt-postfix/TestConfig.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import copy import itertools import logging diff --git a/golden-domains.txt b/share/golden-domains.txt similarity index 100% rename from golden-domains.txt rename to share/golden-domains.txt diff --git a/google-starttls-domains.csv b/share/google-starttls-domains.csv similarity index 100% rename from google-starttls-domains.csv rename to share/google-starttls-domains.csv diff --git a/CheckSTARTTLS.py b/tools/CheckSTARTTLS.py similarity index 99% rename from CheckSTARTTLS.py rename to tools/CheckSTARTTLS.py index e5c5a4323..ef0bf2e5c 100755 --- a/CheckSTARTTLS.py +++ b/tools/CheckSTARTTLS.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys import os import errno diff --git a/ProcessGoogleSTARTTLSDomains.py b/tools/ProcessGoogleSTARTTLSDomains.py similarity index 98% rename from ProcessGoogleSTARTTLSDomains.py rename to tools/ProcessGoogleSTARTTLSDomains.py index 3078bd93a..815ec5d4e 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/tools/ProcessGoogleSTARTTLSDomains.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ Process Google's TLS delivery data from https://www.google.com/transparencyreport/saferemail/data/?hl=en From 6db285882552e73e7ffcb6eff10293dbe0f48ce8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 24 Feb 2016 18:19:47 -0800 Subject: [PATCH 097/631] Correct policy map delimitation --- MTAConfigGenerator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 43dad5f37..ea4eeaa26 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -126,11 +126,11 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" + entry += " protocols=!SSLv2:!SSLv3" elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1" elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: print mx_policy.min_tls_version self.policy_lines.append(entry) From 6e2b6a081706ced2e6ca1ddbc7cd3556e141934f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:37:56 -0800 Subject: [PATCH 098/631] Update README --- README.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 288a6ea10..f244d91e6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,36 @@ # STARTTLS Everywhere -NOTE: this is a pre-alpha codebase. Do not run it on non-experimental systems -yet! + +## Example usage + +**WARNING: this is a pre-alpha codebase. Do not run it on production +mailservers!!!** + + +If you have a Postfix server you're willing to endanger deliverability on, you +can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), not the directory it lives in below `/etc/letsencrypt/live` and then do: + +``` +git clone https://github.com/EFForg/starttls-everywhere +cd starttls-everywhere +# Promise you don't care if deliverability breaks on this mail server +letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/YOUR.DOMAIN.EXAMPLE.COM +``` + +This will: +* Install the cert in Postfix +* Enforce mandatory TLS to some major email domains +* Enforce minimum TLS versions to some major email domains + +## Project status + +* Postfix configuration generation: working pre-alpha, not yet safe +* Email security database: working pre-alpha, definitely not yet safe +* Let's Encrypt client plugin: in progress ## Authors -Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox +Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox , Aaron Zauner ## Mailing List From 1210c04f147f6bf6f539ee0683f46af9c8939de1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:38:55 -0800 Subject: [PATCH 099/631] tweak README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f244d91e6..8a4e04601 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ mailservers!!!** If you have a Postfix server you're willing to endanger deliverability on, you -can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), not the directory it lives in below `/etc/letsencrypt/live` and then do: +can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), note the directory it lives in below `/etc/letsencrypt/live` and then do: ``` git clone https://github.com/EFForg/starttls-everywhere @@ -26,7 +26,7 @@ This will: * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe -* Let's Encrypt client plugin: in progress +* Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready ## Authors From 3a8e1d7a707a509022839fa31944b1daf8f4b2e6 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:40:03 -0800 Subject: [PATCH 100/631] Further status update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8a4e04601..e0746c4a1 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ This will: * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready +* DANE support: none yet +* SMTP-STS integration: none yet ## Authors From a435036a1ecedcfe143d00f186dbf808e06d1983 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 19:03:47 -0800 Subject: [PATCH 101/631] Document MVP objectives --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e0746c4a1..9f20c4457 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,18 @@ This will: ## Project status +STARTTLS Everywhere development is re-starting after a hiatus. Initial +objectives: + * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready * DANE support: none yet * SMTP-STS integration: none yet +* Direct mechanisms for mail domains to request inclusion: none yet +* Failure reporting mechanisms: early progress, not yet ready +* Mechanisms for secure mutli-organization signature on the policy database: + none yet ## Authors From b2977ad6a996aae6bf2cd1527098722286282b3e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 19:06:02 -0800 Subject: [PATCH 102/631] Documentation details --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f20c4457..008907baf 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json ``` This will: -* Install the cert in Postfix +* Ensure your mail server initiates STARTTLS encryption +* Install the Let's Encrypt cert in Postfix * Enforce mandatory TLS to some major email domains * Enforce minimum TLS versions to some major email domains From 874c59012ab219d24678907891a1249b1277d0d8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 2 Mar 2016 10:47:52 -0800 Subject: [PATCH 103/631] Update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 008907baf..27563a899 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ objectives: * SMTP-STS integration: none yet * Direct mechanisms for mail domains to request inclusion: none yet * Failure reporting mechanisms: early progress, not yet ready -* Mechanisms for secure mutli-organization signature on the policy database: +* Mechanisms for secure multi-organization signature on the policy database: none yet +* Support for mail servers other than Postfix: none yet ## Authors From e88eac65da32fe33228c650eeee51ca40ef138ea Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 2 Mar 2016 10:47:59 -0800 Subject: [PATCH 104/631] [documentation] Add links to LEPC interface sources --- letsencrypt-postfix/PostfixConfigGenerator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 43a3c19cf..25cab9ce7 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -135,6 +135,7 @@ class PostfixConfigGenerator: f.close() ### Let's Encrypt client IPlugin ### + # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 def prepare(self): """Prepare the plugin. @@ -164,6 +165,7 @@ class PostfixConfigGenerator: ### Let's Encrypt client IInstaller ### + # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 def get_all_names(self): """Returns all names that may be authenticated. From 0ce1684ba6895706422349dce5d2b1c94e60ae05 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:50:15 -0800 Subject: [PATCH 105/631] Changes to MTAConfigGenerator needed to be moved by hand --- MTAConfigGenerator.py | 160 ------------------ letsencrypt-postfix/PostfixConfigGenerator.py | 18 +- 2 files changed, 12 insertions(+), 166 deletions(-) delete mode 100755 MTAConfigGenerator.py diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py deleted file mode 100755 index 5ca8f5aee..000000000 --- a/MTAConfigGenerator.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python - -import sys -import string -import os, os.path - -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) - -class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config - -class ExistingConfigError(ValueError): pass - -class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.postfix_cf_file = self.find_postfix_cf() - if not os.access(self.postfix_cf_file, os.W_OK): - raise Exception("Can't write to %s, please re-run as root." - % self.postfix_cf_file) - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") - MTAConfigGenerator.__init__(self, policy_config) - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - self.update_CAfile() - os.system("sudo service postfix reload") - - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - """ - acceptable = [ideal] + also_acceptable - - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Conflicting existing config values " + `l` - val = values[0][2] - if val not in acceptable: - if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file - - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - - self.maybe_add_config_lines() - - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - self.new_cf = "" - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() - - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - - def set_domainwise_tls_policies(self): - self.policy_lines = [] - for address_domain, properties in self.policy_config.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - mx_policy = self.policy_config.tls_policies[mx_domain] - entry = address_domain + " encrypt" - if "min-tls-version" in mx_policy: - if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2:!SSLv3" - elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - else: - print mx_policy["min-tls-version"] - self.policy_lines.append(entry) - - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() - - def update_CAfile(self): - os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + - self.ca_file) - -if __name__ == "__main__": - import ConfigParser - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) - c = ConfigParser.Config(sys.argv[1]) - postfix_dir = sys.argv[2] - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) - print "Done." diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 25cab9ce7..af29d493c 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -25,6 +25,7 @@ class PostfixConfigGenerator: self.postfix_dir = postfix_dir self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -40,7 +41,6 @@ class PostfixConfigGenerator: values = map(parse_line, l) if len(set(values)) > 1: if self.fixup: - #print "Scheduling deletions:" + `values` conflicting_lines = [num for num,_var,val in values] self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) @@ -48,7 +48,6 @@ class PostfixConfigGenerator: raise ExistingConfigError, "Conflicting existing config values " + `l` val = values[0][2] if val not in acceptable: - #print "Scheduling deletions:" + `values` if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) @@ -99,10 +98,11 @@ class PostfixConfigGenerator: self.new_cf += line self.new_cf += sep + new_cf_lines - #print self.new_cf - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() + if not os.access(self.postfix_cf_file, os.W_OK): + raise Exception("Can't write to %s, please re-run as root." + % self.postfix_cf_file) + with open(self.fn, "w") as f: + f.write(self.new_cf) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -186,6 +186,7 @@ class PostfixConfigGenerator: self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() + self.update_CAfile() def enhance(self, domain, enhancement, options=None): """Perform a configuration enhancement. @@ -259,17 +260,22 @@ class PostfixConfigGenerator: """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ + print "Reloading postfix config..." if os.geteuid() != 0: os.system("sudo service postfix reload") else: os.system("service postfix reload") + def update_CAfile(self): + os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + def usage(): print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % sys.argv[0]) sys.exit(1) + if __name__ == "__main__": import Config as config if len(sys.argv) != 4: From a58c6524436d04adc43a89a0fa7534e7d6cdfdba Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:54:15 -0800 Subject: [PATCH 106/631] ConfigParser is gone! --- ConfigParser.py | 114 ------------------------------------------------ 1 file changed, 114 deletions(-) delete mode 100755 ConfigParser.py diff --git a/ConfigParser.py b/ConfigParser.py deleted file mode 100755 index d1c413f74..000000000 --- a/ConfigParser.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -import sys -import json -from datetime import datetime -import string -import collections - -def parse_timestamp(ts): - try: - int(ts) - dt = datetime.fromtimestamp(ts) - return dt - except: - raise ValueError, "Invalid timestamp integer: " + `ts` - -legal = string.letters + string.digits + ".-" -known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list -def looks_like_a_domain(s): - "Return true if string looks like a domain, as best we can tell..." - global known_tlds - try: - domain = s.lower() - assert domain[0].islower() - assert all([c in legal for c in domain]) - tld = s.split(".")[-1] - if tld not in known_tlds: - # XXX perform DNS query to determine that this TLD exists - pass - return True - except: - return False - -class Config: - def __init__(self, cfg_file_name = "config.json"): - f = open(cfg_file_name) - self.cfg = json.loads(f.read()) - self.tls_policies = {} - self.mx_map = {} - for atr, val in self.cfg.items(): - # Verify each attribute of the structure - if atr.startswith("comment"): - continue - if atr == "author": - if type(val) not in [str, unicode]: - raise TypeError, "Author must be a string: " + `val` - elif atr == "timestamp": - self.timestamp = parse_timestamp(val) - elif atr == "expires": - self.expires = parse_timestamp(val) - elif atr == "tls-policies": - for domain, policies in self.check_tls_policy_domains(val): - if type(policies) != dict: - raise TypeError, domain + "'s policies should be a dict: " + `policies` - self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, v in policies.items(): - value = str(v).lower() - if policy == "require-tls": - if value in ("true", "1", "yes"): - self.tls_policies[domain]["required"] = True - elif value in ("false", "0", "no"): - self.tls_policies[domain]["required"] = False - else: - raise ValueError, "Unknown require-tls value " + `value` - elif policy == "min-tls-version": - reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] - reasonable = map(string.lower, reasonable) - if not value in reasonable: - raise ValueError, "Not a valid TLS version string: " + `value` - self.tls_policies[domain]["min-tls-version"] = str(value) - elif policy == "enforce-mode": - if value == "enforce": - self.tls_policies[domain]["enforce"] = True - elif value == "log-only": - self.tls_policies[domain]["enforce"] = False - else: - raise ValueError, "Not a known enoforcement policy " + `value` - elif atr == "acceptable-mxs": - self.acceptable_mxs = val - self.mx_domain_to_address_domains = collections.defaultdict(set) - for address_domain, properties in self.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - self.mx_domain_to_address_domains[mx_domain].add(address_domain) - pass - else: - sys.stderr.write("Unknown attribute: " + `atr` + "\n") - # XXX is it ever permissible to have a domain with an acceptable-mx - # that does not point to a TLS security policy? If not, check/warn/fail - # here - - def get_address_domains(self, mx_hostname): - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in self.mx_domain_to_address_domains: - return self.mx_domain_to_address_domains[parent] - return None - - def check_tls_policy_domains(self, val): - if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` - for domain, policies in val.items(): - try: - assert type(domain) == unicode - d = str(domain) # convert from unicode - except: - raise TypeError, "tls-policy domain not a string" + `domain` - yield (d, policies) - -if __name__ == "__main__": - c = Config() From 87022782fb0fa8431632a5713f12927874d0a7b3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:59:10 -0800 Subject: [PATCH 107/631] Catch stray missing line --- letsencrypt-postfix/PostfixConfigGenerator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index af29d493c..20ca3edb4 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -76,6 +76,7 @@ class PostfixConfigGenerator: policy_cf_entry = "texthash:" + self.policy_file self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) def maybe_add_config_lines(self): From e42a222c5d54d08836371200afb0af30106d3685 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 15:02:59 +0200 Subject: [PATCH 108/631] preliminary Postfix version check --- letsencrypt-postfix/PostfixConfigGenerator.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 25cab9ce7..162a0d832 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys import string +import subprocess import os, os.path def parse_line(line_data): @@ -148,13 +149,29 @@ class PostfixConfigGenerator: :raises .NoInstallationError: when the necessary programs/files cannot be located. Plugin will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - """ + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + """ # XXX ensure we raise the right kinds of exceptions self.postfix_cf_file = self.find_postfix_cf() + # Parse Postfix version number (feature support, syntax changes etc.) + mail_version = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], \ + stdout=subprocess.PIPE) \ + .communicate()[0].split()[2] + maj, min, rev = mail_version.split('.') + + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html + self.postfix_version = mail_version + + return maj, min, rev def more_info(self): """Human-readable string to help the user. From 1cde7f9b5429bc70bee06c38034b51947c5cb758 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 16:19:34 +0200 Subject: [PATCH 109/631] added doc. on postfix version dependent features --- letsencrypt-postfix/PostfixConfigGenerator.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 162a0d832..5e219cc43 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -162,6 +162,7 @@ class PostfixConfigGenerator: stdout=subprocess.PIPE) \ .communicate()[0].split()[2] maj, min, rev = mail_version.split('.') + self.postfix_version = mail_version # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -169,8 +170,44 @@ class PostfixConfigGenerator: # see: # http://www.postfix.org/TLS_README.html # http://www.postfix.org/FORWARD_SECRECY_README.html - self.postfix_version = mail_version + + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added + + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added + + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html + return maj, min, rev def more_info(self): From 1d37e94e17b552712d1ade5f981aed8ba4835b17 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 18:43:08 +0200 Subject: [PATCH 110/631] disable SSLv3 and v3 by default #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 4b6783633..dfb20ccea 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -79,6 +79,10 @@ class PostfixConfigGenerator: self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) + # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed + # between Postfix version 2.5 and 2.6, since we only support => 2.11 + # we don't use nor support legacy Postfix syntax. + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: From 0ba508ee2dc88bd86aa4209efd55c498e394ffea Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 19:02:00 +0200 Subject: [PATCH 111/631] disable SSLv2,3 client-side too #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index dfb20ccea..609477f0b 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -82,7 +82,10 @@ class PostfixConfigGenerator: # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed # between Postfix version 2.5 and 2.6, since we only support => 2.11 # we don't use nor support legacy Postfix syntax. + # - Server: self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + # - Client: + self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: From 2900d5122ca91531fe9696fba4a13de0f7cefb8c Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 19:29:46 +0200 Subject: [PATCH 112/631] set _all_ client&server options to exclude v2 and v3 #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 609477f0b..2066b74e2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -83,8 +83,10 @@ class PostfixConfigGenerator: # between Postfix version 2.5 and 2.6, since we only support => 2.11 # we don't use nor support legacy Postfix syntax. # - Server: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) # - Client: + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): From 5cc317408ca73c7b741a65db5b20ccde4e032402 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:19:13 -0700 Subject: [PATCH 113/631] Move attributes into init and allow for injecting file contents for testing. --- letsencrypt-postfix/PostfixConfigGenerator.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index af1953208..34654a2d1 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -3,6 +3,7 @@ import sys import string import os, os.path + def parse_line(line_data): """ Return the (line number, left hand side, right hand side) of a stripped @@ -17,15 +18,30 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) + class ExistingConfigError(ValueError): pass + class PostfixConfigGenerator: - def __init__(self, policy_config, postfix_dir, fixup=False): + def __init__(self, policy_config, postfix_dir, fixup=False, fopen=open): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = fopen(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + self.policy_lines = [] + self.new_cf = "" + + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.postfix_dir, "main.cf") + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -60,13 +76,6 @@ class PostfixConfigGenerator: Try to ensure/mutate that the config file is in a sane state. Fixup means we'll delete existing lines if necessary to get there. """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - # Check we're currently accepting inbound STARTTLS sensibly self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction @@ -91,7 +100,6 @@ class PostfixConfigGenerator: if self.raw_cf[-1][-1] == "\n": sep = "" else: sep = "\n" - self.new_cf = "" for num, line in enumerate(self.raw_cf): if self.fixup and num in self.deletions: self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line @@ -104,12 +112,7 @@ class PostfixConfigGenerator: f.write(self.new_cf) f.close() - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - def set_domainwise_tls_policies(self): - self.policy_lines = [] all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -152,7 +155,6 @@ class PostfixConfigGenerator: currently supported. """ # XXX ensure we raise the right kinds of exceptions - self.postfix_cf_file = self.find_postfix_cf() def more_info(self): From fee9c862334bfd290a2b9bd1e07b93badfb66ae5 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:20:33 -0700 Subject: [PATCH 114/631] Add failing test for get_all_names. --- .../TestPostfixConfigGenerator.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 letsencrypt-postfix/TestPostfixConfigGenerator.py diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py new file mode 100644 index 000000000..98a0afb5d --- /dev/null +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import io +import logging +import unittest + +import Config +import PostfixConfigGenerator as pcg + + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + + +# Fake Postfix Configs +names_only_config = """myhostname = mail.fubard.org +mydomain = fubard.org +myorigin = fubard.org""" + + +def GetFakeOpen(fake_file_contents): + fake_file = io.StringIO() + # cast this to unicode for py2 + fake_file.write(fake_file_contents) + fake_file.seek(0) + + def FakeOpen(_): + return fake_file + + return FakeOpen + + +class TestPostfixConfigGenerator(unittest.TestCase): + + def setUp(self): + self.fopen_names_only_config = GetFakeOpen(names_only_config) + #self.config = Config.Config() + self.config = None + self.postfix_dir = 'tests/' + + def tearDown(self): + pass + + def testGetAllNames(self): + sorted_names = ('fubard.org', 'mail.fubard.org') + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_names_only_config + ) + self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + + +if __name__ == '__main__': + unittest.main() From fd1cef3fa03d026e1a0936a84985d68f67b98614 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:31:27 -0700 Subject: [PATCH 115/631] Implement get_all_names. --- letsencrypt-postfix/PostfixConfigGenerator.py | 9 +++++++++ letsencrypt-postfix/TestPostfixConfigGenerator.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 34654a2d1..9863d05d2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -171,6 +171,15 @@ class PostfixConfigGenerator: """Returns all names that may be authenticated. :rtype: `list` of `str` """ + var_names = ('myhostname', 'mydomain', 'myorigin') + names_found = set() + for num, line in enumerate(self.cf): + num, found_var, found_value = parse_line((num, line)) + if found_var in var_names: + names_found.add(found_value) + name_list = list(names_found) + name_list.sort() + return name_list def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index 98a0afb5d..e1d921c6f 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -48,7 +48,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): pass def testGetAllNames(self): - sorted_names = ('fubard.org', 'mail.fubard.org') + sorted_names = ['fubard.org', 'mail.fubard.org'] postfix_config_gen = pcg.PostfixConfigGenerator( self.config, self.postfix_dir, From 0bf2537a55db2752ee5a8f536b80e5e544128155 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:32:53 -0700 Subject: [PATCH 116/631] Add initial gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 94c6f7089..cc957df18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .* *.orig +*.pyc From 1f95ac964071cb904585599ac565fd0fe7dda914 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Wed, 20 Apr 2016 12:43:51 +0700 Subject: [PATCH 117/631] README fixup pt. 1 --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 27563a899..774556e7e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ objectives: * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready * DANE support: none yet +* DEEP support: none yet * SMTP-STS integration: none yet * Direct mechanisms for mail domains to request inclusion: none yet * Failure reporting mechanisms: early progress, not yet ready @@ -41,7 +42,10 @@ objectives: ## Authors -Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox , Aaron Zauner +Jacob Hoffman-Andrews , +Peter Eckersley , +Daniel Wilcox , +Aaron Zauner ## Mailing List @@ -49,13 +53,13 @@ starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-eve ## Background -Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. +Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to trivial downgrade attacks. -To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. +To illustrate an easy version of this attack, suppose a network-based attacker `Mallory` notices that `Alice` has just uploaded message to her mail server. `Mallory` can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So `Alice`'s message will be transmitted in the clear. Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM or Denial of Service is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. @@ -63,7 +67,8 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d * Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. * Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. -* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring +* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring. +* Create feedback-loops on targeted attacks and bulk surveilance in an opt-in, anonymized way. ## Non-goals @@ -86,7 +91,7 @@ Attacker has control of routers on the path between two MTAs of interest. Attack ## Alternatives -Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of a DANE solution easier. +Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of DANE-based solutions easier. ## Detailed design @@ -147,7 +152,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. -The _timestamp_ field is an integer number of epoch seconds. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. +The _timestamp_ field is an integer number of epoch seconds from 00:00:00 UTC on 1 January 1970. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. @@ -183,7 +188,7 @@ The _accept-pinset_ field references an entry in the pinsets list, which has the ## Pinning and hostname verification -Like Chrome (and soon Firefox) we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. +Like Chrome and Firefox we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. @@ -194,7 +199,7 @@ We do not attempt to solve the self-signed certificate problem. For mail hosts w We have three options for creating the configuration file: 1. Ask mail operators to submit policies for their domains which we incorporate. -2. Manually curate a set of policies for the top N mail domains. +2. Manually curate a set of policies for the top `N` mail domains. 3. Programmatically create a set of policies by connecting to the top N mail domains. For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. @@ -225,6 +230,6 @@ Additionally, for ongoing monitoring of third-party deployments, we will create ## Failure reporting -For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job and send notices only when STARTTLS Everywhere-related failures exceed 0.1% for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. +For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job or small opt-in daemon and send notices only when STARTTLS Everywhere-related failures exceed a certain percentage for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. From cc83e9ba52d9bd13dd43e72942458e95b0e6fba8 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 12:26:56 -0700 Subject: [PATCH 118/631] Wrap some lines, new style exceptions, return check for restart. --- letsencrypt-postfix/PostfixConfigGenerator.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 1ac9d9fc6..927b7e729 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -28,7 +28,8 @@ class PostfixConfigGenerator: self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.policy_file = os.path.join(postfix_dir, + "starttls_everywhere_policy") self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") self.additions = [] @@ -51,7 +52,8 @@ class PostfixConfigGenerator: """ acceptable = [ideal] + also_acceptable - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] + l = [(num,line) for num,line in enumerate(self.cf) + if line.startswith(var)] if not any(l): self.additions.append(var + " = " + ideal) else: @@ -62,14 +64,18 @@ class PostfixConfigGenerator: self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Conflicting existing config values " + `l` + raise ExistingConfigError( + "Conflicting existing config values " + `l` + ) val = values[0][2] if val not in acceptable: if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + raise ExistingConfigError( + "Existing config has %s=%s"%(var,val) + ) def wrangle_existing_config(self): """ @@ -96,12 +102,14 @@ class PostfixConfigGenerator: # - Client: self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - def maybe_add_config_lines(self): + def maybe_add_config_lines(self, fopen=open): if not self.additions: return if self.fixup: print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + self.additions[:0]=["#", + "# New config lines added by STARTTLS Everywhere", + "#"] new_cf_lines = "\n".join(self.additions) + "\n" print "Adding to %s:" % self.fn print new_cf_lines @@ -118,10 +126,10 @@ class PostfixConfigGenerator: if not os.access(self.postfix_cf_file, os.W_OK): raise Exception("Can't write to %s, please re-run as root." % self.postfix_cf_file) - with open(self.fn, "w") as f: + with fopen(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self): + def set_domainwise_tls_policies(self, fopen=open): all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -142,9 +150,8 @@ class PostfixConfigGenerator: print mx_policy.min_tls_version self.policy_lines.append(entry) - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() + with fopen(self.policy_file, "w") as f: + f.write("\n".join(self.policy_lines) + "\n") ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 @@ -335,17 +342,19 @@ class PostfixConfigGenerator: """ print "Reloading postfix config..." if os.geteuid() != 0: - os.system("sudo service postfix reload") + rc = os.system("sudo service postfix reload") else: - os.system("service postfix reload") + rc = os.system("service postfix reload") + if rc != 0: + raise Exception('PluginError: cannot restart postfix') def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) def usage(): - print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % - sys.argv[0]) + print ("Usage: %s starttls-everywhere.json /etc/postfix " + "/etc/letsencrypt/live/example.com/" % sys.argv[0]) sys.exit(1) From e75bafa43908d9f66f94ffae1393f18c9262f5a1 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 12:27:49 -0700 Subject: [PATCH 119/631] Add basic test for get_all_certs_keys IInstaller interface method. --- .../TestPostfixConfigGenerator.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index e1d921c6f..ebcf12a4c 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -24,6 +24,12 @@ mydomain = fubard.org myorigin = fubard.org""" +certs_only_config = """ +smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem +smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem +""" + + def GetFakeOpen(fake_file_contents): fake_file = io.StringIO() # cast this to unicode for py2 @@ -40,6 +46,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.fopen_names_only_config = GetFakeOpen(names_only_config) + self.fopen_certs_only_config = GetFakeOpen(certs_only_config) #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' @@ -57,6 +64,18 @@ class TestPostfixConfigGenerator(unittest.TestCase): ) self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + def testGetAllCertAndKeys(self): + return_vals = ('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', + '/etc/letsencrypt/live/www.fubard.org/privkey.pem', + None) + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_certs_only_config + ) + self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + if __name__ == '__main__': unittest.main() From c6baa82ee4030bc75c8da3ea60ff22865d3a9137 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 15:14:06 -0700 Subject: [PATCH 120/631] Implement basic get_all_certs_keys, tests pass. --- letsencrypt-postfix/PostfixConfigGenerator.py | 11 +++++++++++ letsencrypt-postfix/TestPostfixConfigGenerator.py | 13 ++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 927b7e729..b5adba1a2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -296,6 +296,17 @@ class PostfixConfigGenerator: - `path` - file path to configuration file :rtype: list """ + cert_materials = {'smtpd_tls_key_file': None, + 'smtpd_tls_cert_file': None, + } + for num, line in enumerate(self.cf): + print 'Line is: %s' % line + num, found_var, found_value = parse_line((num, line)) + if found_var in cert_materials.keys(): + cert_materials[found_var] = found_value + return [(cert_materials['smtpd_tls_cert_file'], + cert_materials['smtpd_tls_key_file'], + self.fn),] def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index ebcf12a4c..57cbb59c0 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -24,10 +24,9 @@ mydomain = fubard.org myorigin = fubard.org""" -certs_only_config = """ -smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem -smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem -""" +certs_only_config = ( +"""smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem +smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") def GetFakeOpen(fake_file_contents): @@ -65,9 +64,9 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): - return_vals = ('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', - '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - None) + return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', + '/etc/letsencrypt/live/www.fubard.org/privkey.pem', + 'tests/main.cf'),] postfix_config_gen = pcg.PostfixConfigGenerator( self.config, self.postfix_dir, From 7edceec8ac976ea5680ab081d8884ec6e6ff2a2c Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 15:27:11 -0700 Subject: [PATCH 121/631] Add test case and fix to properly handle configs with no smtpd_tls_* vars. --- letsencrypt-postfix/PostfixConfigGenerator.py | 12 ++++++++---- letsencrypt-postfix/TestPostfixConfigGenerator.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index b5adba1a2..859d86ffe 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -300,13 +300,17 @@ class PostfixConfigGenerator: 'smtpd_tls_cert_file': None, } for num, line in enumerate(self.cf): - print 'Line is: %s' % line num, found_var, found_value = parse_line((num, line)) if found_var in cert_materials.keys(): cert_materials[found_var] = found_value - return [(cert_materials['smtpd_tls_cert_file'], - cert_materials['smtpd_tls_key_file'], - self.fn),] + + if not all(cert_materials.values()): + cert_material_tuples = [] + else: + cert_material_tuples = [(cert_materials['smtpd_tls_cert_file'], + cert_materials['smtpd_tls_key_file'], + self.fn),] + return cert_material_tuples def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index 57cbb59c0..4a96aa30f 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -46,6 +46,8 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.fopen_names_only_config = GetFakeOpen(names_only_config) self.fopen_certs_only_config = GetFakeOpen(certs_only_config) + self.fopen_no_certs_only_config = self.fopen_names_only_config + #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' @@ -75,6 +77,15 @@ class TestPostfixConfigGenerator(unittest.TestCase): ) self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + def testGetAllCertsAndKeys_With_None(self): + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_no_certs_only_config + ) + self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + if __name__ == '__main__': unittest.main() From 4d24eb83a82095b7a740263629bbc55f7972e0f9 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:11:37 -0700 Subject: [PATCH 122/631] Move version fetching into get_version and implement more_info method. --- letsencrypt-postfix/PostfixConfigGenerator.py | 57 ++++++++++++++++--- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 859d86ffe..0163e4bff 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -24,7 +24,12 @@ class ExistingConfigError(ValueError): pass class PostfixConfigGenerator: - def __init__(self, policy_config, postfix_dir, fixup=False, fopen=open): + def __init__(self, + policy_config, + postfix_dir, + fixup=False, + fopen=open, + version=None): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config @@ -41,6 +46,9 @@ class PostfixConfigGenerator: self.policy_lines = [] self.new_cf = "" + # Set in .prepare() unless running in a test + self.postfix_version = version + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -174,13 +182,14 @@ class PostfixConfigGenerator: """ # XXX ensure we raise the right kinds of exceptions - # Parse Postfix version number (feature support, syntax changes etc.) - mail_version = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], \ - stdout=subprocess.PIPE) \ - .communicate()[0].split()[2] - maj, min, rev = mail_version.split('.') - self.postfix_version = mail_version - + if not self.postfix_version: + self.postfix_version = self.get_version() + + if self.postfix_version < (2, 11, 0): + raise Exception( + 'NotSupportedError: Postfix version is too old -- test.' + ) + # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend # differences here and pass them onto other configuration functions. @@ -225,7 +234,28 @@ class PostfixConfigGenerator: # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html - return maj, min, rev + def get_version(self): + """Return the mail version of Postfix. + + Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + + :returns: version + :rtype: tuple + + :raises .PluginError: + Unable to find Postfix version. + """ + # Parse Postfix version number (feature support, syntax changes etc.) + cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + stdout=subprocess.PIPE) + stdout, _ = cmd.communicate() + if cmd.returncode != 0: + raise Exception('PluginError: Unable to determine Postfix version.') + + # grabs version component of string like "mail_version = 2.11.3" + mail_version = stdout.split()[2] + postfix_version = tuple([int(i) for i in mail_version.split('.')]) + return postfix_version def more_info(self): """Human-readable string to help the user. @@ -233,6 +263,15 @@ class PostfixConfigGenerator: decide which plugin to use. :rtype str: """ + return ( + "Configures Postfix to try to authenticate mail servers, use " + "installed certificates and disable weak ciphers and protocols.{0}" + "Server root: {root}{0}" + "Version: {version}".format( + os.linesep, + root=self.postfix_dir, + version='.'.join([str(i) for i in self.postfix_version])) + ) ### Let's Encrypt client IInstaller ### From c43602c90808d19e78bd2ee213f007e6ee7fc9ad Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:30:08 -0700 Subject: [PATCH 123/631] Add simple config_test implementation. --- letsencrypt-postfix/PostfixConfigGenerator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 0163e4bff..166206560 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -364,7 +364,6 @@ class PostfixConfigGenerator: be quickly reversed in the future (challenges) :raises .PluginError: when save is unsuccessful """ - self.maybe_add_config_lines() def rollback_checkpoints(self, rollback=1): @@ -389,6 +388,12 @@ class PostfixConfigGenerator: """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ + if os.geteuid() != 0: + rc = os.system('sudo /usr/sbin/postfix check') + else: + rc = os.system('/usr/sbin/postfix check') + if rc != 0: + raise Exception('MisconfigurationError: Postfix failed self-check.') def restart(self): """Restart or refresh the server content. From 5d07b702695f08dfd0fb35a59f74c411f756c22c Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:40:06 -0700 Subject: [PATCH 124/631] Change over to using logging module from print statements. --- letsencrypt-postfix/PostfixConfigGenerator.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 166206560..43a8fa116 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,10 +1,15 @@ #!/usr/bin/env python + +import logging import sys import string import subprocess import os, os.path +logger = logging.getLogger(__name__) + + def parse_line(line_data): """ Return the (line number, left hand side, right hand side) of a stripped @@ -114,13 +119,13 @@ class PostfixConfigGenerator: if not self.additions: return if self.fixup: - print "Deleting lines:", self.deletions + logger.info('Deleting lines: {}'.format(self.deletions)) self.additions[:0]=["#", "# New config lines added by STARTTLS Everywhere", "#"] new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines + logger.info('Adding to {}:'.format(self.fn)) + logger.info(new_cf_lines) if self.raw_cf[-1][-1] == "\n": sep = "" else: sep = "\n" @@ -142,9 +147,12 @@ class PostfixConfigGenerator: for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported." - print "Using MX %s for %s" % (mx_list[0], address_domain) - print "Ignoring: %s" % (', '.join(mx_list[1:])) + logger.warn('Lists of multiple accept-mx-domains not yet ' + 'supported.') + logger.warn('Using MX {} for {}".format(mx_list[0], + address_domain) + ) + logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) mx_domain = mx_list[0] mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" @@ -155,7 +163,9 @@ class PostfixConfigGenerator: elif mx_policy.min_tls_version.lower() == "tlsv1.2": entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: - print mx_policy.min_tls_version + logger.warn('Unknown minimum TLS version: {} '.format( + mx_policy.min_tls_version) + ) self.policy_lines.append(entry) with fopen(self.policy_file, "w") as f: @@ -399,7 +409,7 @@ class PostfixConfigGenerator: """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ - print "Reloading postfix config..." + logger.info('Reloading postfix config...') if os.geteuid() != 0: rc = os.system("sudo service postfix reload") else: From 887871833d407f4597066ca18c07a0c204a4e097 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:44:25 -0700 Subject: [PATCH 125/631] Fix typo in changing quotes. --- letsencrypt-postfix/PostfixConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 43a8fa116..214d98954 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -149,7 +149,7 @@ class PostfixConfigGenerator: if len(mx_list) > 1: logger.warn('Lists of multiple accept-mx-domains not yet ' 'supported.') - logger.warn('Using MX {} for {}".format(mx_list[0], + logger.warn('Using MX {} for {}'.format(mx_list[0], address_domain) ) logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) From af38c30c9c506cb6e40cd4b1b32103d964ab9bee Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 17:02:29 -0700 Subject: [PATCH 126/631] Fix path to postfix config variable. --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 214d98954..8fcfb7e8b 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -136,9 +136,9 @@ class PostfixConfigGenerator: self.new_cf += line self.new_cf += sep + new_cf_lines - if not os.access(self.postfix_cf_file, os.W_OK): + if not os.access(self.fn, os.W_OK): raise Exception("Can't write to %s, please re-run as root." - % self.postfix_cf_file) + % self.fn) with fopen(self.fn, "w") as f: f.write(self.new_cf) From a5f23b5314c6afbac26c27e494d3d92603d777e1 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 17:10:21 -0700 Subject: [PATCH 127/631] Configure logger to be a touch louder... than silent --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 8fcfb7e8b..38ef0f4c9 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -8,6 +8,10 @@ import os, os.path logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +log_handler = logging.StreamHandler() +log_handler.setLevel(logging.DEBUG) +logger.addHandler(log_handler) def parse_line(line_data): From 619e273ae5ef6800c584cbe1ef9160fb6045a1e6 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 10 May 2017 15:44:55 +0200 Subject: [PATCH 128/631] Correct markdown link syntax --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 774556e7e..7413228f6 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,7 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d ## Motivating examples * [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to - prevent negotiating an encrypted connection] - (https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) + prevent negotiating an encrypted connection](https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) * [Unknown party removes STARTTLS flag from all SMTP connections leaving Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) From e2d95b371992247ba5806c5b107b5449047e8c8d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:02:56 -0700 Subject: [PATCH 129/631] Create packaging around PostfixConfigGenerator. --- certbot-postfix/LICENSE.txt | 190 ++++++++++++++++++ certbot-postfix/MANIFEST.in | 2 + certbot-postfix/README.rst | 1 + certbot-postfix/certbot_postfix/__init__.py | 3 + .../certbot_postfix/installer.py | 0 .../certbot_postfix/installer_test.py | 0 certbot-postfix/setup.cfg | 2 + certbot-postfix/setup.py | 57 ++++++ 8 files changed, 255 insertions(+) create mode 100644 certbot-postfix/LICENSE.txt create mode 100644 certbot-postfix/MANIFEST.in create mode 100644 certbot-postfix/README.rst create mode 100644 certbot-postfix/certbot_postfix/__init__.py rename letsencrypt-postfix/PostfixConfigGenerator.py => certbot-postfix/certbot_postfix/installer.py (100%) mode change 100755 => 100644 rename letsencrypt-postfix/TestPostfixConfigGenerator.py => certbot-postfix/certbot_postfix/installer_test.py (100%) create mode 100644 certbot-postfix/setup.cfg create mode 100644 certbot-postfix/setup.py diff --git a/certbot-postfix/LICENSE.txt b/certbot-postfix/LICENSE.txt new file mode 100644 index 000000000..c8314fd1c --- /dev/null +++ b/certbot-postfix/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2017 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-postfix/MANIFEST.in b/certbot-postfix/MANIFEST.in new file mode 100644 index 000000000..97e2ad3df --- /dev/null +++ b/certbot-postfix/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE.txt +include README.rst diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst new file mode 100644 index 000000000..ee88648d3 --- /dev/null +++ b/certbot-postfix/README.rst @@ -0,0 +1 @@ +Postfix plugin for Certbot diff --git a/certbot-postfix/certbot_postfix/__init__.py b/certbot-postfix/certbot_postfix/__init__.py new file mode 100644 index 000000000..703cb659b --- /dev/null +++ b/certbot-postfix/certbot_postfix/__init__.py @@ -0,0 +1,3 @@ +"""Certbot Postfix plugin.""" + +from certbot_postfix.installer import PostfixConfigGenerator as Installer diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/certbot-postfix/certbot_postfix/installer.py old mode 100755 new mode 100644 similarity index 100% rename from letsencrypt-postfix/PostfixConfigGenerator.py rename to certbot-postfix/certbot_postfix/installer.py diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/certbot-postfix/certbot_postfix/installer_test.py similarity index 100% rename from letsencrypt-postfix/TestPostfixConfigGenerator.py rename to certbot-postfix/certbot_postfix/installer_test.py diff --git a/certbot-postfix/setup.cfg b/certbot-postfix/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-postfix/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py new file mode 100644 index 000000000..df78948bb --- /dev/null +++ b/certbot-postfix/setup.py @@ -0,0 +1,57 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.18.0.dev0' + +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', +] + +setup( + name='certbot-postfix', + version=version, + description="Postfix plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Communications :: Email :: Mail Transport Agents', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + entry_points={ + 'certbot.plugins': [ + 'postfix = certbot_postfix:Installer', + ], + }, + test_suite='certbot_postfix', +) From 74b22a596e07e76626a014dac68a90fb7e50e30e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:03:30 -0700 Subject: [PATCH 130/631] Ignore egg-info dirs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cc957df18..e36c9c50b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .* *.orig *.pyc +*.egg-info/ From f89051cc2ad7ddd514cacf873d0758fece796c06 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:18:51 -0700 Subject: [PATCH 131/631] Completely implement the Certbot plugin interfaces --- certbot-postfix/certbot_postfix/installer.py | 12 +++++++++++- certbot-postfix/setup.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d660e35f5..a4500cf24 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -6,6 +6,11 @@ import string import subprocess import os, os.path +import zope.interface + +from certbot import interfaces +from certbot.plugins import common as plugins_common + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -32,7 +37,12 @@ def parse_line(line_data): class ExistingConfigError(ValueError): pass -class PostfixConfigGenerator: +@zope.interface.implementer(interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) +class PostfixConfigGenerator(plugins_common.Plugin): + + description = "Configure TLS with the Postfix MTA" + def __init__(self, policy_config, postfix_dir, diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index df78948bb..1f087ccc0 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -12,6 +12,7 @@ install_requires = [ # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'zope.interface', ] setup( From ae08dc6beaab6e6d090de85e710e2d913a0cd1aa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:24:04 -0700 Subject: [PATCH 132/631] Fix Postfix installer tests --- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 4a96aa30f..cc86653f6 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -10,8 +10,7 @@ import io import logging import unittest -import Config -import PostfixConfigGenerator as pcg +import certbot_postfix logger = logging.getLogger(__name__) @@ -57,7 +56,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, @@ -69,7 +68,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, @@ -78,7 +77,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, From 5bf4ad1f52b28badc6a0dd2d82f7d987d6cb184b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:25:12 -0700 Subject: [PATCH 133/631] Rename PostfixConfigGenerator to simply Installer --- certbot-postfix/certbot_postfix/__init__.py | 2 +- certbot-postfix/certbot_postfix/installer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/__init__.py b/certbot-postfix/certbot_postfix/__init__.py index 703cb659b..122c54bc6 100644 --- a/certbot-postfix/certbot_postfix/__init__.py +++ b/certbot-postfix/certbot_postfix/__init__.py @@ -1,3 +1,3 @@ """Certbot Postfix plugin.""" -from certbot_postfix.installer import PostfixConfigGenerator as Installer +from certbot_postfix.installer import Installer diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index a4500cf24..ac2adbdf0 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -39,7 +39,7 @@ class ExistingConfigError(ValueError): pass @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) -class PostfixConfigGenerator(plugins_common.Plugin): +class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" From c2a8ce59ae298b80cdc334f2bb62d74629c45043 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:28:22 -0700 Subject: [PATCH 134/631] Remove code to run the installer as on its own. --- certbot-postfix/certbot_postfix/installer.py | 34 -------------------- 1 file changed, 34 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index ac2adbdf0..52244ce92 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import sys import string @@ -13,10 +11,6 @@ from certbot.plugins import common as plugins_common logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -log_handler = logging.StreamHandler() -log_handler.setLevel(logging.DEBUG) -logger.addHandler(log_handler) def parse_line(line_data): @@ -435,31 +429,3 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - - -def usage(): - print ("Usage: %s starttls-everywhere.json /etc/postfix " - "/etc/letsencrypt/live/example.com/" % sys.argv[0]) - sys.exit(1) - - -if __name__ == "__main__": - import Config as config - if len(sys.argv) != 4: - usage() - c = config.Config() - c.load_from_json_file(sys.argv[1]) - postfix_dir = sys.argv[2] - le_lineage = sys.argv[3] - pieces = [os.path.join(le_lineage, f) for f in ( - "cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] - if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) : - print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" - print - usage() - cert, key, chain, fullchain = pieces - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) - pcgen.prepare() - pcgen.deploy_cert("example.com", cert, key, chain, fullchain) - pcgen.save() - pcgen.restart() From 6c4b3c08a7ce8e5453fcafcb9333fca043951e7d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:30:11 -0700 Subject: [PATCH 135/631] Clean up installer imports --- certbot-postfix/certbot_postfix/installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 52244ce92..0a109e1de 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,8 +1,8 @@ import logging -import sys +import os import string import subprocess -import os, os.path +import sys import zope.interface From 1c258c0a2c382a3a404091d1b099c19a9bd3b8ef Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:31:10 -0700 Subject: [PATCH 136/631] Add basic docstrings to Installer --- certbot-postfix/certbot_postfix/installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 0a109e1de..cd9c00714 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,3 +1,4 @@ +"""Certbot installer plugin for Postfix.""" import logging import os import string @@ -34,6 +35,7 @@ class ExistingConfigError(ValueError): pass @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): + """Certbot installer plugin for Postfix.""" description = "Configure TLS with the Postfix MTA" From 694746409f45904082f635557a933b900e34e901 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:42:26 -0700 Subject: [PATCH 137/631] s/ExistingConfigError/MisconfigurationError --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index cd9c00714..b0a075eeb 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -7,6 +7,7 @@ import sys import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import common as plugins_common @@ -29,9 +30,6 @@ def parse_line(line_data): return (num, left.strip(), right.strip()) -class ExistingConfigError(ValueError): pass - - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): @@ -72,6 +70,10 @@ class Installer(plugins_common.Plugin): """ Ensure that existing postfix config @var is in the list of @acceptable values; if not, set it to the ideal value. + + :raises .errors.MisconfigurationError: if conflicting existing values + are found for var + """ acceptable = [ideal] + also_acceptable @@ -87,8 +89,8 @@ class Installer(plugins_common.Plugin): self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError( - "Conflicting existing config values " + `l` + raise errors.MisconfigurationError( + "Conflicting existing config values {0}".format(l) ) val = values[0][2] if val not in acceptable: @@ -96,7 +98,7 @@ class Installer(plugins_common.Plugin): self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError( + raise errors.MisconfigurationError( "Existing config has %s=%s"%(var,val) ) From 61c2209110292c84d18b0bd5087ba783d3269b25 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:56:39 -0700 Subject: [PATCH 138/631] Use Certbot error types in the Postfix Installer --- certbot-postfix/certbot_postfix/installer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index b0a075eeb..00491e735 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -210,9 +210,7 @@ class Installer(plugins_common.Plugin): self.postfix_version = self.get_version() if self.postfix_version < (2, 11, 0): - raise Exception( - 'NotSupportedError: Postfix version is too old -- test.' - ) + raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -274,7 +272,7 @@ class Installer(plugins_common.Plugin): stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: - raise Exception('PluginError: Unable to determine Postfix version.') + raise errors.PluginError('Unable to determine Postfix version.') # grabs version component of string like "mail_version = 2.11.3" mail_version = stdout.split()[2] @@ -417,7 +415,7 @@ class Installer(plugins_common.Plugin): else: rc = os.system('/usr/sbin/postfix check') if rc != 0: - raise Exception('MisconfigurationError: Postfix failed self-check.') + raise errors.MisconfigurationError('Postfix failed self-check.') def restart(self): """Restart or refresh the server content. @@ -429,7 +427,7 @@ class Installer(plugins_common.Plugin): else: rc = os.system("service postfix reload") if rc != 0: - raise Exception('PluginError: cannot restart postfix') + raise errors.MisconfigurationError('cannot restart postfix') def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) From 66ba0b5276b63ccdfc0c986f1c751d214f41f8b7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:57:44 -0700 Subject: [PATCH 139/631] Remove invalid permissions exception. Once things like locks are added, this error shouldn't be possible as it will have occurred earlier. --- certbot-postfix/certbot_postfix/installer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 00491e735..5196e1e64 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -150,9 +150,6 @@ class Installer(plugins_common.Plugin): self.new_cf += line self.new_cf += sep + new_cf_lines - if not os.access(self.fn, os.W_OK): - raise Exception("Can't write to %s, please re-run as root." - % self.fn) with fopen(self.fn, "w") as f: f.write(self.new_cf) From 4a3fd19c93b8dc5543859459d920662138274156 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:00:04 -0700 Subject: [PATCH 140/631] Move parse_line to the end of installer.py --- certbot-postfix/certbot_postfix/installer.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 5196e1e64..da573baaa 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -15,21 +15,6 @@ from certbot.plugins import common as plugins_common logger = logging.getLogger(__name__) -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) - - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): @@ -428,3 +413,18 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + + +def parse_line(line_data): + """ + Return the (line number, left hand side, right hand side) of a stripped + postfix config line. + + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + num,line = line_data + left, sep, right = line.partition("=") + if not sep: + return None + return (num, left.strip(), right.strip()) From b37be61807159443442f1df738c1caf16849a40c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:12:54 -0700 Subject: [PATCH 141/631] Import installer module directly in tests. --- certbot-postfix/certbot_postfix/installer_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index cc86653f6..7ee40a955 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -10,7 +10,7 @@ import io import logging import unittest -import certbot_postfix +from certbot_postfix import installer logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, @@ -68,7 +68,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, @@ -77,7 +77,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, From b50a71ff4e0b95dd9e491ea89bf2ce1bb1cd7bd6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:19:46 -0700 Subject: [PATCH 142/631] Remove fopen argument in favor of mock. This simplifies the actual production code and is a more standard approach in Python. --- certbot-postfix/certbot_postfix/installer.py | 11 ++-- .../certbot_postfix/installer_test.py | 63 +++++++------------ 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index da573baaa..96fc1b4d1 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -26,7 +26,6 @@ class Installer(plugins_common.Plugin): policy_config, postfix_dir, fixup=False, - fopen=open, version=None): self.fixup = fixup self.postfix_dir = postfix_dir @@ -38,7 +37,7 @@ class Installer(plugins_common.Plugin): self.additions = [] self.deletions = [] self.fn = self.find_postfix_cf() - self.raw_cf = fopen(self.fn).readlines() + self.raw_cf = open(self.fn).readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] self.policy_lines = [] @@ -114,7 +113,7 @@ class Installer(plugins_common.Plugin): self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - def maybe_add_config_lines(self, fopen=open): + def maybe_add_config_lines(self): if not self.additions: return if self.fixup: @@ -135,10 +134,10 @@ class Installer(plugins_common.Plugin): self.new_cf += line self.new_cf += sep + new_cf_lines - with fopen(self.fn, "w") as f: + with open(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self, fopen=open): + def set_domainwise_tls_policies(self): all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -164,7 +163,7 @@ class Installer(plugins_common.Plugin): ) self.policy_lines.append(entry) - with fopen(self.policy_file, "w") as f: + with open(self.policy_file, "w") as f: f.write("\n".join(self.policy_lines) + "\n") ### Let's Encrypt client IPlugin ### diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 7ee40a955..93aa171a9 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -6,10 +6,12 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import io import logging import unittest +import mock +import six + from certbot_postfix import installer @@ -28,61 +30,44 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -def GetFakeOpen(fake_file_contents): - fake_file = io.StringIO() - # cast this to unicode for py2 - fake_file.write(fake_file_contents) - fake_file.seek(0) - - def FakeOpen(_): - return fake_file - - return FakeOpen - - class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): - self.fopen_names_only_config = GetFakeOpen(names_only_config) - self.fopen_certs_only_config = GetFakeOpen(certs_only_config) - self.fopen_no_certs_only_config = self.fopen_names_only_config - - #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' - def tearDown(self): - pass - def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_names_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(names_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_certs_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(certs_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_no_certs_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(names_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual([], postfix_config_gen.get_all_certs_keys()) From 49cdfcec06564a9c56b1f592225b8600d0417cda Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:20:36 -0700 Subject: [PATCH 143/631] add six dependency --- certbot-postfix/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 1f087ccc0..2fdcd46aa 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -12,6 +12,7 @@ install_requires = [ # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'six', 'zope.interface', ] From a66500ea386e16d88b858469707436f767c3fa18 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:25:09 -0700 Subject: [PATCH 144/631] Remove unused version argument. --- certbot-postfix/certbot_postfix/installer.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 96fc1b4d1..7fa32b8f5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -25,8 +25,7 @@ class Installer(plugins_common.Plugin): def __init__(self, policy_config, postfix_dir, - fixup=False, - version=None): + fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config @@ -43,9 +42,6 @@ class Installer(plugins_common.Plugin): self.policy_lines = [] self.new_cf = "" - # Set in .prepare() unless running in a test - self.postfix_version = version - def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -187,10 +183,7 @@ class Installer(plugins_common.Plugin): """ # XXX ensure we raise the right kinds of exceptions - if not self.postfix_version: - self.postfix_version = self.get_version() - - if self.postfix_version < (2, 11, 0): + if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions @@ -273,7 +266,7 @@ class Installer(plugins_common.Plugin): "Version: {version}".format( os.linesep, root=self.postfix_dir, - version='.'.join([str(i) for i in self.postfix_version])) + version='.'.join([str(i) for i in self.get_version()])) ) From 6c5a8423b865881dddc15a1ee7b60c69d42faf19 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:28:52 -0700 Subject: [PATCH 145/631] Remove unused logger from tests --- certbot-postfix/certbot_postfix/installer_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 93aa171a9..db35511ac 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -15,10 +15,6 @@ import six from certbot_postfix import installer -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org From a15fe57225e06c9c00b3d21b2b2176eb0027038f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Aug 2017 15:57:19 -0700 Subject: [PATCH 146/631] remove policy config param --- certbot-postfix/certbot_postfix/installer.py | 60 +++++++++---------- .../certbot_postfix/installer_test.py | 4 -- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 7fa32b8f5..a6c1cd9fe 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -23,12 +23,10 @@ class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" def __init__(self, - policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") @@ -133,35 +131,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self): - all_acceptable_mxs = self.policy_config.acceptable_mxs - for address_domain, properties in all_acceptable_mxs.items(): - mx_list = properties.accept_mx_domains - if len(mx_list) > 1: - logger.warn('Lists of multiple accept-mx-domains not yet ' - 'supported.') - logger.warn('Using MX {} for {}'.format(mx_list[0], - address_domain) - ) - logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) - mx_domain = mx_list[0] - mx_policy = self.policy_config.get_tls_policy(mx_domain) - entry = address_domain + " encrypt" - if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2:!SSLv3" - elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - else: - logger.warn('Unknown minimum TLS version: {} '.format( - mx_policy.min_tls_version) - ) - self.policy_lines.append(entry) - - with open(self.policy_file, "w") as f: - f.write("\n".join(self.policy_lines) + "\n") - ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 @@ -406,6 +375,35 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + # def set_domainwise_tls_policies(self): + # all_acceptable_mxs = self.policy_config.acceptable_mxs + # for address_domain, properties in all_acceptable_mxs.items(): + # mx_list = properties.accept_mx_domains + # if len(mx_list) > 1: + # logger.warn('Lists of multiple accept-mx-domains not yet ' + # 'supported.') + # logger.warn('Using MX {} for {}'.format(mx_list[0], + # address_domain) + # ) + # logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) + # mx_domain = mx_list[0] + # mx_policy = self.policy_config.get_tls_policy(mx_domain) + # entry = address_domain + " encrypt" + # if mx_policy.min_tls_version.lower() == "tlsv1": + # entry += " protocols=!SSLv2:!SSLv3" + # elif mx_policy.min_tls_version.lower() == "tlsv1.1": + # entry += " protocols=!SSLv2:!SSLv3:!TLSv1" + # elif mx_policy.min_tls_version.lower() == "tlsv1.2": + # entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" + # else: + # logger.warn('Unknown minimum TLS version: {} '.format( + # mx_policy.min_tls_version) + # ) + # self.policy_lines.append(entry) + + # with open(self.policy_file, "w") as f: + # f.write("\n".join(self.policy_lines) + "\n") + def parse_line(line_data): """ diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index db35511ac..04d86468e 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -29,7 +29,6 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): - self.config = None self.postfix_dir = 'tests/' def testGetAllNames(self): @@ -37,7 +36,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) @@ -50,7 +48,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) @@ -60,7 +57,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) From d97a15861bc96eb7638bb9af80bdf105f47af9bc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Aug 2017 16:06:06 -0700 Subject: [PATCH 147/631] Add --postfix-config-dir argument --- certbot-postfix/certbot_postfix/installer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index a6c1cd9fe..6175c06f3 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -22,6 +22,12 @@ class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" + @classmethod + def add_parser_arguments(cls, add): + add("config-dir", help="Path to the directory containing the " + "Postfix main.cf file to modify instead of using the " + "default configuration paths") + def __init__(self, postfix_dir, fixup=False): From 481fb8413b6f6a68a403e2080834abfdf589aea2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 08:50:08 -0700 Subject: [PATCH 148/631] Fix Postfix Installer __init__() --- certbot-postfix/certbot_postfix/installer.py | 13 ++++----- .../certbot_postfix/installer_test.py | 29 +++++++++++-------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 6175c06f3..5f9fe7322 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -28,14 +28,13 @@ class Installer(plugins_common.Plugin): "Postfix main.cf file to modify instead of using the " "default configuration paths") - def __init__(self, - postfix_dir, - fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, + def __init__(self, *args, **kwargs): + super(Installer, self).__init__(*args, **kwargs) + self.fixup = False + self.postfix_dir = self.conf("config-dir") + self.policy_file = os.path.join(self.postfix_dir, "starttls_everywhere_policy") - self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") + self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] self.deletions = [] diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 04d86468e..b9e9e1463 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -35,10 +35,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): @@ -47,21 +44,29 @@ class TestPostfixConfigGenerator(unittest.TestCase): 'tests/main.cf'),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def _create_installer(self): + """Creates and returns a new Postfix Installer. + + :returns: a new Postfix installer + :rtype: certbot_postfix.installer.Installer + + """ + config = mock.MagicMock(postfix_config_dir=self.postfix_dir) + name = "postfix" + + from certbot_postfix import installer + return installer.Installer(config, name) + + if __name__ == '__main__': unittest.main() From 2a217189a6cbf9768bcdc9a4783cc1340caf500b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 10:58:27 -0700 Subject: [PATCH 149/631] (temporarily) remove policy_file --- certbot-postfix/certbot_postfix/installer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 5f9fe7322..c3bf2a189 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -32,8 +32,6 @@ class Installer(plugins_common.Plugin): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.postfix_dir = self.conf("config-dir") - self.policy_file = os.path.join(self.postfix_dir, - "starttls_everywhere_policy") self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] @@ -97,9 +95,9 @@ class Installer(plugins_common.Plugin): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file + # policy_cf_entry = "texthash:" + self.policy_file - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed From 89ae874f890bde600f12e52ae56508631225f5f1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:03:47 -0700 Subject: [PATCH 150/631] (temporarily) remove ca-certificates logic --- certbot-postfix/certbot_postfix/installer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index c3bf2a189..2663e44fd 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -32,7 +32,6 @@ class Installer(plugins_common.Plugin): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.postfix_dir = self.conf("config-dir") - self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] self.deletions = [] @@ -98,7 +97,7 @@ class Installer(plugins_common.Plugin): # policy_cf_entry = "texthash:" + self.policy_file # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) + # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed # between Postfix version 2.5 and 2.6, since we only support => 2.11 @@ -375,9 +374,9 @@ class Installer(plugins_common.Plugin): if rc != 0: raise errors.MisconfigurationError('cannot restart postfix') - def update_CAfile(self): - os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - + # def update_CAfile(self): + # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + # # def set_domainwise_tls_policies(self): # all_acceptable_mxs = self.policy_config.acceptable_mxs # for address_domain, properties in all_acceptable_mxs.items(): From 86fe5ad3623d590b3f93212a712b38a9f8234bdd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:16:46 -0700 Subject: [PATCH 151/631] Move calls to postconf to prepare(). --- certbot-postfix/certbot_postfix/installer.py | 11 +++++--- .../certbot_postfix/installer_test.py | 28 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 2663e44fd..43037e886 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -35,12 +35,11 @@ class Installer(plugins_common.Plugin): self.additions = [] self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] self.policy_lines = [] self.new_cf = "" + self.fn = None + self.raw_cf = [] + self.cf = [] def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -152,6 +151,10 @@ class Installer(plugins_common.Plugin): currently supported. :rtype tuple: """ + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] # XXX ensure we raise the right kinds of exceptions if self.get_version() < (2, 11, 0): diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index b9e9e1463..9be3744e3 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -12,9 +12,6 @@ import unittest import mock import six -from certbot_postfix import installer - - # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org @@ -35,7 +32,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): @@ -44,15 +41,34 @@ class TestPostfixConfigGenerator(unittest.TestCase): 'tests/main.cf'),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def _create_prepared_installer(self): + """Creates and returns a new prepared Postfix Installer. + + Calls in prepare() are mocked out so the Postfix version check + is successful. + + :returns: a prepared Postfix installer + :rtype: certbot_postfix.installer.Installer + + """ + installer = self._create_installer() + + popen_path = "certbot_postfix.installer.subprocess.Popen" + with mock.patch(popen_path) as mock_popen: + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + installer.prepare() + + return installer def _create_installer(self): """Creates and returns a new Postfix Installer. From a2dbf2fe4cc03750c1d9dfb2f4b4d185ce6ed104 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:20:21 -0700 Subject: [PATCH 152/631] Fix spacing --- certbot-postfix/certbot_postfix/installer.py | 119 ++++++++++--------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 43037e886..aade01f9b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -98,15 +98,15 @@ class Installer(plugins_common.Plugin): # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed - # between Postfix version 2.5 and 2.6, since we only support => 2.11 - # we don't use nor support legacy Postfix syntax. - # - Server: - self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - # - Client: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) + # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed + # between Postfix version 2.5 and 2.6, since we only support => 2.11 + # we don't use nor support legacy Postfix syntax. + # - Server: + self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) + # - Client: + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: @@ -137,7 +137,9 @@ class Installer(plugins_common.Plugin): def prepare(self): """Prepare the plugin. + Finish up any additional initialization. + :raises .PluginError: when full initialization cannot be completed. :raises .MisconfigurationError: @@ -146,11 +148,12 @@ class Installer(plugins_common.Plugin): :raises .NoInstallationError: when the necessary programs/files cannot be located. Plugin will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: - """ + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + + """ self.fn = self.find_postfix_cf() self.raw_cf = open(self.fn).readlines() self.cf = map(string.strip, self.raw_cf) @@ -160,49 +163,49 @@ class Installer(plugins_common.Plugin): if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html def get_version(self): """Return the mail version of Postfix. @@ -212,12 +215,12 @@ class Installer(plugins_common.Plugin): :returns: version :rtype: tuple - :raises .PluginError: - Unable to find Postfix version. + :raises .PluginError: Unable to find Postfix version. + """ - # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], - stdout=subprocess.PIPE) + # Parse Postfix version number (feature support, syntax changes etc.) + cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: raise errors.PluginError('Unable to determine Postfix version.') @@ -225,7 +228,7 @@ class Installer(plugins_common.Plugin): # grabs version component of string like "mail_version = 2.11.3" mail_version = stdout.split()[2] postfix_version = tuple([int(i) for i in mail_version.split('.')]) - return postfix_version + return postfix_version def more_info(self): """Human-readable string to help the user. From b395b72d1ba6704ea26489840645858001976714 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:51:01 -0700 Subject: [PATCH 153/631] Don't hardcode postconf path. --- certbot-postfix/certbot_postfix/installer.py | 165 ++++++++++-------- .../certbot_postfix/installer_test.py | 10 +- 2 files changed, 99 insertions(+), 76 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index aade01f9b..394ed5525 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -9,7 +9,9 @@ import zope.interface from certbot import errors from certbot import interfaces +from certbot import util from certbot.plugins import common as plugins_common +from certbot.plugins import util as plugins_util logger = logging.getLogger(__name__) @@ -27,6 +29,8 @@ class Installer(plugins_common.Plugin): add("config-dir", help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " "default configuration paths") + add("config-utility", default="postconf", + help="Path to the 'postconf' executable.") def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) @@ -41,6 +45,94 @@ class Installer(plugins_common.Plugin): self.raw_cf = [] self.cf = [] + def prepare(self): + """Prepare the installer. + + Finish up any additional initialization. + + :raises .PluginError: + when full initialization cannot be completed. + :raises .MisconfigurationError: + when full initialization cannot be completed. Plugin will + be displayed on a list of available plugins. + :raises .NoInstallationError: + when the necessary programs/files cannot be located. Plugin + will NOT be displayed on a list of available plugins. + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + + """ + self._verify_postconf_available() + + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + # XXX ensure we raise the right kinds of exceptions + + if self.get_version() < (2, 11, 0): + raise errors.NotSupportedError('Postfix version is too old') + + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html + + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html + + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added + + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added + + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html + + def _verify_postconf_available(self): + """Ensure 'postconf' can be found. + + :raises .NoInstallationError: when unable to find 'postconf' + + """ + if not util.exe_exists(self.conf("config-utility")): + if not plugins_util.path_surgery(self.conf("config-utility")): + raise errors.NoInstallationError( + "Cannot find executable '{0}'. You can provide the " + "path to this command with --{1}".format( + self.conf("config-utility"), + self.option_name("config-utility"))) + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -135,77 +227,6 @@ class Installer(plugins_common.Plugin): ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def prepare(self): - """Prepare the plugin. - - Finish up any additional initialization. - - :raises .PluginError: - when full initialization cannot be completed. - :raises .MisconfigurationError: - when full initialization cannot be completed. Plugin will - be displayed on a list of available plugins. - :raises .NoInstallationError: - when the necessary programs/files cannot be located. Plugin - will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: - - """ - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - # XXX ensure we raise the right kinds of exceptions - - if self.get_version() < (2, 11, 0): - raise errors.NotSupportedError('Postfix version is too old') - - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html - - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added - - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added - - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly - - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html def get_version(self): """Return the mail version of Postfix. @@ -219,7 +240,7 @@ class Installer(plugins_common.Plugin): """ # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + cmd = subprocess.Popen([self.conf('config-utility'), '-d', 'mail_version'], stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9be3744e3..332c6ec60 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -62,11 +62,13 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() + exe_exists_path = "certbot_postfix.installer.util.exe_exists" popen_path = "certbot_postfix.installer.subprocess.Popen" - with mock.patch(popen_path) as mock_popen: - mock_popen().returncode = 0 - mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") - installer.prepare() + with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: + with mock.patch(popen_path) as mock_popen: + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + installer.prepare() return installer From 192f0f60daadac4a44541b10aa90181b4156a248 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 12:19:59 -0700 Subject: [PATCH 154/631] test add_parser_arguments --- certbot-postfix/certbot_postfix/installer_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 332c6ec60..b3e8f560d 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,6 +28,15 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.postfix_dir = 'tests/' + def test_add_parser_arguments(self): + mock_add = mock.MagicMock() + + from certbot_postfix import installer + installer.Installer.add_parser_arguments(mock_add) + + for call in mock_add.call_args_list: + self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: From dfd1cceb9b5e18d646ef55d0b12b561712510379 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 12:26:47 -0700 Subject: [PATCH 155/631] Test prepare() failure due to missing postconf --- certbot-postfix/certbot_postfix/installer_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index b3e8f560d..721790797 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -12,6 +12,8 @@ import unittest import mock import six +from certbot import errors + # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org @@ -37,6 +39,17 @@ class TestPostfixConfigGenerator(unittest.TestCase): for call in mock_add.call_args_list: self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + def test_no_postconf_prepare(self): + installer = self._create_installer() + + installer_path = "certbot_postfix.installer" + exe_exists_path = installer_path + ".util.exe_exists" + path_surgery_path = installer_path + ".plugins_util.path_surgery" + + with mock.patch(path_surgery_path, return_value=False): + with mock.patch(exe_exists_path, return_value=False): + self.assertRaises(errors.NoInstallationError, installer.prepare) + def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: From 5beaae3b6502fa4e39ddc80a564f8308347cb67f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:17:13 -0700 Subject: [PATCH 156/631] Add check_output function and tests. --- certbot-postfix/certbot_postfix/util.py | 49 +++++++++++++++++++ certbot-postfix/certbot_postfix/util_test.py | 50 ++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 certbot-postfix/certbot_postfix/util.py create mode 100644 certbot-postfix/certbot_postfix/util_test.py diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py new file mode 100644 index 000000000..8ad6408c4 --- /dev/null +++ b/certbot-postfix/certbot_postfix/util.py @@ -0,0 +1,49 @@ +"""Utility functions for use in the Postfix installer.""" + +import logging +import subprocess + + +logger = logging.getLogger(__name__) + + +def check_output(*args, **kwargs): + """Backported version of subprocess.check_output for Python 2.6+. + + This is the same as subprocess.check_output from newer versions of + Python, except: + + 1. The return value is a string rather than a byte string. To + accomplish this, the caller cannot set the parameter + universal_newlines. + 2. If the command exits with a nonzero status, output is not + included in the raised subprocess.CalledProcessError because + subprocess.CalledProcessError on Python 2.6 does not support this. + Instead, the failure including the output is logged. + + :param tuple args: positional arguments for Popen + :param dict kwargs: keyword arguments for Popen + + :returns: data printed to stdout + :rtype: str + + """ + for keyword in ('stdout', 'universal_newlines',): + if keyword in kwargs: + raise ValueError( + keyword + ' argument not allowed, it will be overridden.') + + kwargs['stdout'] = subprocess.PIPE + kwargs['universal_newlines'] = True + + process = subprocess.Popen(*args, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get('args') + if cmd is None: + cmd = args[0] + logger.debug( + "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) + raise subprocess.CalledProcessError(retcode, cmd) + return output diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py new file mode 100644 index 000000000..019f34532 --- /dev/null +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -0,0 +1,50 @@ +"""Tests for certbot_postfix.util.""" + +import subprocess +import unittest + +import mock + +class CheckOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_output + return check_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + + mock_popen().communicate.return_value = (output, '') + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + + log_args = mock_logger.debug.call_args[0] + self.assertTrue(command in log_args) + self.assertTrue(retcode in log_args) + self.assertTrue(output in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + output = 'bar' + mock_popen().communicate.return_value = (output, '') + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), output) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() From 4715b2b12ceede6b8b40e5a336a546435a7315dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:18:27 -0700 Subject: [PATCH 157/631] Further document check_output --- certbot-postfix/certbot_postfix/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index 8ad6408c4..d7145b7c8 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -27,6 +27,9 @@ def check_output(*args, **kwargs): :returns: data printed to stdout :rtype: str + :raises ValueError: if arguments are invalid + :raises subprocess.CalledProcessError: if the command fails + """ for keyword in ('stdout', 'universal_newlines',): if keyword in kwargs: From 5a1d031f07cc2a64410258920c48f14562a1addd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:19:44 -0700 Subject: [PATCH 158/631] Rename util to certbot_util --- certbot-postfix/certbot_postfix/installer.py | 4 ++-- certbot-postfix/certbot_postfix/installer_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 394ed5525..3de96d9a5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -9,7 +9,7 @@ import zope.interface from certbot import errors from certbot import interfaces -from certbot import util +from certbot import util as certbot_util from certbot.plugins import common as plugins_common from certbot.plugins import util as plugins_util @@ -125,7 +125,7 @@ class Installer(plugins_common.Plugin): :raises .NoInstallationError: when unable to find 'postconf' """ - if not util.exe_exists(self.conf("config-utility")): + if not certbot_util.exe_exists(self.conf("config-utility")): if not plugins_util.path_surgery(self.conf("config-utility")): raise errors.NoInstallationError( "Cannot find executable '{0}'. You can provide the " diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 721790797..9c8aa4f77 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -43,7 +43,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): installer = self._create_installer() installer_path = "certbot_postfix.installer" - exe_exists_path = installer_path + ".util.exe_exists" + exe_exists_path = installer_path + ".certbot_util.exe_exists" path_surgery_path = installer_path + ".plugins_util.path_surgery" with mock.patch(path_surgery_path, return_value=False): @@ -84,7 +84,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() - exe_exists_path = "certbot_postfix.installer.util.exe_exists" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" popen_path = "certbot_postfix.installer.subprocess.Popen" with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: with mock.patch(popen_path) as mock_popen: From 4e5740615c2d300b57785126ec84030aadb55445 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:25:08 -0700 Subject: [PATCH 159/631] Use util.check_output in Postfix installer --- certbot-postfix/certbot_postfix/installer.py | 13 +++++++++---- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3de96d9a5..44f77227e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -13,6 +13,8 @@ from certbot import util as certbot_util from certbot.plugins import common as plugins_common from certbot.plugins import util as plugins_util +from certbot_postfix import util + logger = logging.getLogger(__name__) @@ -240,10 +242,13 @@ class Installer(plugins_common.Plugin): """ # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen([self.conf('config-utility'), '-d', 'mail_version'], - stdout=subprocess.PIPE) - stdout, _ = cmd.communicate() - if cmd.returncode != 0: + try: + stdout = util.check_output( + [self.conf('config-utility'), '-d', 'mail_version']) + except subprocess.CalledProcessError: + logger.debug( + 'Encountered an error when trying to determine Postfix version.', + exc_info=True) raise errors.PluginError('Unable to determine Postfix version.') # grabs version component of string like "mail_version = 2.11.3" diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9c8aa4f77..a0aa060e8 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,12 +84,11 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() + check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - popen_path = "certbot_postfix.installer.subprocess.Popen" - with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: - with mock.patch(popen_path) as mock_popen: - mock_popen().returncode = 0 - mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + with mock.patch(check_output_path) as mock_check_output: + with mock.patch(exe_exists_path, return_value=True): + mock_check_output.return_value = "mail_version = 3.1.4" installer.prepare() return installer From 7334fc3066dc51ada34c0c8e88abf6f3552c711d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:29:56 -0700 Subject: [PATCH 160/631] Rename postfix_dir to config_dir --- certbot-postfix/certbot_postfix/installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 44f77227e..54928279b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -37,7 +37,7 @@ class Installer(plugins_common.Plugin): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) self.fixup = False - self.postfix_dir = self.conf("config-dir") + self.config_dir = self.conf("config-dir") self.additions = [] self.deletions = [] @@ -137,7 +137,7 @@ class Installer(plugins_common.Plugin): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") + return os.path.join(self.config_dir, "main.cf") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -268,7 +268,7 @@ class Installer(plugins_common.Plugin): "Server root: {root}{0}" "Version: {version}".format( os.linesep, - root=self.postfix_dir, + root=self.config_dir, version='.'.join([str(i) for i in self.get_version()])) ) From b72dfc0c08e43149f9e9d11e1a6e3ff111dfb1a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:07:32 -0700 Subject: [PATCH 161/631] Add get_config_var --- certbot-postfix/certbot_postfix/installer.py | 69 ++++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 54928279b..51455a340 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -241,20 +241,8 @@ class Installer(plugins_common.Plugin): :raises .PluginError: Unable to find Postfix version. """ - # Parse Postfix version number (feature support, syntax changes etc.) - try: - stdout = util.check_output( - [self.conf('config-utility'), '-d', 'mail_version']) - except subprocess.CalledProcessError: - logger.debug( - 'Encountered an error when trying to determine Postfix version.', - exc_info=True) - raise errors.PluginError('Unable to determine Postfix version.') - - # grabs version component of string like "mail_version = 2.11.3" - mail_version = stdout.split()[2] - postfix_version = tuple([int(i) for i in mail_version.split('.')]) - return postfix_version + mail_version = self.get_config_var("mail_version", default=True) + return tuple(int(i) for i in mail_version.split('.')) def more_info(self): """Human-readable string to help the user. @@ -406,6 +394,59 @@ class Installer(plugins_common.Plugin): if rc != 0: raise errors.MisconfigurationError('cannot restart postfix') + def get_config_var(self, name, default=False): + """Return the value of the specified Postfix config parameter. + + :param str name: name of the Postfix config parameter to return + :param bool default: whether or not to return the default value + instead of the actual value + + :returns: value of the specified configuration parameter + :rtype: str + + """ + cmd = self._build_cmd_for_config_var(name, default) + + try: + output = util.check_output(cmd) + except subprocess.CalledProcessError: + logger.debug("Encountered an error when running 'postconf'", + exc_info=True) + raise errors.PluginError( + "Unable to determine the value " + "of Postfix parameter {0}".format(name)) + + expected_prefix = name + " =" + if not output.startswith(expected_prefix): + raise errors.PluginError( + "Unexpected output from '{0}'".format(''.join(cmd))) + + return output[len(expected_prefix):].strip() + + def _build_cmd_for_config_var(self, name, default): + """Return a command to run to get a Postfix config parameter. + + :param str name: name of the Postfix config parameter to return + :param bool default: whether or not to return the default value + instead of the actual value + + :returns: command to run + :rtype: list + + """ + cmd = [self.conf("config-utility")] + + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + + if default: + cmd.append("-d") + + cmd.append(name) + + return cmd + + # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) # From 02c7eca6daa6727683161105ab2758b7f3adff07 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:08:41 -0700 Subject: [PATCH 162/631] Rename test classes and methods. --- certbot-postfix/certbot_postfix/installer_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a0aa060e8..6c7a3b628 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -25,7 +25,7 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -class TestPostfixConfigGenerator(unittest.TestCase): +class InstallerTest(unittest.TestCase): def setUp(self): self.postfix_dir = 'tests/' @@ -50,14 +50,14 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch(exe_exists_path, return_value=False): self.assertRaises(errors.NoInstallationError, installer.prepare) - def testGetAllNames(self): + def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = self._create_prepared_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) - def testGetAllCertAndKeys(self): + def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] @@ -66,7 +66,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): postfix_config_gen = self._create_prepared_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) - def testGetAllCertsAndKeys_With_None(self): + def test_get_all_certs_and_keys_with_none(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = self._create_prepared_installer() From 4c4b63437f9c2652be9d0633c61ac19c186a7b21 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:28:17 -0700 Subject: [PATCH 163/631] Test building of get_config_var command --- .../certbot_postfix/installer_test.py | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 6c7a3b628..3eeb5288c 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,7 +28,7 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class InstallerTest(unittest.TestCase): def setUp(self): - self.postfix_dir = 'tests/' + self.config = mock.MagicMock(postfix_config_dir="tests/") def test_add_parser_arguments(self): mock_add = mock.MagicMock() @@ -72,6 +72,36 @@ class InstallerTest(unittest.TestCase): postfix_config_gen = self._create_prepared_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def test_get_config_var_success(self): + self.config.postfix_config_dir = None + + command = self._test_get_config_var_success_common('foo', False) + self.assertFalse("-c" in command) + self.assertFalse("-d" in command) + + def test_get_config_var_success_with_config(self): + command = self._test_get_config_var_success_common('foo', False) + self.assertTrue("-c" in command) + self.assertFalse("-d" in command) + + def test_get_config_var_success_with_default(self): + self.config.postfix_config_dir = None + + command = self._test_get_config_var_success_common('foo', True) + self.assertFalse("-c" in command) + self.assertTrue("-d" in command) + + def _test_get_config_var_success_common(self, name, default): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + with mock.patch(check_output_path) as mock_check_output: + value = "bar" + mock_check_output.return_value = name + " = " + value + self.assertEqual(installer.get_config_var(name, default), value) + + return mock_check_output.call_args[0][0] + def _create_prepared_installer(self): """Creates and returns a new prepared Postfix Installer. @@ -100,11 +130,10 @@ class InstallerTest(unittest.TestCase): :rtype: certbot_postfix.installer.Installer """ - config = mock.MagicMock(postfix_config_dir=self.postfix_dir) name = "postfix" from certbot_postfix import installer - return installer.Installer(config, name) + return installer.Installer(self.config, name) if __name__ == '__main__': From 25d1f6ec7522ccb76b20ef89db2527946067f335 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:11:09 -0700 Subject: [PATCH 164/631] Test all branches of test_get_config_var --- .../certbot_postfix/installer_test.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 3eeb5288c..9fd192d66 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -7,6 +7,7 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import subprocess import unittest import mock @@ -28,7 +29,8 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class InstallerTest(unittest.TestCase): def setUp(self): - self.config = mock.MagicMock(postfix_config_dir="tests/") + self.config = mock.MagicMock(postfix_config_dir="tests/", + postfix_config_utility="postconf") def test_add_parser_arguments(self): mock_add = mock.MagicMock() @@ -91,6 +93,22 @@ class InstallerTest(unittest.TestCase): self.assertFalse("-c" in command) self.assertTrue("-d" in command) + @mock.patch("certbot_postfix.installer.logger") + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_config_var_failure(self, mock_check_output, mock_logger): + mock_check_output.side_effect = subprocess.CalledProcessError(42, "foo") + installer = self._create_installer() + self.assertRaises(errors.PluginError, installer.get_config_var, "foo") + self.assertTrue(mock_logger.debug.call_args[1]["exc_info"]) + + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_config_var_unexpected_output(self, mock_check_output): + self.config.postfix_config_dir = None + mock_check_output.return_value = "foo" + + installer = self._create_installer() + self.assertRaises(errors.PluginError, installer.get_config_var, "foo") + def _test_get_config_var_success_common(self, name, default): installer = self._create_installer() From 2e8a8dfed528be9e485775215a5bf976757d3576 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:17:46 -0700 Subject: [PATCH 165/631] add _set_config_dir --- certbot-postfix/certbot_postfix/installer.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 51455a340..9756da684 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -37,7 +37,7 @@ class Installer(plugins_common.Plugin): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) self.fixup = False - self.config_dir = self.conf("config-dir") + self.config_dir = None self.additions = [] self.deletions = [] @@ -67,6 +67,7 @@ class Installer(plugins_common.Plugin): """ self._verify_postconf_available() + self._set_config_dir() self.fn = self.find_postfix_cf() self.raw_cf = open(self.fn).readlines() @@ -135,6 +136,19 @@ class Installer(plugins_common.Plugin): self.conf("config-utility"), self.option_name("config-utility"))) + def _set_config_dir(self): + """Ensure self.config_dir is set to the correct path. + + If the configuration directory to use was set by the user, we'll + use that value, otherwise, we'll find the default path using + 'postconf'. + + """ + if self.conf("config-dir") is None: + self.config_dir = self.get_config_var("config_directory") + else: + self.config_dir = self.conf("config-dir") + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.config_dir, "main.cf") From 749f758adb699cb540d4d0fc1e781c0e87a79a6e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:25:30 -0700 Subject: [PATCH 166/631] use a temporary directory --- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9fd192d66..08c9d478f 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -7,6 +7,7 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import os import subprocess import unittest @@ -14,6 +15,7 @@ import mock import six from certbot import errors +from certbot.tests import util as certbot_test_util # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org @@ -26,10 +28,11 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -class InstallerTest(unittest.TestCase): +class InstallerTest(certbot_test_util.TempDirTestCase): def setUp(self): - self.config = mock.MagicMock(postfix_config_dir="tests/", + super(InstallerTest, self).setUp() + self.config = mock.MagicMock(postfix_config_dir=self.tempdir, postfix_config_utility="postconf") def test_add_parser_arguments(self): @@ -62,7 +65,7 @@ class InstallerTest(unittest.TestCase): def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - 'tests/main.cf'),] + os.path.join(self.tempdir, 'main.cf')),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) postfix_config_gen = self._create_prepared_installer() From 48c5731a6b18032b244ba1ed2dd9f26b17efecf4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:29:08 -0700 Subject: [PATCH 167/631] Write out temp config instead of mocking. --- .../certbot_postfix/installer_test.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 08c9d478f..9e83699ab 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -57,25 +57,26 @@ class InstallerTest(certbot_test_util.TempDirTestCase): def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + self._write_config(names_only_config) + installer = self._create_prepared_installer() + self.assertEqual(sorted_names, installer.get_all_names()) def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', os.path.join(self.tempdir, 'main.cf')),] - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + self._write_config(certs_only_config) + installer = self._create_prepared_installer() + self.assertEqual(return_vals, installer.get_all_certs_keys()) def test_get_all_certs_and_keys_with_none(self): - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + self._write_config(names_only_config) + installer = self._create_prepared_installer() + self.assertEqual([], installer.get_all_certs_keys()) + + def _write_config(self, content): + with open(os.path.join(self.tempdir, "main.cf"), "w") as f: + f.write(content) def test_get_config_var_success(self): self.config.postfix_config_dir = None From 290f5b8ce7ae544fc42dca1d383906fcc5dad7b1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:35:21 -0700 Subject: [PATCH 168/631] add test_set_config_dir --- certbot-postfix/certbot_postfix/installer.py | 2 +- .../certbot_postfix/installer_test.py | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 9756da684..546cf49cc 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -433,7 +433,7 @@ class Installer(plugins_common.Plugin): expected_prefix = name + " =" if not output.startswith(expected_prefix): raise errors.PluginError( - "Unexpected output from '{0}'".format(''.join(cmd))) + "Unexpected output from '{0}'".format(' '.join(cmd))) return output[len(expected_prefix):].strip() diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9e83699ab..aae61bd59 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -55,6 +55,25 @@ class InstallerTest(certbot_test_util.TempDirTestCase): with mock.patch(exe_exists_path, return_value=False): self.assertRaises(errors.NoInstallationError, installer.prepare) + def test_set_config_dir(self): + self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") + os.mkdir(self.config.postfix_config_dir) + self._write_config(names_only_config) + installer = self._create_installer() + + expected = self.config.postfix_config_dir + self.config.postfix_config_dir = None + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = [ + "config_directory = " + expected, "mail_version = 3.1.4" + ] + with mock.patch(exe_exists_path, return_value=True): + installer.prepare() + self.assertEqual(installer.config_dir, expected) + def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] self._write_config(names_only_config) @@ -75,7 +94,8 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertEqual([], installer.get_all_certs_keys()) def _write_config(self, content): - with open(os.path.join(self.tempdir, "main.cf"), "w") as f: + config_dir = self.config.postfix_config_dir + with open(os.path.join(config_dir, "main.cf"), "w") as f: f.write(content) def test_get_config_var_success(self): From 8c4ff5cb63dab6632f643da49f87cc86dcc3d8c9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:36:08 -0700 Subject: [PATCH 169/631] Use context manager to read conf file. --- certbot-postfix/certbot_postfix/installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 546cf49cc..bba46a168 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -70,7 +70,8 @@ class Installer(plugins_common.Plugin): self._set_config_dir() self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() + with open(self.fn) as f: + self.raw_cf = f.readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] # XXX ensure we raise the right kinds of exceptions From 50a1f6340ff2c33d80fe2bf1f2b7b95b3cdebe69 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:38:27 -0700 Subject: [PATCH 170/631] Add _check_version. --- certbot-postfix/certbot_postfix/installer.py | 61 +++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index bba46a168..2c3f4397e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -74,11 +74,42 @@ class Installer(plugins_common.Plugin): self.raw_cf = f.readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] - # XXX ensure we raise the right kinds of exceptions + def _verify_postconf_available(self): + """Ensure 'postconf' can be found. + + :raises .NoInstallationError: when unable to find 'postconf' + + """ + if not certbot_util.exe_exists(self.conf("config-utility")): + if not plugins_util.path_surgery(self.conf("config-utility")): + raise errors.NoInstallationError( + "Cannot find executable '{0}'. You can provide the " + "path to this command with --{1}".format( + self.conf("config-utility"), + self.option_name("config-utility"))) + + def _set_config_dir(self): + """Ensure self.config_dir is set to the correct path. + + If the configuration directory to use was set by the user, we'll + use that value, otherwise, we'll find the default path using + 'postconf'. + + """ + if self.conf("config-dir") is None: + self.config_dir = self.get_config_var("config_directory") + else: + self.config_dir = self.conf("config-dir") + + def _check_version(self): + """Verifies that the installed Postfix version is supported. + + :raises errors.NotSupportedError: if the version is unsupported + + """ if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend # differences here and pass them onto other configuration functions. @@ -123,32 +154,6 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html - def _verify_postconf_available(self): - """Ensure 'postconf' can be found. - - :raises .NoInstallationError: when unable to find 'postconf' - - """ - if not certbot_util.exe_exists(self.conf("config-utility")): - if not plugins_util.path_surgery(self.conf("config-utility")): - raise errors.NoInstallationError( - "Cannot find executable '{0}'. You can provide the " - "path to this command with --{1}".format( - self.conf("config-utility"), - self.option_name("config-utility"))) - - def _set_config_dir(self): - """Ensure self.config_dir is set to the correct path. - - If the configuration directory to use was set by the user, we'll - use that value, otherwise, we'll find the default path using - 'postconf'. - - """ - if self.conf("config-dir") is None: - self.config_dir = self.get_config_var("config_directory") - else: - self.config_dir = self.conf("config-dir") def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" From b98f541b9107fe6ccc34963b22d4b93f8d1ec67c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:41:59 -0700 Subject: [PATCH 171/631] clean up prepare() --- certbot-postfix/certbot_postfix/installer.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 2c3f4397e..bac33fd59 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -52,22 +52,14 @@ class Installer(plugins_common.Plugin): Finish up any additional initialization. - :raises .PluginError: - when full initialization cannot be completed. - :raises .MisconfigurationError: - when full initialization cannot be completed. Plugin will - be displayed on a list of available plugins. - :raises .NoInstallationError: - when the necessary programs/files cannot be located. Plugin - will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: + :raises errors.PluginError: when an unexpected error occurs + :raises errors.NoInstallationError: when can't find installation + :raises errors.NotSupportedError: when version is not supported """ self._verify_postconf_available() self._set_config_dir() + self._check_version() self.fn = self.find_postfix_cf() with open(self.fn) as f: From c9813a44d7b8a86964012ff09e62211833ace1bd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:42:48 -0700 Subject: [PATCH 172/631] protect get_version() --- certbot-postfix/certbot_postfix/installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index bac33fd59..c0200ff3d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -100,7 +100,7 @@ class Installer(plugins_common.Plugin): :raises errors.NotSupportedError: if the version is unsupported """ - if self.get_version() < (2, 11, 0): + if self._get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -242,7 +242,7 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def get_version(self): + def _get_version(self): """Return the mail version of Postfix. Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) @@ -269,7 +269,7 @@ class Installer(plugins_common.Plugin): "Version: {version}".format( os.linesep, root=self.config_dir, - version='.'.join([str(i) for i in self.get_version()])) + version='.'.join([str(i) for i in self._get_version()])) ) From d1f3a2deefacc5b9bd678c3c241ac23df3e13765 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:13:06 -0700 Subject: [PATCH 173/631] move _get_version --- certbot-postfix/certbot_postfix/installer.py | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index c0200ff3d..292be3f8d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -146,6 +146,19 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def _get_version(self): + """Return the mail version of Postfix. + + Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + + :returns: version + :rtype: tuple + + :raises .PluginError: Unable to find Postfix version. + + """ + mail_version = self.get_config_var("mail_version", default=True) + return tuple(int(i) for i in mail_version.split('.')) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -242,20 +255,6 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def _get_version(self): - """Return the mail version of Postfix. - - Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) - - :returns: version - :rtype: tuple - - :raises .PluginError: Unable to find Postfix version. - - """ - mail_version = self.get_config_var("mail_version", default=True) - return tuple(int(i) for i in mail_version.split('.')) - def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user From 83e37acc8b794fad3ce495fcb209be65c8becf3e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:16:07 -0700 Subject: [PATCH 174/631] group IPlugin methods --- certbot-postfix/certbot_postfix/installer.py | 41 ++++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 292be3f8d..d8e8420d9 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -146,6 +146,26 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.config_dir, "main.cf") + + def more_info(self): + """Human-readable string to help the user. + Should describe the steps taken and any relevant info to help the user + decide which plugin to use. + :rtype str: + """ + return ( + "Configures Postfix to try to authenticate mail servers, use " + "installed certificates and disable weak ciphers and protocols.{0}" + "Server root: {root}{0}" + "Version: {version}".format( + os.linesep, + root=self.config_dir, + version='.'.join([str(i) for i in self._get_version()])) + ) + def _get_version(self): """Return the mail version of Postfix. @@ -160,10 +180,6 @@ class Installer(plugins_common.Plugin): mail_version = self.get_config_var("mail_version", default=True) return tuple(int(i) for i in mail_version.split('.')) - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.config_dir, "main.cf") - def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -255,23 +271,6 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def more_info(self): - """Human-readable string to help the user. - Should describe the steps taken and any relevant info to help the user - decide which plugin to use. - :rtype str: - """ - return ( - "Configures Postfix to try to authenticate mail servers, use " - "installed certificates and disable weak ciphers and protocols.{0}" - "Server root: {root}{0}" - "Version: {version}".format( - os.linesep, - root=self.config_dir, - version='.'.join([str(i) for i in self._get_version()])) - ) - - ### Let's Encrypt client IInstaller ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 From b342f40c2ba68b73ed7428b4190705671c5e7caf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:16:36 -0700 Subject: [PATCH 175/631] remove old comments --- certbot-postfix/certbot_postfix/installer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d8e8420d9..92ba3f13d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -267,13 +267,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - ### Let's Encrypt client IPlugin ### - # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - - - ### Let's Encrypt client IInstaller ### - # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 - def get_all_names(self): """Returns all names that may be authenticated. :rtype: `list` of `str` From 90ffe2aac0d310947b7931792acfcfc96c036601 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 09:04:57 -0700 Subject: [PATCH 176/631] Remove legacy get_all_certs_and_keys() method --- certbot-postfix/certbot_postfix/installer.py | 24 ------------------- .../certbot_postfix/installer_test.py | 13 ---------- 2 files changed, 37 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 92ba3f13d..3eb670161 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -317,30 +317,6 @@ class Installer(plugins_common.Plugin): :rtype: :class:`list` of :class:`str` """ - def get_all_certs_keys(self): - """Retrieve all certs and keys set in configuration. - :returns: tuples with form `[(cert, key, path)]`, where: - - `cert` - str path to certificate file - - `key` - str path to associated key file - - `path` - file path to configuration file - :rtype: list - """ - cert_materials = {'smtpd_tls_key_file': None, - 'smtpd_tls_cert_file': None, - } - for num, line in enumerate(self.cf): - num, found_var, found_value = parse_line((num, line)) - if found_var in cert_materials.keys(): - cert_materials[found_var] = found_value - - if not all(cert_materials.values()): - cert_material_tuples = [] - else: - cert_material_tuples = [(cert_materials['smtpd_tls_cert_file'], - cert_materials['smtpd_tls_key_file'], - self.fn),] - return cert_material_tuples - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index aae61bd59..0c83a25e6 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -80,19 +80,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer = self._create_prepared_installer() self.assertEqual(sorted_names, installer.get_all_names()) - def test_get_all_certs_and_keys(self): - return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', - '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - os.path.join(self.tempdir, 'main.cf')),] - self._write_config(certs_only_config) - installer = self._create_prepared_installer() - self.assertEqual(return_vals, installer.get_all_certs_keys()) - - def test_get_all_certs_and_keys_with_none(self): - self._write_config(names_only_config) - installer = self._create_prepared_installer() - self.assertEqual([], installer.get_all_certs_keys()) - def _write_config(self, content): config_dir = self.config.postfix_config_dir with open(os.path.join(config_dir, "main.cf"), "w") as f: From 967a1830e6fdab696d9d0edf9d8416622811b341 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 14:59:25 -0700 Subject: [PATCH 177/631] Rewrite get_all_names --- certbot-postfix/certbot_postfix/installer.py | 26 ++++++++----------- .../certbot_postfix/installer_test.py | 14 ++++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3eb670161..3bbb112ba 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -180,6 +180,15 @@ class Installer(plugins_common.Plugin): mail_version = self.get_config_var("mail_version", default=True) return tuple(int(i) for i in mail_version.split('.')) + def get_all_names(self): + """Returns all names that may be authenticated. + + :rtype: `set` of `str` + + """ + return set(self.get_config_var(var) + for var in ('mydomain', 'myhostname', 'myorigin',)) + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -267,20 +276,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - def get_all_names(self): - """Returns all names that may be authenticated. - :rtype: `list` of `str` - """ - var_names = ('myhostname', 'mydomain', 'myorigin') - names_found = set() - for num, line in enumerate(self.cf): - num, found_var, found_value = parse_line((num, line)) - if found_var in var_names: - names_found.add(found_value) - name_list = list(names_found) - name_list.sort() - return name_list - def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. :param str domain: domain to deploy certificate file @@ -398,7 +393,8 @@ class Installer(plugins_common.Plugin): expected_prefix = name + " =" if not output.startswith(expected_prefix): raise errors.PluginError( - "Unexpected output from '{0}'".format(' '.join(cmd))) + "Unexpected output '{0}' from '{1}'".format(output, + ' '.join(cmd))) return output[len(expected_prefix):].strip() diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 0c83a25e6..467eed752 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -18,8 +18,8 @@ from certbot import errors from certbot.tests import util as certbot_test_util # Fake Postfix Configs -names_only_config = """myhostname = mail.fubard.org -mydomain = fubard.org +names_only_config = """mydomain = fubard.org +myhostname = mail.fubard.org myorigin = fubard.org""" @@ -74,11 +74,15 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer.prepare() self.assertEqual(installer.config_dir, expected) - def test_get_all_names(self): - sorted_names = ['fubard.org', 'mail.fubard.org'] + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_all_names(self, mock_check_output): self._write_config(names_only_config) installer = self._create_prepared_installer() - self.assertEqual(sorted_names, installer.get_all_names()) + mock_check_output.side_effect = names_only_config.splitlines() + + result = installer.get_all_names() + self.assertTrue("fubard.org" in result) + self.assertTrue("mail.fubard.org" in result) def _write_config(self, content): config_dir = self.config.postfix_config_dir From 0efc02d6eeeabe6383ffa0c3a9c43842fca9aedb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:03:11 -0700 Subject: [PATCH 178/631] Lock the Postfix config dir --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3bbb112ba..362fe5e8c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -60,6 +60,7 @@ class Installer(plugins_common.Plugin): self._verify_postconf_available() self._set_config_dir() self._check_version() + self._lock_config_dir() self.fn = self.find_postfix_cf() with open(self.fn) as f: @@ -146,6 +147,19 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def _lock_config_dir(self): + """Stop two Postfix plugins from modifying the config at once. + + :raises .PluginError: if unable to acquire the lock + + """ + try: + certbot_util.lock_dir_until_exit(self.config_dir) + except (OSError, errors.LockError): + logger.debug("Encountered error:", exc_info=True) + raise errors.PluginError( + "Unable to lock %s", self.config_dir) + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.config_dir, "main.cf") From 3b2e9e49be4183f980f5d0c981438df2151209d8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:04:36 -0700 Subject: [PATCH 179/631] Remove unneeded instance variables --- certbot-postfix/certbot_postfix/installer.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 362fe5e8c..9a7c3802e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -39,14 +39,6 @@ class Installer(plugins_common.Plugin): self.fixup = False self.config_dir = None - self.additions = [] - self.deletions = [] - self.policy_lines = [] - self.new_cf = "" - self.fn = None - self.raw_cf = [] - self.cf = [] - def prepare(self): """Prepare the installer. @@ -62,12 +54,6 @@ class Installer(plugins_common.Plugin): self._check_version() self._lock_config_dir() - self.fn = self.find_postfix_cf() - with open(self.fn) as f: - self.raw_cf = f.readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - def _verify_postconf_available(self): """Ensure 'postconf' can be found. From b9177948d393b590d11342d2463681e8a2bdc01e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:05:24 -0700 Subject: [PATCH 180/631] Remove _write_config --- certbot-postfix/certbot_postfix/installer_test.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 467eed752..5122bcfe9 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -58,7 +58,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): def test_set_config_dir(self): self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") os.mkdir(self.config.postfix_config_dir) - self._write_config(names_only_config) installer = self._create_installer() expected = self.config.postfix_config_dir @@ -76,7 +75,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): @mock.patch("certbot_postfix.installer.util.check_output") def test_get_all_names(self, mock_check_output): - self._write_config(names_only_config) installer = self._create_prepared_installer() mock_check_output.side_effect = names_only_config.splitlines() @@ -84,11 +82,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) - def _write_config(self, content): - config_dir = self.config.postfix_config_dir - with open(os.path.join(config_dir, "main.cf"), "w") as f: - f.write(content) - def test_get_config_var_success(self): self.config.postfix_config_dir = None From b94e268f83f383b13d3ea61e1a16e6b166ddeffa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:05:39 -0700 Subject: [PATCH 181/631] Remove unused certs_only_config --- certbot-postfix/certbot_postfix/installer_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 5122bcfe9..429974e49 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -23,11 +23,6 @@ myhostname = mail.fubard.org myorigin = fubard.org""" -certs_only_config = ( -"""smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem -smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") - - class InstallerTest(certbot_test_util.TempDirTestCase): def setUp(self): From cc3896d5d45bc3c68cc7b888447ce49a62ec0e2e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:12:11 -0700 Subject: [PATCH 182/631] Add test_lock_error --- certbot-postfix/certbot_postfix/installer_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 429974e49..29dcf3200 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -6,6 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +import functools import logging import os import subprocess @@ -68,6 +69,12 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer.prepare() self.assertEqual(installer.config_dir, expected) + def test_lock_error(self): + assert_raises = functools.partial(self.assertRaises, + errors.PluginError, + self._create_prepared_installer) + certbot_test_util.lock_and_call(assert_raises, self.tempdir) + @mock.patch("certbot_postfix.installer.util.check_output") def test_get_all_names(self, mock_check_output): installer = self._create_prepared_installer() From baf0d3343aceba3568dc06898c654c0322bd428a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:13:02 -0700 Subject: [PATCH 183/631] Remove postfix version notes --- certbot-postfix/certbot_postfix/installer.py | 43 -------------------- 1 file changed, 43 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 9a7c3802e..0f3796e1f 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -89,49 +89,6 @@ class Installer(plugins_common.Plugin): """ if self._get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html - - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added - - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added - - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly - - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html def _lock_config_dir(self): """Stop two Postfix plugins from modifying the config at once. From b92df1b71cf6b21b5414176c74f048e0af9c793d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:15:41 -0700 Subject: [PATCH 184/631] Remove unused find_postfix_cf --- certbot-postfix/certbot_postfix/installer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 0f3796e1f..d9aeb4887 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -103,10 +103,6 @@ class Installer(plugins_common.Plugin): raise errors.PluginError( "Unable to lock %s", self.config_dir) - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.config_dir, "main.cf") - def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user From 00e28592b6163f314f68a7c81dc5faa1854ca2f6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:18:40 -0700 Subject: [PATCH 185/631] Add supported_enhancements --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++------ certbot-postfix/certbot_postfix/installer_test.py | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d9aeb4887..45768df6c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -142,6 +142,14 @@ class Installer(plugins_common.Plugin): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def supported_enhancements(self): + """Returns a list of supported enhancements. + + :rtype: list + + """ + return [] + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -258,12 +266,6 @@ class Installer(plugins_common.Plugin): an error occurs during the enhancement. """ - def supported_enhancements(self): - """Returns a list of supported enhancements. - :returns: supported enhancements which should be a subset of - :const:`~letsencrypt.constants.ENHANCEMENTS` - :rtype: :class:`list` of :class:`str` - """ def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 29dcf3200..ea7d0e503 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,6 +84,10 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) + def test_supported_enhancements(self): + self.assertEqual( + self._create_prepared_installer().supported_enhancements(), []) + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 60c6cc5f2abf12884506172233894e358e18e670 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:26:52 -0700 Subject: [PATCH 186/631] Write enhance() --- certbot-postfix/certbot_postfix/installer.py | 24 ++++++++----------- .../certbot_postfix/installer_test.py | 5 ++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 45768df6c..189836bbe 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -142,6 +142,16 @@ class Installer(plugins_common.Plugin): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def enhance(self, domain, enhancement, options=None): + """Raises an exception for request for unsupported enhancement. + + :raises .PluginError: this is always raised as no enhancements + are currently supported + + """ + raise errors.PluginError( + "Unsupported enhancement: {0}".format(enhancement)) + def supported_enhancements(self): """Returns a list of supported enhancements. @@ -253,20 +263,6 @@ class Installer(plugins_common.Plugin): self.set_domainwise_tls_policies() self.update_CAfile() - def enhance(self, domain, enhancement, options=None): - """Perform a configuration enhancement. - :param str domain: domain for which to provide enhancement - :param str enhancement: An enhancement as defined in - :const:`~letsencrypt.constants.ENHANCEMENTS` - :param options: Flexible options parameter for enhancement. - Check documentation of - :const:`~letsencrypt.constants.ENHANCEMENTS` - for expected options for each enhancement. - :raises .PluginError: If Enhancement is not supported, or if - an error occurs during the enhancement. - """ - - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index ea7d0e503..1796c60ca 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,6 +84,11 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) + def test_enhance(self): + self.assertRaises(errors.PluginError, + self._create_prepared_installer().enhance, + "example.org", "redirect") + def test_supported_enhancements(self): self.assertEqual( self._create_prepared_installer().supported_enhancements(), []) From 0f4c5c230538458f99418f28b4f3658477d59bc7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 11:41:30 -0700 Subject: [PATCH 187/631] Use common installer base --- certbot-postfix/certbot_postfix/installer.py | 20 +------------------ .../certbot_postfix/installer_test.py | 6 +++--- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 189836bbe..366824f07 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) -class Installer(plugins_common.Plugin): +class Installer(plugins_common.Installer): """Certbot installer plugin for Postfix.""" description = "Configure TLS with the Postfix MTA" @@ -278,24 +278,6 @@ class Installer(plugins_common.Plugin): """ self.maybe_add_config_lines() - def rollback_checkpoints(self, rollback=1): - """Revert `rollback` number of configuration checkpoints. - :raises .PluginError: when configuration cannot be fully reverted - """ - - def recovery_routine(self): - """Revert configuration to most recent finalized checkpoint. - Remove all changes (temporary and permanent) that have not been - finalized. This is useful to protect against crashes and other - execution interruptions. - :raises .errors.PluginError: If unable to recover the configuration - """ - - def view_config_changes(self): - """Display all of the LE config changes. - :raises .PluginError: when config changes cannot be parsed - """ - def config_test(self): """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 1796c60ca..547a954e2 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -24,12 +24,12 @@ myhostname = mail.fubard.org myorigin = fubard.org""" -class InstallerTest(certbot_test_util.TempDirTestCase): +class InstallerTest(certbot_test_util.ConfigTestCase): def setUp(self): super(InstallerTest, self).setUp() - self.config = mock.MagicMock(postfix_config_dir=self.tempdir, - postfix_config_utility="postconf") + self.config.postfix_config_dir = self.tempdir + self.config.postfix_config_utility = "postconf" def test_add_parser_arguments(self): mock_add = mock.MagicMock() From a29a99fb6ff359630b34629f2b1ec35cfc1455a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:34:03 -0700 Subject: [PATCH 188/631] Add --postfix-ctl flag. --- certbot-postfix/certbot_postfix/installer.py | 4 +++- certbot-postfix/certbot_postfix/installer_test.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 366824f07..3c4a4a85b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -28,9 +28,11 @@ class Installer(plugins_common.Installer): @classmethod def add_parser_arguments(cls, add): + add("ctl", default="postfix", + help="Path to the 'postfix' control program.") add("config-dir", help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " - "default configuration paths") + "default configuration paths.") add("config-utility", default="postconf", help="Path to the 'postconf' executable.") diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 547a954e2..a761a4386 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -32,13 +32,14 @@ class InstallerTest(certbot_test_util.ConfigTestCase): self.config.postfix_config_utility = "postconf" def test_add_parser_arguments(self): + options = set(('ctl', 'config-dir', 'config-utility',)) mock_add = mock.MagicMock() from certbot_postfix import installer installer.Installer.add_parser_arguments(mock_add) for call in mock_add.call_args_list: - self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + self.assertTrue(call[0][0] in options) def test_no_postconf_prepare(self): installer = self._create_installer() From b4b5c447500ef3f328d14efc99b6ca0a2ce54f28 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:38:07 -0700 Subject: [PATCH 189/631] Check postfix executable is found. --- certbot-postfix/certbot_postfix/installer.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3c4a4a85b..73e54b4f5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -51,24 +51,27 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: when version is not supported """ - self._verify_postconf_available() + for param in ("ctl", "config_dir",): + self._verify_executable_is_available(param) self._set_config_dir() self._check_version() self._lock_config_dir() - def _verify_postconf_available(self): - """Ensure 'postconf' can be found. + def _verify_executable_is_available(self, config_name): + """Asserts the program in the specified config param is found. - :raises .NoInstallationError: when unable to find 'postconf' + :param str config_name: name of the config param + + :raises .NoInstallationError: when the executable isn't found """ - if not certbot_util.exe_exists(self.conf("config-utility")): - if not plugins_util.path_surgery(self.conf("config-utility")): + if not certbot_util.exe_exists(self.conf(config_name)): + if not plugins_util.path_surgery(self.conf(config_name)): raise errors.NoInstallationError( "Cannot find executable '{0}'. You can provide the " "path to this command with --{1}".format( - self.conf("config-utility"), - self.option_name("config-utility"))) + self.conf(config_name), + self.option_name(config_name))) def _set_config_dir(self): """Ensure self.config_dir is set to the correct path. From 2ae187b1d6bffb6cc63aaa36ddea14be3b7675d3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:55:21 -0700 Subject: [PATCH 190/631] Update config_test method --- certbot-postfix/certbot_postfix/installer.py | 46 +++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 73e54b4f5..60c8b3109 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -165,6 +165,41 @@ class Installer(plugins_common.Installer): """ return [] + def config_test(self): + """Make sure the configuration is valid. + + :raises .MisconfigurationError: if the config is invalid + + """ + try: + self._run_postfix_subcommand("check") + except subprocess.CalledProcessError: + raise errors.MisconfigurationError( + "Postfix failed internal configuration check.") + + def _run_postfix_subcommand(self, subcommand): + """Runs a subcommand of the 'postfix' control program. + + If the command fails, the exception is logged at the DEBUG + level. + + :param str subcommand: subcommand to run + + :raises subprocess.CalledProcessError: if the command fails + + """ + cmd = [self.conf("ctl")] + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + cmd.append(subcommand) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + logger.debug("%s exited with a non-zero status.", + "".join(cmd), exc_info=True) + raise + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -283,17 +318,6 @@ class Installer(plugins_common.Installer): """ self.maybe_add_config_lines() - def config_test(self): - """Make sure the configuration is valid. - :raises .MisconfigurationError: when the config is not in a usable state - """ - if os.geteuid() != 0: - rc = os.system('sudo /usr/sbin/postfix check') - else: - rc = os.system('/usr/sbin/postfix check') - if rc != 0: - raise errors.MisconfigurationError('Postfix failed self-check.') - def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted From 5f3be9b1cf667bae9ff8c66c6ee3ae38c1873ae8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:58:45 -0700 Subject: [PATCH 191/631] test config_test failure --- certbot-postfix/certbot_postfix/installer_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a761a4386..a7a9d27fa 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,6 +28,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): def setUp(self): super(InstallerTest, self).setUp() + self.config.postfix_ctl = "postfix" self.config.postfix_config_dir = self.tempdir self.config.postfix_config_utility = "postconf" @@ -94,6 +95,12 @@ class InstallerTest(certbot_test_util.ConfigTestCase): self.assertEqual( self._create_prepared_installer().supported_enhancements(), []) + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_config_test_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") + self.assertRaises(errors.MisconfigurationError, installer.config_test) + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 68dc678eed9d997a446f54eca55266a9e8531d4f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:02:57 -0700 Subject: [PATCH 192/631] Call config_test in prepare --- certbot-postfix/certbot_postfix/installer.py | 1 + certbot-postfix/certbot_postfix/installer_test.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 60c8b3109..910fa67a6 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -55,6 +55,7 @@ class Installer(plugins_common.Installer): self._verify_executable_is_available(param) self._set_config_dir() self._check_version() + self.config_test() self._lock_config_dir() def _verify_executable_is_available(self, config_name): diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a7a9d27fa..425f0dd54 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -61,6 +61,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): expected = self.config.postfix_config_dir self.config.postfix_config_dir = None + check_call_path = "certbot_postfix.installer.subprocess.check_call" check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" with mock.patch(check_output_path) as mock_check_output: @@ -68,7 +69,8 @@ class InstallerTest(certbot_test_util.ConfigTestCase): "config_directory = " + expected, "mail_version = 3.1.4" ] with mock.patch(exe_exists_path, return_value=True): - installer.prepare() + with mock.patch(check_call_path): + installer.prepare() self.assertEqual(installer.config_dir, expected) def test_lock_error(self): @@ -159,12 +161,14 @@ class InstallerTest(certbot_test_util.ConfigTestCase): """ installer = self._create_installer() + check_call_path = "certbot_postfix.installer.subprocess.check_call" check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" with mock.patch(check_output_path) as mock_check_output: with mock.patch(exe_exists_path, return_value=True): - mock_check_output.return_value = "mail_version = 3.1.4" - installer.prepare() + with mock.patch(check_call_path): + mock_check_output.return_value = "mail_version = 3.1.4" + installer.prepare() return installer From a6c08a2e255801eaa9daedb050ee4d588c1b89ab Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:03:29 -0700 Subject: [PATCH 193/631] Update prepare docstring. --- certbot-postfix/certbot_postfix/installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 910fa67a6..94cfa3194 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -47,6 +47,7 @@ class Installer(plugins_common.Installer): Finish up any additional initialization. :raises errors.PluginError: when an unexpected error occurs + :raises errors.MisconfigurationError: when the config is invalid :raises errors.NoInstallationError: when can't find installation :raises errors.NotSupportedError: when version is not supported From 218e15c9d4fc5b74ae57b1a57e8c1ab28a57b620 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:25:22 -0700 Subject: [PATCH 194/631] Make restart() more robust. --- certbot-postfix/certbot_postfix/installer.py | 65 ++++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 94cfa3194..d73b73a37 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -179,6 +179,59 @@ class Installer(plugins_common.Installer): raise errors.MisconfigurationError( "Postfix failed internal configuration check.") + def restart(self): + """Restart or refresh the server content. + + :raises .PluginError: when server cannot be restarted + + """ + logger.info("Reloading Postfix configuration...") + if self._is_postfix_running(): + self._reload() + else: + self._start() + + def _is_postfix_running(self): + """Is Postfix currently running? + + Uses the 'postfix status' command to determine if Postfix is + currently running using the specified configuration files. + + :returns: True if Postfix is running, otherwise, False + :rtype: bool + + """ + try: + self._run_postfix_subcommand("status") + except subprocess.CalledProcessError: + return False + return True + + def _reload(self): + """Instructions Postfix to reload its configuration. + + If Postfix isn't currently running, this method will fail. + + :raises .PluginError: when Postfix cannot reload + + """ + try: + self._run_postfix_subcommand("reload") + except subprocess.CalledProcessError: + raise errors.PluginError( + "Postfix failed to reload its configuration.") + + def _start(self): + """Instructions Postfix to start running. + + :raises .PluginError: when Postfix cannot start + + """ + try: + self._run_postfix_subcommand("start") + except subprocess.CalledProcessError: + raise errors.PluginError("Postfix failed to start") + def _run_postfix_subcommand(self, subcommand): """Runs a subcommand of the 'postfix' control program. @@ -320,18 +373,6 @@ class Installer(plugins_common.Installer): """ self.maybe_add_config_lines() - def restart(self): - """Restart or refresh the server content. - :raises .PluginError: when server cannot be restarted - """ - logger.info('Reloading postfix config...') - if os.geteuid() != 0: - rc = os.system("sudo service postfix reload") - else: - rc = os.system("service postfix reload") - if rc != 0: - raise errors.MisconfigurationError('cannot restart postfix') - def get_config_var(self, name, default=False): """Return the value of the specified Postfix config parameter. From b21b66c0c07ac3b33db0dc199a946c667dee6106 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:29:50 -0700 Subject: [PATCH 195/631] Test restart command. --- .../certbot_postfix/installer_test.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 425f0dd54..69d999b31 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -103,6 +103,33 @@ class InstallerTest(certbot_test_util.ConfigTestCase): mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") self.assertRaises(errors.MisconfigurationError, installer.config_test) + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_reload_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = [ + None, subprocess.CalledProcessError(42, "foo") + ] + self.assertRaises(errors.PluginError, installer.restart) + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_reload_success(self, mock_check_call): + installer = self._create_prepared_installer() + installer.restart() + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_start_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") + self.assertRaises(errors.PluginError, installer.restart) + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_start_success(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = [ + subprocess.CalledProcessError(42, "foo"), None + ] + installer.restart() + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 11b820c0e4b8a60b9145cfea97d9a0ba89149bb7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:41:07 -0700 Subject: [PATCH 196/631] add set_config_var --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d73b73a37..e384e0396 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -40,6 +40,7 @@ class Installer(plugins_common.Installer): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.config_dir = None + self.proposed_changes = {} def prepare(self): """Prepare the installer. @@ -426,6 +427,19 @@ class Installer(plugins_common.Installer): return cmd + def set_config_var(self, name, value): + """Set the Postfix config parameter name to value. + + This method only stores the requested change in memory. The + Postfix configuration is not modified until save() is called. + + :param str name: name of the Postfix config parameter + :param str value: value to set the Postfix config parameter to + + """ + assert isinstance(name, str), "Invalid name value" + assert isinstance(value, str), "Invalid key value" + self.proposed_changes[name] = value # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) From 4805fb4b88945863f03a3aaa7638ce2684ca6fd8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:52:05 -0700 Subject: [PATCH 197/631] Add deploy_cert --- certbot-postfix/certbot_postfix/installer.py | 36 +++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index e384e0396..3b67cfa9c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -150,6 +150,26 @@ class Installer(plugins_common.Installer): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def deploy_cert(self, domain, cert_path, + key_path, chain_path, fullchain_path): + """Configure the Postfix SMTP server to use the given TLS cert. + + :param str domain: domain to deploy certificate file + :param str cert_path: absolute path to the certificate file + :param str key_path: absolute path to the private key file + :param str chain_path: absolute path to the certificate chain file + :param str fullchain_path: absolute path to the certificate fullchain + file (cert plus chain) + + :raises .PluginError: when cert cannot be deployed + + """ + self.set_config_var("smtpd_tls_cert_file", fullchain_path) + self.set_config_var("smtpd_tls_key_file", key_path) + self.set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") + self.set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") + self.set_config_var("smtpd_use_tls", "yes") + def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -343,22 +363,6 @@ class Installer(plugins_common.Installer): with open(self.fn, "w") as f: f.write(self.new_cf) - def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): - """Deploy certificate. - :param str domain: domain to deploy certificate file - :param str cert_path: absolute path to the certificate file - :param str key_path: absolute path to the private key file - :param str chain_path: absolute path to the certificate chain file - :param str fullchain_path: absolute path to the certificate fullchain - file (cert plus chain) - :raises .PluginError: when cert cannot be deployed - """ - self.wrangle_existing_config() - self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) - self.ensure_cf_var("smtpd_tls_key_file", key_path, []) - self.set_domainwise_tls_policies() - self.update_CAfile() - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be From d0ea5958f90505b6ac6c3b45b3bc06eda165e17e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:54:53 -0700 Subject: [PATCH 198/631] Protect _set_config_var --- certbot-postfix/certbot_postfix/installer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3b67cfa9c..7281a7e4d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -164,11 +164,11 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ - self.set_config_var("smtpd_tls_cert_file", fullchain_path) - self.set_config_var("smtpd_tls_key_file", key_path) - self.set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") - self.set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - self.set_config_var("smtpd_use_tls", "yes") + self._set_config_var("smtpd_tls_cert_file", fullchain_path) + self._set_config_var("smtpd_tls_key_file", key_path) + self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") + self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") + self._set_config_var("smtpd_use_tls", "yes") def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -431,7 +431,7 @@ class Installer(plugins_common.Installer): return cmd - def set_config_var(self, name, value): + def _set_config_var(self, name, value): """Set the Postfix config parameter name to value. This method only stores the requested change in memory. The From 72637b2cf6ae546c1d2b316e677b937206967cf8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 15:20:22 -0700 Subject: [PATCH 199/631] Add util.check_call --- certbot-postfix/certbot_postfix/util.py | 33 ++++++++++++++++++-- certbot-postfix/certbot_postfix/util_test.py | 23 ++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index d7145b7c8..b65e4231e 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -7,6 +7,24 @@ import subprocess logger = logging.getLogger(__name__) +def check_call(*args, **kwargs): + """A simple wrapper of subprocess.check_call that logs errors. + + :param tuple args: positional arguments to subprocess.check_call + :param dict kargs: keyword arguments to subprocess.check_call + + :raises subprocess.CalledProcessError: if the call fails + + """ + try: + subprocess.check_call(*args, **kwargs) + except subprocess.CalledProcessError: + cmd = _get_cmd(*args, **kwargs) + logger.debug("%s exited with a non-zero status.", + "".join(cmd), exc_info=True) + raise + + def check_output(*args, **kwargs): """Backported version of subprocess.check_output for Python 2.6+. @@ -43,10 +61,19 @@ def check_output(*args, **kwargs): output, unused_err = process.communicate() retcode = process.poll() if retcode: - cmd = kwargs.get('args') - if cmd is None: - cmd = args[0] + cmd = _get_cmd(*args, **kwargs) logger.debug( "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) raise subprocess.CalledProcessError(retcode, cmd) return output + + +def _get_cmd(*args, **kwargs): + """Return the command from Popen args. + + :param tuple args: Popen args + :param dict kwargs: Popen kwargs + + """ + cmd = kwargs.get('args') + return args[0] if cmd is None else cmd diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py index 019f34532..95253e1fd 100644 --- a/certbot-postfix/certbot_postfix/util_test.py +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -5,6 +5,29 @@ import unittest import mock +class CheckCallTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_call.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_call + return check_call(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.check_call') + def test_failure(self, mock_check_call, mock_logger): + cmd = "postconf smtpd_use_tls=yes".split() + mock_check_call.side_effect = subprocess.CalledProcessError(42, cmd) + self.assertRaises(subprocess.CalledProcessError, self._call, cmd) + self.assertTrue(mock_logger.method_calls) + + @mock.patch('certbot_postfix.util.subprocess.check_call') + def test_success(self, mock_check_call): + cmd = "postconf smtpd_use_tls=yes".split() + self._call(cmd) + mock_check_call.assert_called_once_with(cmd) + + class CheckOutputTest(unittest.TestCase): """Tests for certbot_postfix.util.check_output.""" From d663f7981a08ff47477047a8d37e09f3be6aaa63 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 15:24:51 -0700 Subject: [PATCH 200/631] Add _write_config_changes --- certbot-postfix/certbot_postfix/installer.py | 37 ++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 7281a7e4d..3f4ff0227 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -269,12 +269,7 @@ class Installer(plugins_common.Installer): cmd.extend(("-c", self.conf("config-dir"),)) cmd.append(subcommand) - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - logger.debug("%s exited with a non-zero status.", - "".join(cmd), exc_info=True) - raise + util.check_call(cmd) def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -419,10 +414,7 @@ class Installer(plugins_common.Installer): :rtype: list """ - cmd = [self.conf("config-utility")] - - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) + cmd = self._postconf_command_base() if default: cmd.append("-d") @@ -445,6 +437,31 @@ class Installer(plugins_common.Installer): assert isinstance(value, str), "Invalid key value" self.proposed_changes[name] = value + def _write_config_changes(self): + """Write proposed changes to the Postfix config. + + :raises errors.PluginError: if an error occurs + + """ + cmd = self._postconf_command_base() + cmd.extend("{0}={1}".format(name, value) + for name, value in self.proposed_changes.items()) + + try: + util.check_call(cmd) + except subprocess.CalledProcessError: + raise errors.PluginError( + "An error occurred while updating your Postfix config.") + + def _postconf_command_base(self): + """Builds start of a postconf command using the selected config.""" + cmd = [self.conf("config-utility")] + + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + + return cmd + # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) # From a339de80f43966e5551a19ba8b5c53a7be129a40 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 17:17:29 -0700 Subject: [PATCH 201/631] Add save() --- certbot-postfix/certbot_postfix/installer.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3f4ff0227..71515c687 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -41,6 +41,7 @@ class Installer(plugins_common.Installer): self.fixup = False self.config_dir = None self.proposed_changes = {} + self.save_notes = [] def prepare(self): """Prepare the installer. @@ -164,6 +165,7 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ + self.save_notes.append("Configuring TLS for {0}".format(domain)) self._set_config_var("smtpd_tls_cert_file", fullchain_path) self._set_config_var("smtpd_tls_key_file", key_path) self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") @@ -188,6 +190,32 @@ class Installer(plugins_common.Installer): """ return [] + def save(self, title=None, temporary=False): + """Creates backups and writes changes to configuration files. + + :param str title: The title of the save. If a title is given, the + configuration will be saved as a new checkpoint and put in a + timestamped directory. `title` has no effect if temporary is true. + + :param bool temporary: Indicates whether the changes made will + be quickly reversed in the future (challenges) + + :raises errors.PluginError: when save is unsuccessful + + """ + if not self.proposed_changes: + return + + self.add_to_checkpoint(os.path.join(self.config_dir, "main.cf"), + "\n".join(self.save_notes), temporary) + self._write_config_changes() + + self.proposed_changes.clear() + del self.save_notes[:] + + if title and not temporary: + self.finalize_checkpoint(title) + def config_test(self): """Make sure the configuration is valid. @@ -436,6 +464,7 @@ class Installer(plugins_common.Installer): assert isinstance(name, str), "Invalid name value" assert isinstance(value, str), "Invalid key value" self.proposed_changes[name] = value + self.save_notes.append("\t* Set {0} to {1}".format(name, value)) def _write_config_changes(self): """Write proposed changes to the Postfix config. From 142bc3354568853c4b32e8998699bc37c0d966ce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Aug 2017 10:38:53 -0700 Subject: [PATCH 202/631] Remove dead code --- certbot-postfix/certbot_postfix/installer.py | 152 +------------------ 1 file changed, 1 insertion(+), 151 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 71515c687..09e1cc18b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -38,8 +38,7 @@ class Installer(plugins_common.Installer): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) - self.fixup = False - self.config_dir = None + self.config_dir = None self.proposed_changes = {} self.save_notes = [] @@ -299,108 +298,6 @@ class Installer(plugins_common.Installer): util.check_call(cmd) - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - - :raises .errors.MisconfigurationError: if conflicting existing values - are found for var - - """ - acceptable = [ideal] + also_acceptable - - l = [(num,line) for num,line in enumerate(self.cf) - if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) - else: - raise errors.MisconfigurationError( - "Conflicting existing config values {0}".format(l) - ) - val = values[0][2] - if val not in acceptable: - if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise errors.MisconfigurationError( - "Existing config has %s=%s"%(var,val) - ) - - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - # policy_cf_entry = "texthash:" + self.policy_file - - # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - - # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed - # between Postfix version 2.5 and 2.6, since we only support => 2.11 - # we don't use nor support legacy Postfix syntax. - # - Server: - self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - # - Client: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - logger.info('Deleting lines: {}'.format(self.deletions)) - self.additions[:0]=["#", - "# New config lines added by STARTTLS Everywhere", - "#"] - new_cf_lines = "\n".join(self.additions) + "\n" - logger.info('Adding to {}:'.format(self.fn)) - logger.info(new_cf_lines) - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - with open(self.fn, "w") as f: - f.write(self.new_cf) - - def save(self, title=None, temporary=False): - """Saves all changes to the configuration files. - Both title and temporary are needed because a save may be - intended to be permanent, but the save is not ready to be a full - checkpoint. If an exception is raised, it is assumed a new - checkpoint was not created. - :param str title: The title of the save. If a title is given, the - configuration will be saved as a new checkpoint and put in a - timestamped directory. `title` has no effect if temporary is true. - :param bool temporary: Indicates whether the changes made will - be quickly reversed in the future (challenges) - :raises .PluginError: when save is unsuccessful - """ - self.maybe_add_config_lines() - def get_config_var(self, name, default=False): """Return the value of the specified Postfix config parameter. @@ -490,50 +387,3 @@ class Installer(plugins_common.Installer): cmd.extend(("-c", self.conf("config-dir"),)) return cmd - - # def update_CAfile(self): - # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - # - # def set_domainwise_tls_policies(self): - # all_acceptable_mxs = self.policy_config.acceptable_mxs - # for address_domain, properties in all_acceptable_mxs.items(): - # mx_list = properties.accept_mx_domains - # if len(mx_list) > 1: - # logger.warn('Lists of multiple accept-mx-domains not yet ' - # 'supported.') - # logger.warn('Using MX {} for {}'.format(mx_list[0], - # address_domain) - # ) - # logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) - # mx_domain = mx_list[0] - # mx_policy = self.policy_config.get_tls_policy(mx_domain) - # entry = address_domain + " encrypt" - # if mx_policy.min_tls_version.lower() == "tlsv1": - # entry += " protocols=!SSLv2:!SSLv3" - # elif mx_policy.min_tls_version.lower() == "tlsv1.1": - # entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - # elif mx_policy.min_tls_version.lower() == "tlsv1.2": - # entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - # else: - # logger.warn('Unknown minimum TLS version: {} '.format( - # mx_policy.min_tls_version) - # ) - # self.policy_lines.append(entry) - - # with open(self.policy_file, "w") as f: - # f.write("\n".join(self.policy_lines) + "\n") - - -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) From dde0bf08217c6fd31d83922977a233328124bcfd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Aug 2017 10:42:31 -0700 Subject: [PATCH 203/631] Remove non-plugin files --- .gitignore | 4 - README.md | 234 - Vagrantfile | 27 - examples/bigger_test_config.json | 36 - examples/config.json | 17 - examples/starttls-everywhere.json | 187 - letsencrypt-postfix/Config.py | 569 -- letsencrypt-postfix/PostfixLogSummary.py | 104 - letsencrypt-postfix/TestConfig.py | 132 - requirements.txt | 4 - share/golden-domains.txt | 32 - share/google-starttls-domains.csv | 6734 ----------------- tools/CheckSTARTTLS.py | 191 - tools/ProcessGoogleSTARTTLSDomains.py | 34 - vagrant-bootstrap.sh | 31 - vagrant-shared/certificates/ca.crt | 22 - vagrant-shared/certificates/ca.key | 27 - vagrant-shared/certificates/certificates | 1 - vagrant-shared/certificates/valid.crt | 21 - vagrant-shared/certificates/valid.csr | 18 - vagrant-shared/certificates/valid.key | 27 - .../postfix-config-sender-tls_policy | 1 - vagrant-shared/postfix-config-sender.cf | 39 - .../postfix-config-valid-example-recipient.cf | 46 - 24 files changed, 8538 deletions(-) delete mode 100644 .gitignore delete mode 100644 README.md delete mode 100644 Vagrantfile delete mode 100644 examples/bigger_test_config.json delete mode 100644 examples/config.json delete mode 100644 examples/starttls-everywhere.json delete mode 100644 letsencrypt-postfix/Config.py delete mode 100755 letsencrypt-postfix/PostfixLogSummary.py delete mode 100755 letsencrypt-postfix/TestConfig.py delete mode 100644 requirements.txt delete mode 100644 share/golden-domains.txt delete mode 100644 share/google-starttls-domains.csv delete mode 100755 tools/CheckSTARTTLS.py delete mode 100755 tools/ProcessGoogleSTARTTLSDomains.py delete mode 100755 vagrant-bootstrap.sh delete mode 100644 vagrant-shared/certificates/ca.crt delete mode 100644 vagrant-shared/certificates/ca.key delete mode 120000 vagrant-shared/certificates/certificates delete mode 100644 vagrant-shared/certificates/valid.crt delete mode 100644 vagrant-shared/certificates/valid.csr delete mode 100644 vagrant-shared/certificates/valid.key delete mode 100644 vagrant-shared/postfix-config-sender-tls_policy delete mode 100644 vagrant-shared/postfix-config-sender.cf delete mode 100644 vagrant-shared/postfix-config-valid-example-recipient.cf diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e36c9c50b..000000000 --- a/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.* -*.orig -*.pyc -*.egg-info/ diff --git a/README.md b/README.md deleted file mode 100644 index 7413228f6..000000000 --- a/README.md +++ /dev/null @@ -1,234 +0,0 @@ -# STARTTLS Everywhere - - -## Example usage - -**WARNING: this is a pre-alpha codebase. Do not run it on production -mailservers!!!** - - -If you have a Postfix server you're willing to endanger deliverability on, you -can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), note the directory it lives in below `/etc/letsencrypt/live` and then do: - -``` -git clone https://github.com/EFForg/starttls-everywhere -cd starttls-everywhere -# Promise you don't care if deliverability breaks on this mail server -letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/YOUR.DOMAIN.EXAMPLE.COM -``` - -This will: -* Ensure your mail server initiates STARTTLS encryption -* Install the Let's Encrypt cert in Postfix -* Enforce mandatory TLS to some major email domains -* Enforce minimum TLS versions to some major email domains - -## Project status - -STARTTLS Everywhere development is re-starting after a hiatus. Initial -objectives: - -* Postfix configuration generation: working pre-alpha, not yet safe -* Email security database: working pre-alpha, definitely not yet safe -* Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready -* DANE support: none yet -* DEEP support: none yet -* SMTP-STS integration: none yet -* Direct mechanisms for mail domains to request inclusion: none yet -* Failure reporting mechanisms: early progress, not yet ready -* Mechanisms for secure multi-organization signature on the policy database: - none yet -* Support for mail servers other than Postfix: none yet - -## Authors - -Jacob Hoffman-Andrews , -Peter Eckersley , -Daniel Wilcox , -Aaron Zauner - -## Mailing List - -starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere - -## Background - -Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to trivial downgrade attacks. - -To illustrate an easy version of this attack, suppose a network-based attacker `Mallory` notices that `Alice` has just uploaded message to her mail server. `Mallory` can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So `Alice`'s message will be transmitted in the clear. - -Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. - -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM or Denial of Service is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. - -STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. - -## Goals - -* Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. -* Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. -* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring. -* Create feedback-loops on targeted attacks and bulk surveilance in an opt-in, anonymized way. - -## Non-goals - -* Prevent fully-targeted exploits of vulnerabilities on endpoints or on mail hosts. -* Refuse delivery on the recipient side if sender does not negotiate TLS (this may be a future project). -* Develop a fully-decentralized solution. -* Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. - -## Motivating examples - -* [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to - prevent negotiating an encrypted connection](https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) -* [Unknown party removes STARTTLS flag from all SMTP connections leaving - Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) - -## Threat model - -Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. - -## Alternatives - -Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of DANE-based solutions easier. - -## Detailed design - -Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: - - (a) a configuration file format to convey STARTTLS support for recipient domains, - - (b) Python code (config-generator) to transform (a) into configuration files for popular MTAs., and - - (c) a method to create and securely distribute files of type (a) for major email domains that that agree to be included, plus any other domains that proactively request to be included. - -## File Format - -The basic file format will be JSON with comments (http://blog.getify.com/json-comments/). Example: - - { - // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version - "timestamp": "2014-06-06T14:30:16+00:00", - // "timestamp": 1401414363, : also acceptable - "author": "Electronic Frontier Foundation https://eff.org", - "expires": "2014-06-06T14:30:16+00:00", - "tls-policies": { - // These match on the MX domain. - "*.yahoodns.net": { - "require-valid-certificate": true, - } - "*.eff.org": { - "require-tls": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "enforce" - "accept-spki-hashes": [ - "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - ] - } - "*.google.com": { - "require-valid-certificate": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "log-only", - "error-notification": "https://google.com/post/reports/here" - }, - } - // Since the MX lookup is not secure, we list valid responses for each - // address domain, to protect against DNS spoofing. - "acceptable-mxs": { - "yahoo.com": { - "accept-mx-domains": ["*.yahoodns.net"] - } - "gmail.com": { - "accept-mx-domains": ["*.google.com"] - } - "eff.org": { - "accept-mx-domains": ["*.eff.org"] - } - } - } - - -A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. - -The _timestamp_ field is an integer number of epoch seconds from 00:00:00 UTC on 1 January 1970. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. - -There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. - -Option 1: Plain JSON distributed with a signature using gpg --clearsign. Config-generator should validate the signature against a known GPG public key before extracting. The public key is part of the permanent system configuration, like the fetch URL. - -Option 2: Git is a revision control system built on top of an authenticated, history-preserving file system. Let's use it as an authenticated, history preserving file system: valid versions of recipient policy files may be fetched and verified via signed git tags. [Here's an example shell recipe to do this.](https://gist.github.com/jsha/6230206e89759cc6e00d) - -Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. - -**address-domains** - -The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. - -Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains_ list must correspond to an exactly matching field in the _mx-domains_ config section. - -The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. - -**mx-domains** - -The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain_can be a subdomain of any other _mx-domain_in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. - -Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. - -If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ - -_Require-valid-certificate_defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: - -1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. -2. The certificate is unexpired. -3. There is a valid chain from the certificate to a root certificate included in [Mozilla's trust store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/included/) (available as [Debian package ca-certificates](https://packages.debian.org/sid/ca-certificates)). - -The _accept-pinset_ field references an entry in the pinsets list, which has the same format and semantics as [Chrome's pinning list](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json). Most _mx-domain_s should specify a pinset that describes trust roots rather than leaf certificates, but both are possible. Pinning will only be added at the request of mail operators because it requires operators be careful when issuing new leaf certificates. - -## Pinning and hostname verification - -Like Chrome and Firefox we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. - -The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. - -We do not attempt to solve the self-signed certificate problem. For mail hosts with self-signed certificates, we can require TLS but will not require validation of the certificates. Such hosts should be encouraged to upgrade to a CA-signed certificate that can be validated by senders. - -## Creating configuration - -We have three options for creating the configuration file: - -1. Ask mail operators to submit policies for their domains which we incorporate. -2. Manually curate a set of policies for the top `N` mail domains. -3. Programmatically create a set of policies by connecting to the top N mail domains. - -For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. - -For option (3) we'd be likely to pull in bad policies that could result in failed delivery. - -We'll initially launch a demo using option (2), do some initial deployments to prove viability and delivery rate impact, and then start reaching out to operators to do option (1). - -## Distribution - -The configuration file will be provided at a long-term maintained URL. It will be signed using a key held offline on an airgapped machine or smartcard. - -Since recipient mail servers may abruptly stop supporting TLS, we will request that mail operators set up auto-updating of the configuration file, with signature verification. This allows us to minimize the delivery impact of such events. However, config-generator should not auto-update its own code, since that would amount to auto-deployment of third party code, which some operators may not wish to do. - -We may choose to implement a form of immutable log along the lines of certificate transparency. This would be appealing if we chose to use this mechanism to distribute expected leaf keys as a primary authentication mechanism, but as described in "Pinning and hostname verification," that's not a viable option. Instead we will rely on the CA ecosystem to do primary authentication, so an immutable log for this system is probably overkill, engineering-wise. - -## Python code - -Config-generator should parse input JSON and produce output configs for various mail servers. It should not be possible for any input JSON to cause arbitrary code execution or even any MTA config directives beyond the ones that specifically impact the decision to deliver or bounce based on TLS support. For instance, it must not be possible for config-generator to output a directive to forward mail from one domain to another. Config-generator will have the option to directly pull the latest config from a URL, or from a file on local disk distributed regularly from another system that has outside network access. - -Config-generator will be manually updated by mail operators. - -## Testing - -We will create a reproducible test configuration that can be run locally and exercises each of the major cases: Enforce mode vs log mode; Enforced TLS negotiation, enforced MX hostname match, and enforced valid certificates. - -Additionally, for ongoing monitoring of third-party deployments, we will create a canary mail domain that intentionally fails one of the tests but is included in the configuration file. For instance, starttls-canary.org would be listed in the configuration as requiring STARTTLS, but would not actually offer STARTTLS. Each time a mail operator commits to configuring STARTTLS Everywhere, we would request an account on their email domain from which to send automated daily email to starttls-canary.org. We should expect bounces. If such mail is successfully delivered to starttls-canary.org, that would indicate a configuration failure on the sending host, and we would manually notify the operator. - -## Failure reporting - -For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job or small opt-in daemon and send notices only when STARTTLS Everywhere-related failures exceed a certain percentage for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. - -For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index b7153a7b8..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,27 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "hashicorp/precise32" - - config.vm.define "sender" do |sender| - sender.vm.network "private_network", ip: "192.168.33.5" - sender.vm.hostname = "sender.example.com" - end - config.vm.define "valid" do |valid| - valid.vm.network "private_network", ip: "192.168.33.7" - valid.vm.hostname = "valid-example-recipient.com" - end - config.vm.synced_folder "vagrant-shared", "/vagrant" - config.vm.synced_folder "vagrant-shared/starttls-everywhere", "/vagrant/starttls-everywhere" - config.vm.provision :shell, path: "vagrant-bootstrap.sh" - - config.vm.provider "virtualbox" do |vb| - # vb.gui = true - vb.customize ["modifyvm", :id, "--memory", "256"] - end - -end diff --git a/examples/bigger_test_config.json b/examples/bigger_test_config.json deleted file mode 100644 index c3c23c455..000000000 --- a/examples/bigger_test_config.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "timestamp": 1401414363, - "author": "Electronic Frontier Foundation https://eff.org", - "expires": "2015-08-01T12:00:00+08:00", - "tls-policies": { - ".yahoodns.net": { - "require-valid-certificate": true - }, - ".eff.org": { - "require-tls": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "enforce", - "accept-spki-hashes": [ - "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - ] - }, - ".google.com": { - "require-valid-certificate": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "log-only", - "error-notification": "https://google.com/post/reports/here" - } - }, - "acceptable-mxs": { - "yahoo.com": { - "accept-mx-domains": [".yahoodns.net"] - }, - "gmail.com": { - "accept-mx-domains": [".google.com"] - }, - "eff.org": { - "accept-mx-domains": [".eff.org"] - } - } -} diff --git a/examples/config.json b/examples/config.json deleted file mode 100644 index 05fc237bf..000000000 --- a/examples/config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "comment": "Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version", - "timestamp": 1401093333, - "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404677353, "comment 2:": "epoch seconds", - "tls-policies": { - ".valid-example-recipient.com": { - "min-tls-version": "TLSv1.1" - } - }, - "acceptable-mxs": { - "valid-example-recipient.com": { - "accept-mx-domains": [ ".valid-example-recipient.com" ] - } - } - -} diff --git a/examples/starttls-everywhere.json b/examples/starttls-everywhere.json deleted file mode 100644 index db76745ef..000000000 --- a/examples/starttls-everywhere.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "acceptable-mxs": { - "163.com": { - "accept-mx-domains": [ - ".163.com" - ] - }, - "aol.com": { - "accept-mx-domains": [ - ".aol.com" - ] - }, - "craigslist.org": { - "accept-mx-domains": [ - ".craigslist.org" - ] - }, - "gmail.com": { - "accept-mx-domains": [ - ".google.com" - ] - }, - "hotmail.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "icloud.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "live.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "mac.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "me.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "msn.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "naver.com": { - "accept-mx-domains": [ - ".naver.com" - ] - }, - "outlook.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "qq.com": { - "accept-mx-domains": [ - ".qq.com" - ] - }, - "rocketmail.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "rogers.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "salesforce.com": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "sbcglobal.net": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "shaw.ca": { - "accept-mx-domains": [ - ".shaw.ca" - ] - }, - "sympatico.ca": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "t-online.de": { - "accept-mx-domains": [ - ".t-online.de" - ] - }, - "wp.pl": { - "accept-mx-domains": [ - ".wp.pl" - ] - }, - "yahoo.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "yahoogroups.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "yandex.ru": { - "accept-mx-domains": [ - ".yandex.ru" - ] - }, - "ymail.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - } - }, - "tls-policies": { - ".163.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".aol.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".craigslist.org": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".google.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".icloud.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".naver.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".outlook.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".psmtp.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".qq.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".shaw.ca": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".t-online.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".wp.pl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yahoo.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yandex.ru": { - "min-tls-version": "TLSv1.1", - "require-tls": true - } - } -} diff --git a/letsencrypt-postfix/Config.py b/letsencrypt-postfix/Config.py deleted file mode 100644 index cc0df00d1..000000000 --- a/letsencrypt-postfix/Config.py +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env python -from datetime import datetime -from dateutil import parser as dateutil_parser -import collections -import json -import logging -import pprint - - -"""Idea here being to start with something that is decomposed so it's easier to -make do json in *and* out, differences between configs and config extension. -""" - -#TODO scope logging and handlers better, control verbosity by command line flags -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - -def parse_bool_from_json(value, attr_name): - if value in ('true', '1', 1, 'yes'): - bool_value = True - elif value in ('false', '0', 0, 'no'): - bool_value = False - elif value in (True, False): - bool_value = value - else: - raise ConfigError('Config value %s is an invalid boolean value.' % attr_name) - return bool_value - - -def parse_timestamp(value, attr_name): - if isinstance(value, datetime): - return value - try: - ts = int(value) - return datetime.fromtimestamp(ts) - except (TypeError, ValueError): - pass - try: - return dateutil_parser.parse(value) - except (TypeError, ValueError): - raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) - - -def verify_member_of(value, member_list, attr_name): - if value not in member_list: - raise ConfigError('Config value "%s" must be one of (%s)' % ( - attr_name, ', '.join(member_list)) - ) - return value - - -def verify_string(value, attr_name, max_length=200): - if not isinstance(value, (str, unicode)): - raise ConfigError('Config value %s must be a string.' % attr_name) - if len(value) > max_length: - raise ConfigError('Config value %s is too long.' % attr_name) - return value - - -def to_dict(config_dict): - """Cleans up BaseConfig children to be serialized.""" - d = {} - for key, val in config_dict.iteritems(): - if isinstance(val, BaseConfig): - d[key] = to_dict(val._data) - elif isinstance(val, datetime): - d[key] = val.strftime('%Y-%m-%dT%H:%M:%S%z') - elif isinstance(val, dict): - d[key] = to_dict(val) - else: - d[key] = val - return d - - -class BaseConfig(object): - """Top level config class for common methods. - - Requirements for using class: - - list all properties with getters *and* setters in class - variable 'config_properties' - - __init__ of child classes must be callable with *only* - keyword arguments to allow method calls to update to create - a new config - ... more ... - """ - - config_properties = [] - - def __init__(self): - # container for validated properties with JSON names - self._data = {} - - def __repr__(self): - s = '< %s %s >' % (self.__class__.__name__, - pprint.pformat(self._data)) - return s - - def update(self, newer_config, merge=False, **kwargs): - """Create a fresh config combining the new and old configs. - - It does this by iterating over the 'config_properties' class - attribute which contains names of property attributes for the config. - - Two methods of combining configs are possible, an 'update' and - a 'merge', the latter set by the keyword argument 'merge=True'. - - An update overrides older values with new values -- even if those - new values are None. Update will remove values that are present in - the old config if they are not present in the new config. - - A merge by comparison will allow old values to persist if they are - not specified in the new config. This can be used for end-user - customizations to override specific settings without having to re-create - large portions of a config to override it. - - Arguments: - newer_config: A config object to combine with the current config. - merge: Allows old values not overridden to survive into the fresh config. - - Returns: - A config object of the same sort as called upon. - """ - # removed 'merge' kw arg - and it was passed to constructor - # make a note to not do that, consume it on the param list - fresh_config = self.__class__(**kwargs) - logger.debug('from parent update kwargs %s' % kwargs) - logger.debug('from parent update merge %s' % merge) - if not isinstance(newer_config, self.__class__): - raise ConfigError('Attempting to update a %s with a %s' % ( - self.__class__, - newer_config.__class__)) - for prop_name in self.config_properties: - # get the specified property off of the current class - prop = self.__class__.__dict__.get(prop_name) - assert prop - new_value = prop.fget(newer_config) - old_value = prop.fget(self) - if new_value is not None: - prop.fset(fresh_config, new_value) - elif merge and old_value is not None: - prop.fset(fresh_config, old_value) - return fresh_config - - def merge(self, newer_config, **kwargs): - """Combines configs and keeps old values if they are not overridden. - - See docstring for 'update' method for more details. - - Arguments: - newer_config: A config object to combine with the current config. - merge: Allows old values not overridden to survive into the fresh config. - - Returns: - A config object of the same sort as called upon. - """ - kwargs['merge'] = True - logger.debug('from parent merge: %s' % kwargs) - return self.update(newer_config, **kwargs) - - def to_json(self): - d = to_dict(self._data) - return json.dumps(d) - - def write_to_json_file(self, json_filename, f_open=open): - data = self.to_json() - try: - with f_open(json_filename, 'w') as f: - f.write(data) - except IOError: - raise - - def load_from_json_file(self, json_filename, f_open=open): - try: - with f_open(json_filename, 'r') as f: - json_str = f.read() - json_dict = json.loads(json_str) - except IOError: - raise - except ValueError: - raise ConfigError('No valid JSON found in file: %s' % json_filename) - self.from_json_dict(json_dict) - - def from_json_dict(self, json_dict): - raise NotImplmented('BaseConfig should not be populated.') - - -class Config(BaseConfig): - """Config container for StartTLS Everywhere configuration. - - Intended as a simple container that unifies where validatation occurs, - and is capable of comparing configs to warn of things like changing - certificate fingerprints from one scan to the next. - - There is a one to one mapping of the object attributes to the JSON - object keys, albeit with dashes replaced with underscores. - """ - - def __init__(self): - super(self.__class__, self).__init__() - self._data['tls-policies'] = {} - self._data['acceptable-mxs'] = {} - - def __add__(self, other_config): - """Allow addition but not really of *full* configs, need to flesh that out.""" - #TODO add this - raise NotImplemented - - def update(self, other_config): - """Update properties of config from a 'newer' config and force verification.""" - #TODO add this - new_config = Config() - raise NotImplemented - - def from_json_dict(self, json_dict): - """Assign JSON data to Config properties and declare sub-objects. - - Let's property verification methods do the heavy lifting and mostly - maps between the JSON config names and attributes. Keeps track of - unused variables and warns about them. - """ - for key, val in json_dict.iteritems(): - if key == 'author': - self.author = val - elif key == 'comment': - self.comment = val - elif key == 'expires': - self.expires = val - elif key == 'timestamp': - self.timestamp = val - elif key == 'tls-policies': - self.make_tls_policy_dict(val) - elif key == 'acceptable-mxs': - self.make_acceptable_mxs_dict(val) - else: - logger.warn('Unknown attribute "%s", skipping' % key) - - @property - def author(self): - return self._data.get('author') - - @author.setter - def author(self, value): - self._data['author'] = verify_string(value, 'author') - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - @property - def expires(self): - return self._data.get('expires') - - @expires.setter - def expires(self, value): - self._data['expires'] = parse_timestamp(value, 'expires') - - @property - def timestamp(self): - return self._data.get('timestamp') - - @timestamp.setter - def timestamp(self, value): - self._data['timestamp'] = parse_timestamp(value, 'timestamp') - - @property - def tls_policies(self): - return self._data.get('tls-policies') - - @property - def acceptable_mxs(self): - return self._data.get('acceptable-mxs') - - def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = self.tls_policies - for domain_suffix, settings in policy_dict.iteritems(): - new_domain_policy = TLSPolicy(domain_suffix) - try: - new_domain_policy.from_json_dict(settings) - except ConfigError as e: - raise - tls_policy_dict[domain_suffix] = new_domain_policy - - def get_tls_policy(self, mx_domain): - return self.tls_policies.get(mx_domain) - - def make_acceptable_mxs_dict(self, mxs_dict): - acceptable_mxs_dict = self._data['acceptable-mxs'] - for domain, settings in mxs_dict.iteritems(): - new_domain_policy = AcceptableMX(domain) - try: - new_domain_policy.from_json_dict(settings) - except ConfigError as e: - raise - acceptable_mxs_dict[domain] = new_domain_policy - - def get_address_domains(self, mx_hostname, mx_to_domain_map): - """Do a fuzzy DNS host match on provided map to get lists of policies. - - Args: - mx_hostname (string): The hostname from an MX record. - mx_to_domain_map: Mapping from MX hosts to AcceptableMX - policies, the same AcceptableMX policy may occur more - than once. e.g. {'mx_host3': set(AcceptableMX, ...)} - The map can be generated by Config.get_mx_to_domain_policy_map. - - Returns: - The set containing all AcceptableMX policies that list the - provided MX host as viable. - """ - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in mx_to_domain_map: - return mx_to_domain_map[parent] - return None - - def get_mx_to_domain_policy_map(self): - """Create mapping of MX hostnames to sets of AcceptableMX policies. - - Generate a dictionary that is typically used in log analysis - (e.g. if your MTA logs interact with beta.innotech.com you use - this mapping to tell you it used the innotech.com AcceptableMX - policy or policies). There are of course complications. - """ - # create reverse mapping dictionary as well for auditing - # and reviewing logs - mx_to_domain_policy = collections.defaultdict(set) - - for mx_host, domain_policy in self.get_all_mx_items(): - existing_mx_policies = mx_to_domain_policy.get(mx_host) - if existing_mx_policies: - existing_domains = [ e.domain for e in existing_mx_policies ] - if domain_policy.domain not in existing_domains: - #TODO plenty of room to enforce a security policy here - # this is also the case of google apps personal domains - msg = ('Attempting to add domain policy (%s) for MX host but MX' - ' host already has a domain policy (%s), appending...') - logger.debug(msg % (domain_policy.domain, - ', '.join(existing_domains))) - mx_to_domain_policy[mx_host].add(domain_policy) - return mx_to_domain_policy - - def get_all_mx_items(self): - """Iterate over (mx_host, mx_policy) - be sure to dedup!""" - all_mx_items = [] - for policy in self.acceptable_mxs.values(): - accepted_mxs = policy.accept_mx_domains - all_mx_items.extend([(mx_host, policy) - for mx_host in accepted_mxs]) - return all_mx_items - - def get_all_mx_hosts(self): - all_mx_hosts = [] - [ all_mx_hosts.extend(domain_policy.acceptable_mxs) - for domain_policy in self.acceptable_mxs.values() ] - return all_mx_hosts - - def is_valid(self): - #TODO implement checks to make sure domains don't overlap - #TODO add debug logging for troubleshooting sake - for mx_config in self.acceptable_mxs.values(): - if not mx_config.is_valid(): - return False - for domain_suffix in mx_config.accept_mx_domains: - # check to make sure every accepted MX has a TLS policy - if not domain_suffix in self.tls_policies: - return False - all_mx_hosts = self.get_all_mx_hosts() - for domain_suffix, tls_config in self.tls_policies.iteritems(): - if not tls_config.is_valid(): - return False - # make sure no unclaimed TLS policies have made their way in - if domain_suffix not in all_mx_hosts: - return False - return True - - -class TLSPolicy(BaseConfig): - - ENFORCE_MODES = ('enforce', 'log-only') - TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') - - config_properties = ['comment', 'enforce_mode', 'min_tls_version', - 'require_tls', 'require_valid_certificate'] - - def __init__(self, domain_suffix=None): - super(self.__class__, self).__init__() - self.domain_suffix = domain_suffix - #TODO add support for two designed but yet unsupported attrs - # self._data['accept-spki-hashs'] = None - # self._data['error-notification'] = None - - def from_json_dict(self, json_dict): - for key, val in json_dict.iteritems(): - if key == 'comment': - self.comment = val - elif key == 'enforce-mode': - self.enforce_mode = val - elif key == 'min-tls-version': - self.min_tls_version = val - elif key == 'require-tls': - self.require_tls = val - elif key == 'require-valid-certificate': - self.require_valid_certificate = val - else: - logger.warn('Unknown key %s' % key) - - def is_valid(self): - """Do simple check that config contains all required values. - - Should find a way to expose easily which config values - are required, at least place in error messages such that - incomplete configs will expose it. - """ - required_attrs = ('enforce-mode', 'min-tls-version', - 'require-tls') - values_set = [self._data.get(attr) for attr in required_attrs] - if not all(values_set): - return False - else: - return True - - def update(self, newer_policy, **kwargs): - if not kwargs.get('domain_suffix'): - kwargs['domain_suffix'] = self.domain_suffix - fresh_policy = super(self.__class__, self).update(newer_policy, - **kwargs) - logger.debug('from TLS child update %s' % kwargs) - return fresh_policy - - def merge(self, newer_policy, **kwargs): - logger.debug('from TLS child merge: %s' % kwargs) - fresh_policy = super(self.__class__, self).merge(newer_policy, - domain_suffix=self.domain_suffix) - return fresh_policy - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - @property - def enforce_mode(self): - return self._data.get('enforce-mode') - - @enforce_mode.setter - def enforce_mode(self, value): - self._data['enforce-mode'] = verify_member_of(value, self.ENFORCE_MODES, 'enforce-mode') - - @property - def min_tls_version(self): - return self._data.get('min-tls-version') - - @min_tls_version.setter - def min_tls_version(self, value): - """TODO: Should this be dealing only with strings processed by map ... lower()?""" - tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] - tls_versions.extend(self.TLS_VERSIONS) - self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') - - @property - def require_tls(self): - return self._data.get('require-tls') - - @require_tls.setter - def require_tls(self, value): - self._data['require-tls'] = parse_bool_from_json(value, 'require-tls') - - @property - def require_valid_certificate(self): - return self._data.get('require-valid-certificate') - - @require_valid_certificate.setter - def require_valid_certificate(self, value): - self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') - - -class AcceptableMX(BaseConfig): - """Holds acceptable MX domain suffixes for a single mail serving domain. - - Such as for gmail.com that single mail serving suffix domain is: - gmail-smtp-in.l.google.com. - - Configuration of the acceptable MX suffix domains must match up with TLS policies - for the suffix domains. - """ - def __init__(self, domain=None): - super(self.__class__, self).__init__() - self.domain = domain - self._data['accept-mx-domains'] = [] - - @property - def accept_mx_domains(self): - return self._data.get('accept-mx-domains') - - def add_acceptable_mx(self, domain_suffix): - unique_domain_suffixes = set(self._data['accept-mx-domains']) - unique_domain_suffixes.add(domain_suffix) - self._data['accept-mx-domains'] = list(unique_domain_suffixes) - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - def is_valid(self): - """Check to make sure there is one acceptable domain suffix. - - This will need to be updated once we can actually test and support - for more than one acceptable domain suffix. - - TODO: could make this object double check the data it is given with - DNS queries. - """ - if len(self._data['accept-mx-domains']) != 1: - return False - else: - return True - - def from_json_dict(self, json_dict): - for key, val in json_dict.iteritems(): - if key == 'accept-mx-domains': - if isinstance(val, list): - for domain_suffix in val: - self.add_acceptable_mx(domain_suffix) - else: - self.add_acceptable_mx(val) - elif key == 'comment': - self.comment = val - else: - logger.warn('warning: unknown key %s' % key) - - def update(self, newer_policy, **kwargs): - logger.debug('from MX child update got %s' % kwargs) - if not kwargs.get('domain'): - kwargs['domain'] = self.domain - fresh_policy = super(self.__class__, self).update(newer_policy, - **kwargs) - if kwargs.get('merge'): - new_accepted_mxs = set(self.accept_mx_domains) - new_accepted_mxs = new_accepted_mxs.union(newer_policy.accept_mx_domains) - else: - new_accepted_mxs = newer_policy.accept_mx_domains - for domain in new_accepted_mxs: - fresh_policy.add_acceptable_mx(domain) - - return fresh_policy - - def merge(self, newer_policy, **kwargs): - logger.debug('from MX child merge: %s' % kwargs) - fresh_policy = super(self.__class__, self).merge(newer_policy, - **kwargs) - return fresh_policy - - -class ConfigError(ValueError): - def __init__(self, message): - super(self.__class__, self).__init__(message) diff --git a/letsencrypt-postfix/PostfixLogSummary.py b/letsencrypt-postfix/PostfixLogSummary.py deleted file mode 100755 index a641a9f31..000000000 --- a/letsencrypt-postfix/PostfixLogSummary.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -import argparse -import collections -import os -import re -import sys -import time - -import Config - -TIME_FORMAT = "%b %d %H:%M:%S" - -# TODO: There's more to be learned from postfix logs! Here's one sample -# observed during failures from the sender vagrant vm: - -# Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] -# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1] -# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1] -# Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix -# Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from= -# Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com> -# Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=, size=576, nrcpt=1 (queue active) -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) -# -# Also: -# Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) -def get_counts(input, config, earliest_timestamp): - seen_trusted = False - - counts = collections.defaultdict(lambda: collections.defaultdict(int)) - tls_deferred = collections.defaultdict(int) - # Typical line looks like: - # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) - # indicate a problem that should be alerted on. - # ([^[]*) <--- any group of characters that is not "[" - # Log lines for when a message is deferred for a TLS-related reason. These - deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") - # Log lines for when a TLS connection was successfully established. These can - # indicate the difference between Untrusted, Trusted, and Verified certs. - connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") - mx_to_domain_mapping = config.get_mx_to_domain_policy_map() - - timestamp = 0 - for line in sys.stdin: - timestamp = time.strptime(line[0:15], TIME_FORMAT) - if timestamp < earliest_timestamp: - continue - deferred = deferred_re.search(line) - connected = connected_re.search(line) - if connected: - validation = connected.group(1) - mx_hostname = connected.group(2).lower() - if validation == "Trusted" or validation == "Verified": - seen_trusted = True - address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) - if address_domains: - domains_str = [ a.domain for a in address_domains ] - d = ', '.join(domains_str) - counts[d][validation] += 1 - counts[d]["all"] += 1 - elif deferred: - mx_hostname = deferred.group(1).lower() - tls_deferred[mx_hostname] += 1 - return (counts, tls_deferred, seen_trusted, timestamp) - -def print_summary(counts): - for mx_hostname, validations in counts.items(): - for validation, validation_count in validations.items(): - if validation == "all": - continue - print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(description='Detect delivery problems' - ' in Postfix log files that may be caused by security policies') - arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) - arg_parser.add_argument("policy_file", nargs='?', - default=os.path.join("examples", "starttls-everywhere.json"), - help="STARTTLS Everywhere policy file") - - args = arg_parser.parse_args() - config = Config.Config() - config.load_from_json_file(args.policy_file) - - last_timestamp_processed = 0 - timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' - if os.path.isfile(timestamp_file): - last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) - (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) - with open(timestamp_file, "w") as f: - f.write(time.strftime(TIME_FORMAT, latest_timestamp)) - - # If not running in cron, print an overall summary of log lines seen from known hosts. - if not args.cron: - print_summary(counts) - if not seen_trusted: - print 'No Trusted connections seen! Probably need to install a CAfile.' - - if len(tls_deferred) > 0: - print "Some mail was deferred due to TLS problems:" - for (k, v) in tls_deferred.iteritems(): - print "%s: %s" % (k, v) diff --git a/letsencrypt-postfix/TestConfig.py b/letsencrypt-postfix/TestConfig.py deleted file mode 100755 index ca8e77654..000000000 --- a/letsencrypt-postfix/TestConfig.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -import copy -import itertools -import logging -import unittest - -import Config - -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - -class TestTLSPolicy(unittest.TestCase): - - def setUp(self): - self.old_config = Config.TLSPolicy(domain_suffix='.eff.org') - self.old_config.comment = 'Testing EFF.org TLS policy' - self.old_config.require_tls = True - self.old_config.require_valid_certificate = False - self.old_config.min_tls_version = 'TLSv1' - self.old_config.enforce_mode = 'log-only' - - self.new_config = Config.TLSPolicy(domain_suffix='.eff.org') - self.new_config.require_valid_certificate = True - self.new_config.min_tls_version = 'TLSv1.2' - self.new_config.enforce_mode = 'enforce' - - def testUpdateDropsOldSettings(self): - logger.debug('old: %s' % self.old_config) - logger.debug('new: %s' % self.new_config) - tls_policy = self.old_config.update(self.new_config) - logger.debug('just generated: %s' % tls_policy) - self.assertFalse(any([tls_policy.require_tls, tls_policy.comment])) - - def testMergeKeepsOldSettings(self): - logger.debug('old: %s' % self.old_config) - logger.debug('new: %s' % self.new_config) - tls_policy = self.old_config.merge(self.new_config, merge=True) - logger.debug('just generated: %s' % tls_policy) - self.assertTrue(all([tls_policy.require_tls, tls_policy.comment])) - - def testUpdateGetsNameSet(self): - tls_policy = self.old_config.update(self.new_config) - self.assertEquals(tls_policy.domain_suffix, self.old_config.domain_suffix) - - -class TestAcceptableMX(unittest.TestCase): - - def setUp(self): - self.old_config = Config.AcceptableMX(domain='eff.org') - self.old_config.add_acceptable_mx('.eff.org') - - def testUpdateDropsOldMXs(self): - new_bogus_mx = '.testing.eff.org' - new_config = Config.AcceptableMX(domain='eff.org') - new_config.add_acceptable_mx(new_bogus_mx) - updated_config = self.old_config.update(new_config) - self.assertNotIn('.eff.org', updated_config.accept_mx_domains) - - def testMergeKeepsOldMXs(self): - new_bogus_mx = '.testing.eff.org' - new_config = Config.AcceptableMX(domain='eff.org') - new_config.add_acceptable_mx(new_bogus_mx) - updated_config = self.old_config.merge(new_config) - self.assertListEqual(sorted(['.eff.org', '.testing.eff.org']), - sorted(updated_config.accept_mx_domains)) - - def testUpdateGetsNameSet(self): - new_policy = Config.AcceptableMX(domain=self.old_config.domain) - mx_policy = self.old_config.update(new_policy) - self.assertEquals(mx_policy.domain, self.old_config.domain) - - -class TestConfig(unittest.TestCase): - """Test entire configuration. - - Currently lower coverage is being obtained since string sets are - being compared rather than returned objects. Comparison logic for - the config objects isn't clear yet and proof that they function is enough. - """ - - def setUp(self): - self.config = Config.Config() - domain_policies = self.config._data['acceptable-mxs'] - self.mail_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', '123.cn', 'qq.com'] - for domain in self.mail_domains: - new = Config.AcceptableMX(domain=domain) - new.add_acceptable_mx('.' + domain) - domain_policies[domain] = new - - def testGetAllMxItems(self): - """Make sure the basic use case of get_all_mx_items functions.""" - # [ ('.gmail.com', 'gmail.com'), ('.yahoo.com', 'yahoo.com'), ... ] - control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] - test_data = [ (mx, p.domain) for mx, p in self.config.get_all_mx_items() ] - self.assertListEqual(sorted(test_data), sorted(control_data)) - - def testGetAllMxItemsMultiMX(self): - config = copy.deepcopy(self.config) - domain_policy = config.acceptable_mxs.get('gmail.com') - # deal with reality, mail.google.com - domain_policy.add_acceptable_mx('.mail.google.com') - control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] - control_data.append(('.mail.google.com', 'gmail.com')) - test_data = [ (mx, p.domain) for mx, p in config.get_all_mx_items() ] - self.assertListEqual(sorted(test_data), sorted(control_data)) - - def testGetMXtoDomainPolicy(self): - control_data = dict([ ('.' + domain, set([domain])) - for domain in self.mail_domains ]) - test_data = {} - for mx, pset in self.config.get_mx_to_domain_policy_map().items(): - policy_list = [ p.domain for p in pset ] - test_data[mx] = set(policy_list) - self.assertDictEqual(test_data, control_data) - - def testGetMXtoDomainPolicyMultiMX(self): - config = copy.deepcopy(self.config) - domain_policy = config.acceptable_mxs.get('gmail.com') - domain_policy.add_acceptable_mx('.mail.google.com') - control_data = dict([ ('.' + domain, set([domain])) - for domain in self.mail_domains ]) - control_data['.mail.google.com'] = set(['gmail.com']) - test_data = {} - for mx, pset in config.get_mx_to_domain_policy_map().items(): - policy_list = [ p.domain for p in pset ] - test_data[mx] = set(policy_list) - self.assertDictEqual(test_data, control_data) - - -if __name__ == '__main__': - unittest.main() diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5334ba03a..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -dnspython -publicsuffix -m2crypto -dateutils diff --git a/share/golden-domains.txt b/share/golden-domains.txt deleted file mode 100644 index bca6d2f89..000000000 --- a/share/golden-domains.txt +++ /dev/null @@ -1,32 +0,0 @@ -163.com -aol.com -bigpond.com -comcast.net -craigslist.org -facebook.com -gmail.com -gmx.de -hotmail.com -icloud.com -live.com -mac.com -me.com -msn.com -naver.com -outlook.com -qq.com -rocketmail.com -rogers.com -salesforce.com -sbcglobal.net -shaw.ca -sympatico.ca -t-online.de -ukr.net -vtext.com -web.de -wp.pl -yahoo.com -yahoogroups.com -yandex.ru -ymail.com diff --git a/share/google-starttls-domains.csv b/share/google-starttls-domains.csv deleted file mode 100644 index 5d6a00f17..000000000 --- a/share/google-starttls-domains.csv +++ /dev/null @@ -1,6734 +0,0 @@ -Address Suffix,Hostname Suffix,Direction,UN M.49 Region Code,Region Name,Fraction Encrypted -0101.co.jp,0101.co.jp,inbound,001,World,0 -04auto.biz,01auto.biz,inbound,001,World,0 -0bz.biz,hmts.jp,inbound,001,World,0 -1-day.co.nz,1-day.co.nz,inbound,001,World,0 -104.com.tw,104.com.tw,inbound,001,World,0.091112 -1105info.com,1105info.com,inbound,001,World,0 -1111.com.tw,1111.com.tw,inbound,001,World,0 -123.com.tw,123.com.tw,inbound,001,World,0 -12manage.com,netarrest.com,inbound,001,World,0.99998 -160by2.us,160by2.us,inbound,001,World,0 -160by2inbox.com,160by2inbox.com,inbound,001,World,0 -160by2invite.com,160by2invite.com,inbound,001,World,0 -160by2mail.com,160by2mail.com,inbound,001,World,0 -163.com,163.com,inbound,001,World,0.711306 -163.com,netease.com,outbound,001,World,1 -17life.com.tw,17life.com.tw,inbound,001,World,0 -1800flowersinc.com,1800flowersinc.com,inbound,001,World,0 -1800petmeds.com,1800petmeds.com,inbound,001,World,0.00599 -1lejend.com,asumeru.com,inbound,001,World,0 -1lejend.com,asumeru001.com,inbound,001,World,0 -1sale.com,1sale.com,inbound,001,World,0 -1v1y.com,euromsg.net,inbound,001,World,0 -2touchbase.com,infimail.com,inbound,001,World,0 -33go.com.tw,33go.com.tw,inbound,001,World,0 -3suisses.be,3suisses.be,inbound,001,World,0 -3suisses.fr,3suisses.fr,inbound,001,World,0 -4shared.com,4shared.com,inbound,001,World,0.999946 -4wheelparts.com,4wheelparts.com,inbound,001,World,0 -518.com.tw,518.com.tw,inbound,001,World,0 -6pm.com,6pm.com,inbound,001,World,1 -6pm.com,zappos.com,inbound,001,World,0.782581 -7net.com.tw,7net.com.tw,inbound,001,World,0 -99acres.com,99acres.com,inbound,001,World,0 -9dot9digital.in,emce2.in,inbound,001,World,0 -a8.net,a8.net,inbound,001,World,0 -aaa.com,nextjump.com,inbound,001,World,0 -aaas-science.org,aaas-science.org,inbound,001,World,0 -aafes.com,aafes.com,inbound,001,World,0 -aanotifier.nl,aanotifier.nl,inbound,001,World,0 -aarp.org,aarp.org,inbound,001,World,0.006249 -ab0.jp,altovision.co.jp,inbound,001,World,0 -abercrombie-email.com,abercrombie-email.com,inbound,001,World,0 -abercrombiekids-email.com,abercrombie-email.com,inbound,001,World,0 -about.com,about.com,inbound,001,World,2.4e-05 -about.com,sailthru.com,inbound,001,World,0 -academy-enews.com,academy-enews.com,inbound,001,World,0 -accenture.com,outlook.com,inbound,001,World,1 -accountonline.com,accountonline.com,inbound,001,World,0.348991 -acehelpfulemails.com,teradatadmc.com,inbound,001,World,0 -acemserv.com,acemserv.com,inbound,001,World,0 -activesafelist.com,zoothost.com,inbound,001,World,0 -activetrail.com,atmailsvr.net,inbound,001,World,0 -activetrail.com,mymarketing.co.il,inbound,001,World,0 -actorsaccess.com,nonfatmedia.com,inbound,001,World,0 -adchiever.com,kinder-rash-marketing.com,inbound,001,World,0 -adidas.com,neolane.net,inbound,001,World,0 -adidasusnews.com,adidasusnews.com,inbound,001,World,0 -adityabirla.com,adityabirla.com,inbound,001,World,0.006468 -adjockeys.com,thomas-j-brown.com,inbound,001,World,0 -admail.hu,sanomaonline.hu,inbound,001,World,0 -admastersafelist.com,zoothost.com,inbound,001,World,0 -adminforfree.com,adminforfree.com,inbound,001,World,1 -adminforfree.net,adminforfree.com,inbound,001,World,1 -administrativejobinsider.com,administrativejobinsider.com,inbound,001,World,0 -adobe.com,obsmtp.com,inbound,001,World,0.999986 -adobesystems.com,adobesystems.com,inbound,001,World,0 -adorama.com,adorama.com,inbound,001,World,0 -adoreme.com,exacttarget.com,inbound,001,World,0 -adp.com,adp.com,inbound,001,World,1 -adpirate.net,thomas-j-brown.com,inbound,001,World,0 -adsender.us,adsender.us,inbound,001,World,0 -adsolutionline.com,adsolutionline.com,inbound,001,World,0 -adtpulse.com,adtpulse.com,inbound,001,World,0 -adultfriendfinder.com,friendfinder.com,inbound,001,World,0 -advanceauto.com,bigfootinteractive.com,inbound,001,World,0 -advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,001,World,0 -adverts.ie,adverts.ie,inbound,001,World,1 -advfn.com,advfn.com,inbound,001,World,0.635382 -ae.com,ae.com,inbound,001,World,0 -aexp.com,aexp.com,inbound,001,World,1 -af.mil,af.mil,inbound,001,World,0.996166 -affairalert.com,iverificationsystems.com,inbound,001,World,0 -agnitas.de,agnitas.de,inbound,001,World,0.999277 -agoda-emails.com,agoda-emails.com,inbound,001,World,0 -agora.co.il,1host.co.il,inbound,001,World,0 -agorafinancial.com,agorafinancial.com,inbound,001,World,0 -agrupemonos.cl,agrupemonos.cl,inbound,001,World,1 -airbnb.com,airbnb.com,inbound,001,World,0.867975 -airbrake.io,mailgun.net,inbound,001,World,1 -airfarewatchdog.com,smartertravelmedia.com,inbound,001,World,0.011961 -airliquide.com,airliquide.com,inbound,001,World,1 -airmiles.ca,bigfootinteractive.com,inbound,001,World,0 -airtel.com,airtel.in,inbound,001,World,0.064377 -akcijatau.lt,akcijatau.lt,inbound,001,World,0 -alarm.com,alarm.com,inbound,001,World,0 -alarmnet.com,alarmnet.com,inbound,001,World,0 -alaskaair.com,alaskaair.com,inbound,001,World,0 -albertsonsemail.com,email4-mywebgrocer.com,inbound,001,World,0 -alerteimmo.com,alerteimmo.com,inbound,001,World,0 -alertid.com,alertid.com,inbound,001,World,0 -alertsindia.in,alertsindia.in,inbound,001,World,1 -alibaba.com,alibaba.com,inbound,001,World,0 -alice.it,alice.it,inbound,001,World,0 -alice.it,aliceposta.it,outbound,001,World,0 -aliexpress.com,alibaba.com,inbound,001,World,0 -alinea.fr,bp06.net,inbound,001,World,0 -alipay.com,alipay.com,inbound,001,World,1 -allegro.pl,allegro.pl,inbound,001,World,0 -allegrogroup.ua,allegrogroup.ua,inbound,001,World,0 -allegroup.hu,allegroup.hu,inbound,001,World,0 -allheart.com,allheart.com,inbound,001,World,0 -alljob.co.il,alljob.co.il,inbound,001,World,0 -allmodern.com,allmodern.com,inbound,001,World,0 -allout.org,allout.org,inbound,001,World,1 -allrecipes.com,allrecipes.com,inbound,001,World,0 -allsaints.com,allsaints.com,inbound,001,World,0 -allstate.com,rsys1.com,inbound,001,World,0 -alm.com,sailthru.com,inbound,001,World,0 -alumniclass.com,alumniclass.com,inbound,001,World,0 -alumniconnections.com,alumniconnections.com,inbound,001,World,0 -alza.cz,alza.cz,inbound,001,World,0.039449 -alza.sk,alza.cz,inbound,001,World,0.027725 -ama-assn.org,elabs10.com,inbound,001,World,0 -amadeus.com,amadeus.net,inbound,001,World,0 -amazon.{...},amazon.{...},inbound,001,World,0.020886 -amazon.{...},amazonses.com,inbound,001,World,0.999971 -amazon.{...},postini.com,inbound,001,World,0.7325 -amazon.{...},yahoo.{...},inbound,001,World,0.995995 -amazonses.com,amazonses.com,inbound,001,World,0.997919 -amazonses.com,postini.com,inbound,001,World,0.843221 -amctheatres.com,amctheatres.com,inbound,001,World,0 -americanas.com,americanas.com,inbound,001,World,0 -americanbar.org,abanet.org,inbound,001,World,0.00574 -americanexpress.com,americanexpress.com,inbound,001,World,0.000688 -americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,001,World,0 -amubm.com,amubm.com,inbound,001,World,1 -amwayemail.com,mailrouter.net,inbound,001,World,1 -ana.co.jp,ana.co.jp,inbound,001,World,0.534416 -ancestry.com,ancestry.com,inbound,001,World,0 -andrewchristian.com,emv8.com,inbound,001,World,0 -angelbroking.in,infimail.com,inbound,001,World,0 -anghami.com,mailgun.net,inbound,001,World,1 -angieslist.com,angieslist.com,inbound,001,World,0.014064 -anntaylor.com,anntaylor.com,inbound,001,World,0 -anpasia.com,anpasia.com,inbound,001,World,0 -anpdm.com,anpdm.com,inbound,001,World,6e-06 -anthropologie.com,freepeople.com,inbound,001,World,0 -aol.com,aol.com,inbound,001,World,0.999529 -aol.com,aol.com,outbound,001,World,0.999992 -aol.com,sailthru.com,inbound,001,World,0 -aol.net,aol.com,inbound,001,World,1 -apache.org,apache.org,inbound,001,World,0 -apnacomplex.com,apnacomplex.com,inbound,001,World,0.999981 -apple.com,apple.com,inbound,001,World,0.974422 -apply-4-jobs.com,apply-4-jobs.com,inbound,001,World,0 -aprovaconcursos.com.br,eadunicid.com.br,inbound,001,World,0 -aptmail.in,mailurja.com,inbound,001,World,0 -ara.cat,ara.cat,inbound,001,World,0.997709 -arcamax.com,arcamax.com,inbound,001,World,1.4e-05 -argos.co.uk,argos.co.uk,inbound,001,World,1 -argos.co.uk,exacttarget.com,inbound,001,World,1 -aritzia.com,aritzia.com,inbound,001,World,0 -armaniexchange.com,bronto.com,inbound,001,World,0 -artists-hub.com,artists-hub.com,inbound,001,World,0 -artscow.com,dyxnet.com,inbound,001,World,0.00062 -aruba.it,aruba.it,inbound,001,World,0.055666 -asadventure.com,asadventure.com,inbound,001,World,0 -asana.com,asana.com,inbound,001,World,1 -asda.com,ec-cluster.com,inbound,001,World,0 -ashampoo.com,ashampoo.com,inbound,001,World,1 -ashleymadison.com,ashleymadison.com,inbound,001,World,1 -ask.fm,ask.fm,inbound,001,World,1e-05 -askmen.com,askmen.com,inbound,001,World,0 -asos.com,asos.com,inbound,001,World,0 -assembla.com,assembla.com,inbound,001,World,1 -astrocenter.com,center.com,inbound,001,World,0 -astrology.com,astrology.com,inbound,001,World,0 -astrology.com,hsnlmailsvc.com,inbound,001,World,0 -astrology.com,webstakes.com,inbound,001,World,0 -astrology.com,wsafmailsvc.com,inbound,001,World,0 -asus.com,asus.com,inbound,001,World,0 -aswatson.com,emarsys.net,inbound,001,World,0 -athleta.com,athleta.com,inbound,001,World,0 -atlassian.net,uc-inf.net,inbound,001,World,1 -atrapalo.cl,atrapalo.com,inbound,001,World,0 -atrapalo.com,atrapalo.com,inbound,001,World,0 -att-mail.com,att-mail.com,inbound,001,World,2.2e-05 -att-mail.com,att.com,inbound,001,World,0.999966 -att.net,att.net,outbound,001,World,0.204629 -att.net,mycingular.net,inbound,001,World,0.00021 -att.net,yahoo.{...},inbound,001,World,0.99997 -auctionzip-email.com,email-auctionholdings.com,inbound,001,World,0 -auinmeio.com.br,fnac.com.br,inbound,001,World,0 -australiagsm.net,australiagsm.net,inbound,001,World,0 -authorize.net,authorize.net,inbound,001,World,0 -authorize.net,visa.com,inbound,001,World,0.993558 -autoloop.us,loop28.com,inbound,001,World,0 -autoreply.com,autoreply.com,inbound,001,World,0 -avaaz.org,avaaz.org,inbound,001,World,0 -avalanchesafelist.com,zoothost.com,inbound,001,World,0 -avast.com,avast.com,inbound,001,World,0.007867 -aveda.com,esteelauder.com,inbound,001,World,0 -avenue.com,avenue.com,inbound,001,World,0 -avg.com,avg.com,inbound,001,World,0.015703 -avira.com,avira.com,inbound,001,World,0.042528 -avito.ru,avito.ru,inbound,001,World,0.002677 -avomail.com,avomail.com,inbound,001,World,0 -avon.com,email-avonglobal.com,inbound,001,World,0 -avon.com,postdirect.com,inbound,001,World,0 -aweber.com,aweber.com,inbound,001,World,3e-06 -ayi.com,ayi.com,inbound,001,World,0 -b2b-mail.net,b2b-mail.net,inbound,001,World,0 -b2b-mail.net,contact-list.net,inbound,001,World,0 -babycenter.com,rsys3.com,inbound,001,World,0 -babyoye.com,babyoye.com,inbound,001,World,0.011564 -backcountry.com,backcountry.com,inbound,001,World,0 -backlog.jp,backlog.jp,inbound,001,World,0 -badoo.com,monopost.com,inbound,001,World,1 -bagitgetitmailer.in,emce2.in,inbound,001,World,0 -baligam.co.il,baligam.co.il,inbound,001,World,1 -balsamik.fr,balsamik.fr,inbound,001,World,0 -banamex.com,citi.com,inbound,001,World,0.999958 -banamex.com,ibrands.es,inbound,001,World,0 -bananarepublic.com,bananarepublic.com,inbound,001,World,3.48869418874254e-07 -bancoahorrofamsa.com,avantel.net.mx,inbound,001,World,1 -bancochile.cl,bancochile.cl,inbound,001,World,0.999504 -bancofalabella.com,bancofalabella.com,inbound,001,World,0 -bancomer.com,postini.com,inbound,001,World,5.9e-05 -bancomercorreo.com,bancomercorreo.com,inbound,001,World,0 -bandsintown.com,bandsintown.com,inbound,001,World,1 -banesco.com,banesco.com,inbound,001,World,0 -bankofamerica.com,bankofamerica.com,inbound,001,World,0.97133 -banorte.com,gfnorte.com.mx,inbound,001,World,0.994999 -barclaycard.co.uk,barclays.co.uk,inbound,001,World,0 -barclaycardus.com,bigfootinteractive.com,inbound,001,World,0 -barenecessities.com,barenecessities.com,inbound,001,World,0 -barleyment.ca,barleyment.ca,inbound,001,World,0 -barneys.com,barneys.com,inbound,001,World,0 -baseballsavings.com,baseballsavings.com,inbound,001,World,0 -basecamp.com,basecamp.com,inbound,001,World,1 -basecamphq.com,basecamphq.com,inbound,001,World,1 -baskinrobbins.com,baskinrobbins.com,inbound,001,World,0 -basspronews.com,basspronews.com,inbound,001,World,0 -bathandbodyworks.com,bathandbodyworks.com,inbound,001,World,0 -baublebar.com,baublebar.com,inbound,001,World,0.011219 -baycrews.co.jp,webcas.net,inbound,001,World,0 -bayt.com,bayt.com,inbound,001,World,2e-06 -bazarchic-invitations.com,bazarchic-emstech.com,inbound,001,World,0 -bbvacompass.com,postini.com,inbound,001,World,0.996402 -bcbg.com,bcbg.com,inbound,001,World,0 -bci.cl,bci.cl,inbound,001,World,0.999963 -bcp.com.pe,bcp.com.pe,inbound,001,World,1 -be2.com,nmp1.net,inbound,001,World,0 -beamtele.com,beamtele.com,inbound,001,World,0 -beanfun.com,beanfun.com,inbound,001,World,1 -beatport-email.com,beatport-email.com,inbound,001,World,0 -beautylish.com,beautylish.com,inbound,001,World,1 -bebe.com,ed10.com,inbound,001,World,0 -befrugal.com,befrugal.com,inbound,001,World,0.050462 -belkemail.com,belkemail.com,inbound,001,World,0 -bellsouth.net,att.net,outbound,001,World,0 -bellsouth.net,yahoo.{...},inbound,001,World,0.999967 -belluna.net,belluna.net,inbound,001,World,0 -benihana-news.com,benihana-news.com,inbound,001,World,0 -bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,001,World,0 -bespokeoffers.co.uk,chtah.net,inbound,001,World,0 -bestbuy.ca,bestbuy.ca,inbound,001,World,0 -bestbuy.com,bestbuy.com,inbound,001,World,0.003289 -bestdealsforyou.in,elabs5.com,inbound,001,World,0 -beta.lt,mailersend3.com,inbound,001,World,0 -betrend.com,betrend.com,inbound,001,World,0 -bevmo.com,bevmo.com,inbound,001,World,0.000967 -beyondtherack.com,beyondtherack.com,inbound,001,World,0 -bharatmatrimony.com,bharatmatrimony.com,inbound,001,World,1 -bhcosmetics.com,bronto.com,inbound,001,World,0 -bhg.com,meredith.com,inbound,001,World,0 -bigfishgames.com,bigfishgames.com,inbound,001,World,0 -biglion.ru,biglion.ru,inbound,001,World,0.999775 -biglist.com,biglist.com,inbound,001,World,0 -biglots.com,biglots.com,inbound,001,World,0.00029 -bigmailsender.com,bigmailsender.com,inbound,001,World,0 -bigpond.com,bigpond.com,inbound,001,World,0 -bigpond.com,bigpond.com,outbound,001,World,1 -bigtent.com,carezen.net,inbound,001,World,0 -bioagri.com.br,postini.com,inbound,001,World,0.991765 -biomedcentral.com,emv5.com,inbound,001,World,0 -bionexo.com,bionexo.com.br,inbound,001,World,0.999594 -birthdayalarm.com,monkeyinferno.net,inbound,001,World,0 -bitbucket.org,bitbucket.org,inbound,001,World,0 -bitlysupport.com,mailgun.info,inbound,001,World,1 -bitlysupport.com,mailgun.us,inbound,001,World,1 -bitslane.email,bitslane.email,inbound,001,World,0 -bitstatement.org,bitstatement.org,inbound,001,World,1 -bizjournals.com,bizjournals.com,inbound,001,World,0 -bizmailtoday.com,bizmailtoday.com,inbound,001,World,0 -bjs.com,bjs.com,inbound,001,World,0 -bjsrestaurants.com,bjsrestaurants.com,inbound,001,World,0 -bk.ru,mail.ru,inbound,001,World,0.992498 -blablacar.com,blablacar.com,inbound,001,World,1 -blackberry.com,blackberry.com,inbound,001,World,0 -blackboard.com,blackboard.com,inbound,001,World,0.998206 -blackboard.com,notification.com,inbound,001,World,0 -blackpeoplemeet.com,blackpeoplemeet.com,inbound,001,World,0 -blayn.jp,bserver.jp,inbound,001,World,0 -blinkboxmusic.com,mediagraft.com,inbound,001,World,1 -blissworld.com,lstrk.net,inbound,001,World,1 -blizzard.com,battle.net,inbound,001,World,0.11977 -bloglovin.com,bloglovin.com,inbound,001,World,0.000154 -blogtrottr.com,blogtrottr.com,inbound,001,World,0 -bloomberg.com,bloomberg.com,inbound,001,World,0.00501 -bloomberg.net,bloomberg.net,inbound,001,World,1 -bloomingdales.com,bloomingdales.com,inbound,001,World,0 -bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,001,World,0 -blue-compass.com,blue-compass.com,inbound,001,World,0 -bluediamondhost3.com,web-hosting.com,inbound,001,World,1 -bluehornet.com,bluehornet.com,inbound,001,World,0 -bluehost.com,bluehost.com,inbound,001,World,0.000943 -bluehost.com,hostmonster.com,inbound,001,World,0 -bluehost.com,unifiedlayer.com,inbound,001,World,1.6e-05 -bluenile.com,bluenile.com,inbound,001,World,0 -blueshellgames.com,blueshellgames.com,inbound,001,World,0 -bluestatedigital.com,bluestatedigital.com,inbound,001,World,0 -bluestonemx.com,bluestonemx.com,inbound,001,World,1 -bm05.net,bm05.net,inbound,001,World,0 -bm23.com,bronto.com,inbound,001,World,0 -bm324.com,bronto.com,inbound,001,World,0 -bmdeda99.com,bmdeda99.com,inbound,001,World,0 -bme.jp,bserver.jp,inbound,001,World,0 -bmnt.jp,bmnt.jp,inbound,001,World,0 -bmsend.com,bmsend.com,inbound,001,World,0 -bn.com,bn.com,inbound,001,World,0 -bncollegemail.com,bncollegemail.com,inbound,001,World,0 -bnetmail.com,bnetmail.com,inbound,001,World,0 -bol.com.br,bol.com.br,inbound,001,World,0 -bol.com.br,bol.com.br,outbound,001,World,0 -boletinrenuevo.com,boletinrenuevo.com,inbound,001,World,0 -bolsfr.fr,colt.net,inbound,001,World,0 -bomnegocio.com,bomnegocio.com,inbound,001,World,0.687113 -bonobos.com,bronto.com,inbound,001,World,0 -bonuszbrigad.hu,bonuszbrigad.hu,inbound,001,World,0 -boohooemail.com,smartfocusdigital.net,inbound,001,World,0 -bookbub.com,bookbub.com,inbound,001,World,1 -booking.com,booking.com,inbound,001,World,1 -bookingbuddy.com,smartertravelmedia.com,inbound,001,World,0.000486 -bookmyshow.com,eccluster.com,inbound,001,World,0 -bookoffonline.co.jp,bookoffonline.co.jp,inbound,001,World,0 -boomtownroi.com,boomtownroi.com,inbound,001,World,0 -boots.com,boots.com,inbound,001,World,0 -boscovs.com,boscovs.com,inbound,001,World,0 -bostonproper.com,bostonproper.com,inbound,001,World,0 -bouncemanager.it,musvc.com,inbound,001,World,0.362026 -boutiquesecret.com,chtah.net,inbound,001,World,0 -box.com,box.com,inbound,001,World,0.955607 -br.com,cmailsys.com,inbound,001,World,0 -bradfordexchange.com,bradfordexchange.com,inbound,001,World,0 -bradsdeals.com,bradsdeals.com,inbound,001,World,0 -brandalley.com,brandalley.com,inbound,001,World,0 -brands4friends.de,emv5.com,inbound,001,World,0 -brands4friends.jp,webcas.net,inbound,001,World,0 -brandsfever.com,mailgun.net,inbound,001,World,1 -brandsvillage.net,brandsvillage.net,inbound,001,World,0 -brassring.com,brassring.com,inbound,001,World,0.999989 -briantracyintl.com,briantracyintl.com,inbound,001,World,0 -brierleycrm.com,brierleycrm.com,inbound,001,World,0 -brijj.com,brijj.com,inbound,001,World,0 -brincltd.com,brincltd.com,inbound,001,World,0 -bronto.com,bronto.com,inbound,001,World,0 -brooksbrothers.com,brooksbrothers.com,inbound,001,World,0 -bsf01.com,bsftransmit33.com,inbound,001,World,0 -bt.com,bt.com,inbound,001,World,0.409404 -btinternet.com,cpcloud.co.uk,inbound,001,World,0 -btinternet.com,cpcloud.co.uk,outbound,001,World,0 -btinternet.com,yahoo.{...},inbound,001,World,0.99998 -budgettravel.com,email-budgettravel.com,inbound,001,World,0 -buffalo.edu,buffalo.edu,inbound,001,World,0.001059 -bumeran.com,bumeran.com,inbound,001,World,0 -burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,001,World,0 -burton.co.uk,burton.co.uk,inbound,001,World,0 -buscojobs.com,amazonaws.com,inbound,001,World,0 -buy123.com.tw,buy123.com.tw,inbound,001,World,1 -buyinvite.com.au,buyinvite.com.au,inbound,001,World,0 -buyma.com,buyma.com,inbound,001,World,0 -bv.com.br,bv.com.br,inbound,001,World,0 -bweeble.com,adlabsinc.com,inbound,001,World,0 -byway.it,byway.it,inbound,001,World,0 -bzm.mobi,nmsrv.com,inbound,001,World,1 -c21stores.com,c21stores.com,inbound,001,World,0 -ca.gov,ca.gov,inbound,001,World,0.694242 -cabelas.com,cabelas.com,inbound,001,World,0 -cabestan.com,cab07.net,inbound,001,World,0 -cadremploi.fr,cadremploi.fr,inbound,001,World,0 -cafepress.com,cafepress.com,inbound,001,World,0.000778 -caixa.gov.br,caixa.gov.br,inbound,001,World,0 -californiajobdepartment.com,californiajobdepartment.com,inbound,001,World,0 -californiapsychicsemail.com,californiapsychicsemail.com,inbound,001,World,0 -callcommand.com,callcommand.com,inbound,001,World,0 -calottery.com,calottery.com,inbound,001,World,0.999517 -cam2life.com,hinet.net,inbound,001,World,0 -camel.com,rjrsignup.com,inbound,001,World,0 -camsonline.com,camsonline.com,inbound,001,World,0.136261 -canadiantire.ca,canadiantire.ca,inbound,001,World,0 -canadianvisaexpert.net,canadianvisaexpert.net,inbound,001,World,0 -canalplus.es,canalplus.es,inbound,001,World,0 -cancer.org,delivery.net,inbound,001,World,0 -capillary.co.in,capillary.co.in,inbound,001,World,1 -capitalone.com,bigfootinteractive.com,inbound,001,World,0 -capitalone360.com,ingdirect.com,inbound,001,World,0 -capitaloneemail.com,capitaloneemail.com,inbound,001,World,0 -cardsys.at,cardsys.at,inbound,001,World,1 -care.com,care.com,inbound,001,World,0 -care2.com,care2.com,inbound,001,World,0 -career-hub.net,career-hub.net,inbound,001,World,1 -careerage.com,careerage.com,inbound,001,World,0 -careerbuilder-email.com,careerbuilder-email.com,inbound,001,World,0 -careerbuilder.com,careerbuilder.com,inbound,001,World,0.000449 -careerflash.net,careerflash.net,inbound,001,World,1 -careers24.com,careers24.com,inbound,001,World,0 -careesma.in,careesma.in,inbound,001,World,0 -carmamail.com,carmamail.com,inbound,001,World,0 -carnivalfunmail.com,carnivalfunmail.com,inbound,001,World,0 -carolsdaughter.com,carolsdaughter.com,inbound,001,World,0 -carrefour.fr,carrefour.fr,inbound,001,World,0 -carters.com,carters.com,inbound,001,World,0.003931 -cartrade.com,cartrade.com,inbound,001,World,0.006389 -carwale.com,carwale.com,inbound,001,World,0 -casasbahia.com.br,casasbahia.com.br,inbound,001,World,0 -case.edu,cwru.edu,inbound,001,World,0.999942 -caseyresearch.com,caseyresearch.com,inbound,001,World,1 -castingnetworks.com,castingnetworks.com,inbound,001,World,0.000102 -catchoftheday.com.au,inxserver.de,inbound,001,World,1 -catchyfreebies.net,mmsend53.com,inbound,001,World,0 -catererglobal.com,madgexjb.com,inbound,001,World,0 -caterermail.com,totaljobsmail.co.uk,inbound,001,World,0 -cathkidston.com,cathkidston.co.uk,inbound,001,World,0 -causes.com,causes.com,inbound,001,World,1 -cb2.com,cb2.com,inbound,001,World,0 -cbsig.net,cbsig.net,inbound,001,World,1 -ccavenue.com,avenues.info,inbound,001,World,1 -ccbchurch.com,ccbchurch.com,inbound,001,World,1 -cccampaigns.com,emv5.com,inbound,001,World,0 -cccampaigns.com,emv8.com,inbound,001,World,0 -cccampaigns.net,01net.com,inbound,001,World,0 -cccampaigns.net,cccampaigns.net,inbound,001,World,0 -cccampaigns.net,emv4.net,inbound,001,World,0 -cccampaigns.net,emv9.net,inbound,001,World,0 -ccialerts.com,ccialerts.com,inbound,001,World,0 -ccmbg.com,benchmark.fr,inbound,001,World,0 -ccs.com,footlocker.com,inbound,001,World,2.5e-05 -cdongroup.com,cdongroup.com,inbound,001,World,0.000147 -cecentertainment.com,cecentertainment.com,inbound,001,World,0 -celebritycruises.com,celebritycruises.com,inbound,001,World,0 -cenlat.com,cenlat.com,inbound,001,World,0.064459 -centaur.co.uk,centaur.co.uk,inbound,001,World,0 -centauro.com.br,centauro.com.br,inbound,001,World,0 -centerparcs.co.uk,ec-cluster.com,inbound,001,World,0 -cerberusapp.com,cerberusapp.com,inbound,001,World,1 -cfmailer.com,elabs11.com,inbound,001,World,0 -cfmvmail.com,cfmvmail.com,inbound,001,World,0 -chabad.org,chabad.org,inbound,001,World,0 -champssports.com,footlocker.com,inbound,001,World,0.000131 -chance.com,data-hotel.net,inbound,001,World,0 -change.org,change.org,inbound,001,World,1 -channel4.com,channel4.com,inbound,001,World,0.000613 -charlestyrwhitt.com,charlestyrwhitt.com,inbound,001,World,0 -charter.net,charter.net,inbound,001,World,0 -charter.net,charter.net,outbound,001,World,0 -chase.com,bigfootinteractive.com,inbound,001,World,0 -chase.com,jpmchase.com,inbound,001,World,0.999999 -chatcitynotifications.com,chatcitynotifications.com,inbound,001,World,0 -chaturbate.com,chaturbate.com,inbound,001,World,1 -cheapairmailer.com,cheapairmailer.com,inbound,001,World,0 -cheaperthandirt.com,cheaperthandirt.com,inbound,001,World,2.2e-05 -cheapflights.co.uk,cheapflights.co.uk,inbound,001,World,0 -cheapflights.com,cheapflights.com,inbound,001,World,0 -check.me,check.me,inbound,001,World,0 -cheekylovers.com,ropot.net,inbound,001,World,0 -chefscatalog.com,chefscatalog.com,inbound,001,World,0 -chelseafc.com,chelseafc.com,inbound,001,World,0.003566 -chemistdirect.co.uk,ec-cluster.com,inbound,001,World,0 -chemistry.com,chemistry.com,inbound,001,World,0 -chess.com,chess.com,inbound,001,World,1 -chiangcn.com,chiangcn.com,inbound,001,World,0 -chicagotribune.com,latimes.com,inbound,001,World,0 -chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,001,World,0 -chicos.com,chicos.com,inbound,001,World,0.000156 -childrensplace.com,childrensplace.com,inbound,001,World,0 -chinatrust.com.tw,chinatrust.com.tw,inbound,001,World,0.001272 -chopra.com,chopra.com,inbound,001,World,1 -christianbook.com,christianbook.com,inbound,001,World,1 -christianmingle.com,christianmingle.com,inbound,001,World,0 -christianmingle.com,postdirect.com,inbound,001,World,0 -chtah.com,chtah.net,inbound,001,World,0 -chtah.net,chtah.net,inbound,001,World,0 -cincghq.com,searchhomesingta.com,inbound,001,World,1 -cinesa.es,cccampaigns.com,inbound,001,World,0 -cipherzone.com,infimail.com,inbound,001,World,0 -cir.ca,cir.ca,inbound,001,World,1 -circleofmomsmail.com,circleofmomsmail.com,inbound,001,World,0 -citi.com,citi.com,inbound,001,World,0.999941 -citibank.com,bigfootinteractive.com,inbound,001,World,0 -citibank.com,citi.com,inbound,001,World,0.999997 -citicorp.com,citi.com,inbound,001,World,0.999999 -citruslane.com,citruslane.com,inbound,001,World,5e-06 -citybrands.hu,webinform.hu,inbound,001,World,1 -cityheaven.net,cityheaven.net,inbound,001,World,0 -ck.com,ck.com,inbound,001,World,0 -clarisonic.com,clarisonic.com,inbound,001,World,0 -clarks.com,clarks.com,inbound,001,World,0 -classmates.com,classmates.com,inbound,001,World,0 -clickdimensions.com,clickdimensions.com,inbound,001,World,0 -clickexperts.net,clickexperts.net,inbound,001,World,0 -clickmailer.jp,clickmailer.jp,inbound,001,World,9e-05 -clickon.com.ar,clickon.com.ar,inbound,001,World,0 -clickon.com.br,clickon.com.br,inbound,001,World,0 -clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,001,World,0 -clicplan.com,dmdelivery.com,inbound,001,World,0 -climber.com,climber.com,inbound,001,World,0 -clinique.com,esteelauder.com,inbound,001,World,0 -clubcupon.com.ar,clubcupon.com.ar,inbound,001,World,0 -cmail1.com,createsend.com,inbound,001,World,0 -cmail2.com,createsend.com,inbound,001,World,0 -cmm01.com,coremotivesmarketing.com,inbound,001,World,0 -cmrfalabella.com,cmrfalabella.com,inbound,001,World,0 -coach.com,delivery.net,inbound,001,World,0 -cobone.com,emarsys.net,inbound,001,World,0 -cocacola.co.jp,cocacola.co.jp,inbound,001,World,0 -codebreak.info,codebreak.info,inbound,001,World,1 -codeproject.com,codeproject.com,inbound,001,World,0 -coldwatercreek.com,coldwatercreek.com,inbound,001,World,0 -collectionsetc.com,collectionsetc.com,inbound,001,World,0 -columbia.edu,columbia.edu,inbound,001,World,0.762355 -combzmail.jp,combzmail.jp,inbound,001,World,0 -comcast.net,comcast.net,inbound,001,World,0.888399 -comcast.net,comcast.net,outbound,001,World,0.999999 -comenity.net,alldata.net,inbound,001,World,1 -comenity.net,bigfootinteractive.com,inbound,001,World,0 -commonfloor.com,commonfloor.com,inbound,001,World,1 -communicatoremail.com,communicatoremail.com,inbound,001,World,0 -communitymatrimony.com,communitymatrimony.com,inbound,001,World,1 -compute.internal,amazonaws.com,inbound,001,World,0.858276 -computerworld.com,computerworld.com,inbound,001,World,0 -comunicacaodemkt.com,locaweb.com.br,inbound,001,World,0 -confirmedoptin.com,confirmedoptin.com,inbound,001,World,0 -confirmsignup.com,mmsend53.com,inbound,001,World,0 -conrepmail.com,conrepmail.com,inbound,001,World,0 -constantcontact.com,confirmedcc.com,inbound,001,World,0 -constantcontact.com,constantcontact.com,inbound,001,World,5.3e-05 -constantcontact.com,postini.com,inbound,001,World,0.078144 -constantcontact.com,yahoo.{...},inbound,001,World,0.999724 -contact-darty.com,mm-send.com,inbound,001,World,0 -contactlab.it,contactlab.it,inbound,001,World,0 -containerstore.com,containerstore.com,inbound,001,World,0 -continente.pt,1-hostingservice.com,inbound,001,World,0 -converse.com,converse.com,inbound,001,World,1 -convio.net,convio.net,inbound,001,World,0 -cookingchanneltv.com,cookingchanneltv.com,inbound,001,World,0 -cookpad.com,cookpad.com,inbound,001,World,0 -copernica.nl,picsrv.net,inbound,001,World,0.011507 -copernica.nl,vicinity.nl,inbound,001,World,0.011753 -coppel.com,coppel.com,inbound,001,World,0 -coremotivesmarketing.com,coremotivesmarketing.com,inbound,001,World,0 -cornell.edu,cornell.edu,inbound,001,World,0.195104 -corporateperks.com,nextjump.com,inbound,001,World,0 -correosocc.com,correosocc.com,inbound,001,World,1 -costco.co.uk,costco.com,inbound,001,World,0 -costco.com,costco.com,inbound,001,World,7e-06 -costcophotocenter.com,wc09.net,inbound,001,World,0 -costcoservices.com,costco.com,inbound,001,World,0 -cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,001,World,0 -couchsurfing.org,couchsurfing.com,inbound,001,World,0 -countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,001,World,0 -couponamama.com,couponamama.com,inbound,001,World,1 -coupondunia.in,coupondunia.in,inbound,001,World,1 -cox.com,cox.com,inbound,001,World,0.001665 -cox.net,cox.net,inbound,001,World,0.009187 -cox.net,cox.net,outbound,001,World,0 -coyotelogistics.com,postini.com,inbound,001,World,0 -cp20.com,cp20.com,inbound,001,World,0 -cpbnc.com,cpbnc.com,inbound,001,World,0 -cpbnc.com,fye.com,inbound,001,World,0 -cpc.gov.in,cpc.gov.in,inbound,001,World,0 -cpm.co.ma,cpm.co.ma,inbound,001,World,0 -crabtree-evelyn.com,crabtree-evelyn.com,inbound,001,World,0.000566 -crackle.com,crackle.com,inbound,001,World,0 -craigslist.org,craigslist.org,inbound,001,World,0 -craigslist.org,craigslist.org,outbound,001,World,1 -crainnewsalerts.com,crainnewsalerts.com,inbound,001,World,0 -crashlytics.com,crashlytics.com,inbound,001,World,1 -crashlytics.com,sendgrid.net,inbound,001,World,1 -crateandbarrel.com,crateandbarrel.com,inbound,001,World,0 -cratusservices.in,ramcorp.in,inbound,001,World,0 -creationsrewards.net,creationsrewards.net,inbound,001,World,0 -creditkarma.com,creditkarma.com,inbound,001,World,1 -credoaction.com,credoaction.com,inbound,001,World,1 -cricinfo.com,cricinfo.com,inbound,001,World,0 -cricut.com,elabs12.com,inbound,001,World,0 -criticalimpactinc.com,criticalimpactinc.com,inbound,001,World,0 -critsend.com,critsend.com,inbound,001,World,0 -crmstyle.com,crmstyle.com,inbound,001,World,0 -crocos.jp,crocos.jp,inbound,001,World,0 -crocs-email.com,crocs-email.com,inbound,001,World,0 -crosswalkmail.com,crosswalkmail.com,inbound,001,World,0 -crowdcut.com,crowdcut.com,inbound,001,World,1 -crsend.com,crsend.com,inbound,001,World,0.008688 -crunchyroll.com,crunchyroll.com,inbound,001,World,0 -csas.cz,csas.cz,inbound,001,World,0.999971 -ctrip.com,ctrip.com,inbound,001,World,0.01917 -cudo.com.au,exacttarget.com,inbound,001,World,0 -cuenote.jp,cuenote.jp,inbound,001,World,0 -cumulusdist.net,cumulusdist.net,inbound,001,World,0 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,001,World,1 -cuponatic.com.pe,cuponatic.com.pe,inbound,001,World,1 -cuponicamail.com,fnbox.com,inbound,001,World,0 -cuppon.pl,cuppon.pl,inbound,001,World,0 -curbednetwork.com,curbednetwork.com,inbound,001,World,1 -curriculum.com.br,curriculum.com.br,inbound,001,World,0 -currys.co.uk,currys.co.uk,inbound,001,World,0 -cuspemail.com,neimanmarcusemail.com,inbound,001,World,0 -custom-emailing.com,elabs12.com,inbound,001,World,0 -custombriefings.com,custombriefings.com,inbound,001,World,0 -customercenter.net,customercenter.net,inbound,001,World,0.996453 -customeriomail.com,customeriomail.com,inbound,001,World,1 -cv-library.co.uk,cv-library.co.uk,inbound,001,World,0 -cvbankas.lt,efadm.eu,inbound,001,World,0 -cvent-planner.com,cvent-planner.com,inbound,001,World,0 -cw.com.tw,cw.com.tw,inbound,001,World,0.002535 -cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 -cxomedia.com,cxomedia.com,inbound,001,World,0 -cybercoders.com,cybercoders.com,inbound,001,World,0 -cyberdiet.com.br,allinmedia.com.br,inbound,001,World,0 -cyberlinkmember.com,cyberlinkmember.com,inbound,001,World,0 -d-reizen.nl,dmdelivery.com,inbound,001,World,0 -dabmail.com,iaires.com,inbound,001,World,0 -dabmail.com,mailurja.com,inbound,001,World,0 -dafiti.cl,dafiti.cl,inbound,001,World,0 -dafiti.com.br,fagms.de,inbound,001,World,0 -dailyhoroscope.com,tarot.com,inbound,001,World,0 -dailyom.com,dailyom.com,inbound,001,World,1 -dairyqueen.com,dairyqueen.com,inbound,001,World,0 -datadrivenemail.com,datadrivenemail.com,inbound,001,World,0 -datehookup.com,datehookup.com,inbound,001,World,0 -datingfactory.com,caerussolutions.net,inbound,001,World,0 -datingvipnotifications.com,datingvipnotifications.com,inbound,001,World,0 -daveramsey.com,daveramsey.com,inbound,001,World,0.025924 -daviacalendar.com,daviacalendar.com,inbound,001,World,1 -davidsbridal.com,davidsbridal.com,inbound,001,World,0 -davidstea.com,bronto.com,inbound,001,World,0 -daz3d.com,bronto.com,inbound,001,World,0 -dbgi.co.uk,emc1.co.uk,inbound,001,World,0 -ddc-emails.com,ddc-emails.com,inbound,001,World,0 -deal.com.sg,emarsys.net,inbound,001,World,0 -dealchicken.com,dealchicken.com,inbound,001,World,0 -dealchicken.com,exacttarget.com,inbound,001,World,0 -dealersocket.com,dealersocket.com,inbound,001,World,4e-06 -dealfind.com,dealfind.com,inbound,001,World,0 -dealnews.com,dealnews.com,inbound,001,World,0 -dealsaver.com,secondstreetmedia.com,inbound,001,World,0.999823 -dealsdirect.com.au,dealsdirect.com.au,inbound,001,World,0 -dealspl.us,dealspl.us,inbound,001,World,0 -debian.org,debian.org,inbound,001,World,1 -debshops.com,lstrk.net,inbound,001,World,1 -deezer.com,dms30.com,inbound,001,World,0 -deliasshopemail.com,deliasshopemail.com,inbound,001,World,0 -delivery.net,delivery.net,inbound,001,World,0 -delivery.net,m0.net,inbound,001,World,0 -dell.com,bfi0.com,inbound,001,World,0 -dell.com,dell.com,inbound,001,World,0.969277 -delta.com,delta.com,inbound,001,World,0.092496 -dena.ne.jp,dena.ne.jp,inbound,001,World,0.000249 -dentalsenders.com,dentalsenders.com,inbound,001,World,0 -dermstore.com,exacttarget.com,inbound,001,World,0 -descontos.pt,descontos.pt,inbound,001,World,0 -designerapparel.com,myperfectsale.com,inbound,001,World,1 -despegar.com,despegar.com,inbound,001,World,0 -dhgate.com,chtah.net,inbound,001,World,0 -dhl.com,dhl.com,inbound,001,World,0.994107 -dice.com,dice.com,inbound,001,World,0 -dietaesaude.com.br,dietaesaude.com.br,inbound,001,World,0 -dietnavi.com,data-hotel.net,inbound,001,World,0 -digitalmailer.com,digitalmailer.com,inbound,001,World,0 -digitalmedia-comunicacion.es,chtah.net,inbound,001,World,0 -digitalromanceinc.com,digitalromanceinc.com,inbound,001,World,1 -dinda.com.br,dinda.com.br,inbound,001,World,0.017061 -dip-net.co.jp,dip-net.co.jp,inbound,001,World,0 -directcrm.ru,directcrm.ru,inbound,001,World,0 -directresponsemanager.com,wide.ne.jp,inbound,001,World,0 -directv.com,directv.com,inbound,001,World,0.050604 -directvla.com,directvla.com,inbound,001,World,0 -disc.co.jp,disc.co.jp,inbound,001,World,0.001051 -discover.com,discover.com,inbound,001,World,0 -discover.com,discoverfinancial.com,inbound,001,World,1 -dishtv.co.in,dishtv.co.in,inbound,001,World,0.047664 -disney.co.uk,emv9.com,inbound,001,World,0 -disneydestinations.com,disneyparks.com,inbound,001,World,0 -disneydestinations.com,disneyworld.com,inbound,001,World,0 -disparadordeemails.com,locaweb.com.br,inbound,001,World,0 -disqus.net,disqus.net,inbound,001,World,0 -diynetwork.com,diynetwork.com,inbound,001,World,0 -dks.com.tw,dks.com.tw,inbound,001,World,0.018134 -dmm.com,dmm.com,inbound,001,World,0 -dn.net,naukri.com,inbound,001,World,0 -docomo.ne.jp,docomo.ne.jp,inbound,001,World,0 -docomo.ne.jp,docomo.ne.jp,outbound,001,World,0 -doctoroz.com,email-sharecare2.com,inbound,001,World,0 -docusign.net,docusign.net,inbound,001,World,0.985435 -dollartree.com,email-dollartree.com,inbound,001,World,0 -dominos.com,dominos.com,inbound,001,World,0.011684 -dominos.com.au,dominos.com.au,inbound,001,World,0 -dominosemail.co.uk,dominosemail.co.uk,inbound,001,World,0 -donationnet.net,donationnet.net,inbound,001,World,0 -donuts.ne.jp,dnuts.jp,inbound,001,World,0 -doodle.com,doodle.com,inbound,001,World,1 -dorothyperkins.com,dorothyperkins.com,inbound,001,World,0 -dotmailer-email.com,dotmailer.com,inbound,001,World,0 -dotmailer.co.uk,dotmailer.com,inbound,001,World,0 -dotz.com.br,dotz.com.br,inbound,001,World,0 -doubletakeoffers.com,doubletakeoffers.com,inbound,001,World,0 -dowjones.info,dowjones.info,inbound,001,World,0 -downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,001,World,0 -dpapp.nl,prikbordmailer.nl,inbound,001,World,0 -dpapp.nl,sslsecuref.nl,inbound,001,World,0 -dptagent.biz,dptagent.biz,inbound,001,World,0 -dptagent.net,dptagent.net,inbound,001,World,0 -draftkings.com,draftkings.com,inbound,001,World,0 -dreamhost.com,dreamhost.com,inbound,001,World,0 -dreammail.ne.jp,dreammail.jp,inbound,001,World,0 -dreamwidth.org,dreamwidth.org,inbound,001,World,0 -dreivip.com,dreivip.com,inbound,001,World,0.000301 -dress-for-less.de,privalia.com,inbound,001,World,0 -drhinternet.net,drhinternet.net,inbound,001,World,0 -driftem.com,emce2.in,inbound,001,World,0 -driftem.com,mailurja.com,inbound,001,World,0 -drjays-mail.com,drjays-mail.com,inbound,001,World,0 -dromadaire-news.com,ecmcluster.com,inbound,001,World,0 -dropbox.com,dropbox.com,inbound,001,World,1 -dropboxmail.com,dropbox.com,inbound,001,World,1 -drushim.co.il,drushim.co.il,inbound,001,World,0 -drweb.com,drweb.com,inbound,001,World,0.969315 -dstyleweb.com,dstyleweb.com,inbound,001,World,0.001099 -dsw.com,dsw.com,inbound,001,World,0 -ducks.org,uptilt.com,inbound,001,World,0 -duke.edu,duke.edu,inbound,001,World,0.308101 -dukecareers.com,dukecareers.com,inbound,001,World,1 -duluthtradingemail.com,email-duluthtrading.com,inbound,001,World,0 -dvor.com,dvor.com,inbound,001,World,0 -dynamite-safelist.com,thomas-j-brown.com,inbound,001,World,0 -dynect-mailer.net,dynect.net,inbound,001,World,0 -dynect-mailer.net,sendlabs.com,inbound,001,World,0 -e-activist.com,e-activist.com,inbound,001,World,0 -e-beallsonline.com,e-stagestores.com,inbound,001,World,0 -e-bodyc.com,email-bodycentral.com,inbound,001,World,0 -e-boks.dk,e-boks.dk,inbound,001,World,1 -e-costco.mx,costco.com,inbound,001,World,0 -e-ebuyer.com,e-ebuyer.com,inbound,001,World,0 -e-goodysonline.com,e-stagestores.com,inbound,001,World,0 -e-jobs-ville.com,e-jobs-ville.com,inbound,001,World,1 -e-leclerc.com,e-leclerc.com,inbound,001,World,0.000248 -e-mark.nl,e-mark.nl,inbound,001,World,0 -e-ngine.nl,e-ngine.nl,inbound,001,World,0 -e-peebles.com,e-stagestores.com,inbound,001,World,0 -e-rewards.net,e-rewards.net,inbound,001,World,1 -e-stagestores.com,e-stagestores.com,inbound,001,World,0 -e-travelclub.es,e-travelclub.es,inbound,001,World,0 -e2ma.net,e2ma.net,inbound,001,World,1 -ea.com,ea.com,inbound,001,World,0.001314 -eaccess.net,postini.com,inbound,001,World,0 -earn-e-miles.com,earn-e-miles.com,inbound,001,World,0 -earnerslist.com,traxweb.net,inbound,001,World,9e-06 -earthfare-email.com,edclient2.com,inbound,001,World,0 -earthlink.net,earthlink.net,inbound,001,World,0.031678 -earthlink.net,earthlink.net,outbound,001,World,0 -eastbay.com,footlocker.com,inbound,001,World,0 -easycanvasprints.com,easycanvasprints.com,inbound,001,World,0 -easyhealthoptions.com,easyhealthoptions.com,inbound,001,World,1 -easyhits4u.com,easyhits4u.com,inbound,001,World,1 -easyhits4u.com,relmax.net,inbound,001,World,0 -easyroommate.com,easyroommate.com,inbound,001,World,0.108483 -ebags.com,ebags.com,inbound,001,World,0 -ebates.com,bfi0.com,inbound,001,World,0 -ebay-kleinanzeigen.de,mobile.de,inbound,001,World,1 -ebay.{...},ebay.{...},inbound,001,World,0.99953 -ebay.{...},emarsys.net,inbound,001,World,0 -ebay.{...},postdirect.com,inbound,001,World,0 -ebizac2.com,ebizac2.com,inbound,001,World,0 -ebizac3.com,ebizac3.com,inbound,001,World,0 -eblastengine.com,secondstreetmedia.com,inbound,001,World,0.999827 -ebuildabear.com,ebuildabear.com,inbound,001,World,8e-06 -ec2.internal,amazonaws.com,inbound,001,World,0.769117 -ec21.com,ec21.com,inbound,001,World,0.002261 -ecasend.com,ecasend.com,inbound,001,World,0 -ecnavi.jp,ecnavi.jp,inbound,001,World,0 -ecommzone.com,ecommzone.com,inbound,001,World,0 -ed.gov,leepfrog.com,inbound,001,World,0.106381 -ed10.net,ed10.com,inbound,001,World,0 -ed10.net,postini.com,inbound,001,World,0.083658 -edarling.fr,fagms.de,inbound,001,World,0 -eddiebauer.com,eddiebauer.com,inbound,001,World,0 -edima.hu,edima.hu,inbound,001,World,0 -edirect1.com,ivytech.edu,inbound,001,World,0 -edmodo.com,edmodo.com,inbound,001,World,1.1e-05 -educationzone.co.in,iaires.com,inbound,001,World,0 -eduk.com,eduk.com,inbound,001,World,0 -efamilydollar.com,efamilydollar.com,inbound,001,World,0 -effectivesafelist.com,zoothost.com,inbound,001,World,0 -efox-shop.com,dmdelivery.com,inbound,001,World,0 -eharmony.com,eharmony.com,inbound,001,World,1e-06 -eigbox.net,eigbox.net,inbound,001,World,0 -ejobs.ro,ejobs.ro,inbound,001,World,8.4e-05 -elabs10.com,elabs10.com,inbound,001,World,0 -elabs12.com,elabs12.com,inbound,001,World,0 -elabs3.com,elabs3.com,inbound,001,World,0 -elabs3.com,meritline.com,inbound,001,World,0 -elabs5.com,elabs5.com,inbound,001,World,0 -elabs6.com,elabs6.com,inbound,001,World,0 -elaine-asp.de,artegic.net,inbound,001,World,0.999997 -elanceonline.com,elanceonline.com,inbound,001,World,0 -elcorteingles.es,elcorteingles.es,inbound,001,World,0 -eleadtrack.net,eleadtrack.net,inbound,001,World,0 -elektronskaposta.si,eprvak.si,inbound,001,World,0 -elettershop.de,servicemail24.de,inbound,001,World,1 -elistas.net,elistas.net,inbound,001,World,0 -elitesafelist.com,elitesafelist.com,inbound,001,World,0 -elkjop.no,ec-cluster.com,inbound,001,World,0 -elkjop.no,eccluster.com,inbound,001,World,0 -elo7.com.br,elo7.com.br,inbound,001,World,0 -emag.ro,emag.ro,inbound,001,World,0.005013 -email-1800contacts.com,email-1800contacts.com,inbound,001,World,0 -email-aaa.com,email-aaa.com,inbound,001,World,0 -email-aeriagames.com,email-aeriagames.com,inbound,001,World,0 -email-comparethemarket.com,smartfocusdigital.net,inbound,001,World,0 -email-cooking.com,email-cooking.com,inbound,001,World,0 -email-dressbarn.com,email-dressbarn.com,inbound,001,World,0 -email-firestone.com,reminder-firestone.com,inbound,001,World,0 -email-galls.com,email-galls.com,inbound,001,World,1 -email-honest.com,email-honest.com,inbound,001,World,0 -email-od.com,email-od.com,inbound,001,World,0.999652 -email-od.com,smtprelayserver.com,inbound,001,World,0.999779 -email-petsmart.com,email-petsmart.com,inbound,001,World,0 -email-sportchalet.com,email-sportchalet.com,inbound,001,World,0 -email-telekom.de,ecm-cluster.com,inbound,001,World,0 -email-ticketdada.com,email-ticketdada.com,inbound,001,World,1 -email-totalwine.com,email-totalwine.com,inbound,001,World,0 -email-wildstar-online.com,email-carbine.com,inbound,001,World,0 -email2-beyond.com,messagebus.com,inbound,001,World,0 -email360api.com,email360api.com,inbound,001,World,0 -email365inc.com,email365inc.com,inbound,001,World,0 -email3m.com,email3m.com,inbound,001,World,0 -email4-beyond.com,email4-beyond.com,inbound,001,World,0 -emailcounts.com,secureserver.net,inbound,001,World,0 -emaildir2.com,emaildirect.net,inbound,001,World,0 -emaildir2.com,espsnd.com,inbound,001,World,0 -emailnotify.net,emailnotify.net,inbound,001,World,0.961424 -emailrestaurant.com,emailrestaurant.com,inbound,001,World,0 -emailsbancoestado.cl,emailsbancoestado.cl,inbound,001,World,0 -emailsripley.cl,etarget.cl,inbound,001,World,0 -emailtoryburch.com,emailtoryburch.com,inbound,001,World,0 -emarsys.net,emarsys.net,inbound,001,World,0.092041 -embarqmail.com,centurylink.net,inbound,001,World,0.999918 -embluejet.com,embluejet.com,inbound,001,World,0 -embluejet.com,emblueuser.com,inbound,001,World,0 -emcsend.com,emcsend.com,inbound,001,World,0 -emergencyemail.org,emergencyemail.org,inbound,001,World,0 -eminentinc.com,eminentinc.com,inbound,001,World,0 -emktsender.net,locaweb.com.br,inbound,001,World,0 -emktviajarbarato.com.br,splio.com.br,inbound,001,World,0.875902 -emma.cl,emma.cl,inbound,001,World,0.996895 -emobile.ad.jp,postini.com,inbound,001,World,0 -employboard.com,employboard.com,inbound,001,World,1 -empoweredcomms.com.au,empoweredcomms.com.au,inbound,001,World,0 -emsecure.net,emsecure.net,inbound,001,World,0 -emsmtp.com,emsmtp.com,inbound,001,World,0.175797 -en-japan.com,en-japan.com,inbound,001,World,0.000204 -en25.com,kqed.org,inbound,001,World,0 -enewscartes.net,bp06.net,inbound,001,World,0 -enewsletter.pl,enewsletter.pl,inbound,001,World,0.005395 -enewsletter.pl,mydeal.pl,inbound,001,World,0 -enewsletter.pl,sare25.com,inbound,001,World,0 -enplenitud.com,enplenitud.com,inbound,001,World,0 -entregadeemails.com,locaweb.com.br,inbound,001,World,0 -entregadordecampanhas.net,locaweb.com.br,inbound,001,World,0 -entrepreneur.com,entrepreneur.com,inbound,001,World,0 -enviodecampanhas.net,locaweb.com.br,inbound,001,World,0 -enviodemkt.com.br,locaweb.com.br,inbound,001,World,0 -eonet.ne.jp,eonet.ne.jp,inbound,001,World,0.99955 -epaper.com.tw,epaper.com.tw,inbound,001,World,0 -eplus.jp,eplus.jp,inbound,001,World,0 -epriority.com,epriority.com,inbound,001,World,0 -equifax.com,equifax.com,inbound,001,World,0.936013 -equussafelist.com,equussafelist.com,inbound,001,World,0.000265 -eslitebooks.com,eslitebooks.com,inbound,001,World,0 -espmp-agfr.net,bp06.net,inbound,001,World,0 -esprit-friends.com,esprit-friends.com,inbound,001,World,0 -esri.com,esri.com,inbound,001,World,0.983104 -esteelauder.com,esteelauder.com,inbound,001,World,0 -ethingsremembered.com,ethingsremembered.com,inbound,001,World,0 -etrade.com,etrade.com,inbound,001,World,0.021422 -etransmail.com,etransmail.com,inbound,001,World,0 -etransmail.com,ptransmail.com,inbound,001,World,0 -etrmailbox.com,etrmailbox.com,inbound,001,World,0 -etsy.com,etsy.com,inbound,001,World,0.020844 -euromsg.net,euromsg.net,inbound,001,World,0 -evaair.com,evaair.com,inbound,001,World,0.007363 -evanguard.com,evanguard.com,inbound,001,World,0 -evanscycles.com,msgfocus.com,inbound,001,World,0 -eventbrite.com,eventbrite.com,inbound,001,World,0 -evernote.com,evernote.com,inbound,001,World,1 -eversavelocal.com,eversavelocal.com,inbound,001,World,0 -everydayfamily.com,everydayfamily.com,inbound,001,World,0 -everydayhealthinc.com,waterfrontmedia.net,inbound,001,World,0 -everyjobforme.com,everyjobforme.com,inbound,001,World,0 -everytown.org,everytown.org,inbound,001,World,1 -exacttarget.com,bazaarvoice.com,inbound,001,World,0 -exacttarget.com,booksamillion.com,inbound,001,World,0 -exacttarget.com,exacttarget.com,inbound,001,World,0.000325 -exacttarget.com,msg.com,inbound,001,World,0 -exacttarget.com,redboxinstant.com,inbound,001,World,0 -exacttarget.com,skylinetechnologies.com,inbound,001,World,0 -exchangesolutions.com,exchangesolutions.com,inbound,001,World,0.000143 -exec-u-net-mail.com,exec-u-net-mail.com,inbound,001,World,0 -expediamail.com,airasiago.com,inbound,001,World,1 -expediamail.com,exacttarget.com,inbound,001,World,1 -expediamail.com,expediamail.com,inbound,001,World,0.839662 -expediamail.com,quotitmail.com,inbound,001,World,0 -experteer.com,experteer.com,inbound,001,World,0 -express.com,expressfashion.com,inbound,001,World,0 -exprpt.com,exprpt.com,inbound,001,World,0 -expvtinboxhub.net,expvtinboxhub.net,inbound,001,World,0 -extra.com.br,emv8.com,inbound,001,World,0 -eyepin.com,eyepin.com,inbound,001,World,0 -ezweb.ne.jp,ezweb.ne.jp,inbound,001,World,0.282076 -ezweb.ne.jp,ezweb.ne.jp,outbound,001,World,0 -fabfurnish.com,fagms.de,inbound,001,World,0 -fabletics.com,bronto.com,inbound,001,World,0 -facebook.com,facebook.com,inbound,001,World,0.553197 -facebook.com,facebook.com,outbound,001,World,1 -facebookappmail.com,facebook.com,inbound,001,World,1 -facebookmail.com,facebook.com,inbound,001,World,1 -facebookmail.com,postini.com,inbound,001,World,0.726315 -facebookmail.com,yahoo.{...},inbound,001,World,0.999904 -facilisimo.com,facilisimo.com,inbound,001,World,1 -fagms.net,fagms.de,inbound,001,World,0 -falabella.com,falabella.com,inbound,001,World,0 -familychristianmail.com,familychristianmail.com,inbound,001,World,0 -famousfootwear.com,famousfootwear.com,inbound,001,World,0 -fanatics.com,fanatics.com,inbound,001,World,0 -fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,001,World,0 -fanbridge.com,fanbridge.com,inbound,001,World,0.000198 -fanfiction.com,fictionpress.com,inbound,001,World,1 -fanofannas.com,fanofannas.com,inbound,001,World,0 -fansedge.com,fansedge.com,inbound,001,World,0 -farmers.com,farmers.com,inbound,001,World,0.999984 -farmersonly.com,mailgun.net,inbound,001,World,1 -farmersonly.com,mailgun.us,inbound,001,World,1 -fashion2hub.in,mgenie.in,inbound,001,World,0 -fastcompany.com,fastcompany.com,inbound,001,World,0 -fastgb.com,fastgb.com,inbound,001,World,0 -fastlistmailer.com,zoothost.com,inbound,001,World,0 -fastweb.com,fastweb.com,inbound,001,World,0 -fbi.gov,fbi.gov,inbound,001,World,0 -fbmta.com,fbmta.com,inbound,001,World,0 -fc2.com,fc2.com,inbound,001,World,0.000798 -fedex.com,fedex.com,inbound,001,World,0.921011 -fedoraproject.org,fedoraproject.org,inbound,001,World,5e-06 -feedblitz.com,feedblitz.com,inbound,001,World,0 -feld-ent.com,postdirect.com,inbound,001,World,0 -felissimo.jp,felissimo.jp,inbound,001,World,0 -fellowshiponemail.com,fellowshiponemail.com,inbound,001,World,0 -fetlifemail.com,fetlifemail.com,inbound,001,World,0 -fibertel.com.ar,fibertel.com.ar,inbound,001,World,0.003897 -fidelity.com,fidelity.com,inbound,001,World,1 -fidelizador.org,fidelizador.org,inbound,001,World,0 -financialfreedommail.com,financialfreedommail.com,inbound,001,World,0 -finansbank.com.tr,finansbank.com.tr,inbound,001,World,0.927 -findexpvtinbox.com,findexpvtinbox.com,inbound,001,World,0 -fingerhut.com,fingerhut.com,inbound,001,World,0.020664 -finishline.com,finishline.com,inbound,001,World,0 -finn.no,schibsted-it.no,inbound,001,World,0.002893 -firemountaingems.com,firemountaingems.com,inbound,001,World,0 -fiscosoft.com.br,fiscosoft.com.br,inbound,001,World,0 -fisher-price.com,fisher-price.com,inbound,001,World,0 -fitbit.com,fitbit.com,inbound,001,World,1 -fitnessmagazine.com,meredith.com,inbound,001,World,0 -fiverr.com,fiverr.com,inbound,001,World,0 -fixeads.com,fixeads.com,inbound,001,World,0 -flets.com,flets.com,inbound,001,World,0 -flexmls.com,flexmls.com,inbound,001,World,0.999992 -flightaware.com,flightaware.com,inbound,001,World,0.020698 -flipkart.com,flipkart.com,inbound,001,World,1 -flirchi.com,flirchi.com,inbound,001,World,1.0 -flirt.com,ropot.net,inbound,001,World,0 -flirthookup.com,flirthookup.com,inbound,001,World,1 -flirtlocal.com,flirtlocal.com,inbound,001,World,1 -flmsecure.com,fling.com,inbound,001,World,0 -flmsecure.com,flmsecure.com,inbound,001,World,0 -floridajobdepartment.com,floridajobdepartment.com,inbound,001,World,0 -flyceb.com,flyceb.com,inbound,001,World,0 -flyfrontier.com,flyfrontier.com,inbound,001,World,0 -flymonarchemail.com,flymonarchemail.com,inbound,001,World,0 -fmworld.net,fmworld.net,inbound,001,World,0 -fnac.com,fnac.com,inbound,001,World,0.021985 -fnb.co.za,fnb.co.za,inbound,001,World,0.104983 -fofa.jp,mpme.jp,inbound,001,World,0 -follow-up.se,follow-up.se,inbound,001,World,1 -foodnetwork.com,foodnetwork.com,inbound,001,World,0 -foolsubs.com,foolcs.com,inbound,001,World,0 -foolsubs.com,foolsubs.com,inbound,001,World,0 -footaction.com,footlocker.com,inbound,001,World,0 -footlocker.com,footlocker.com,inbound,001,World,0.000605 -forcemail.in,iaires.com,inbound,001,World,0 -foreseegame.com,iaires.com,inbound,001,World,0 -forever21.com,forever21.com,inbound,001,World,0 -fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,001,World,0 -fotffamily.com,fotffamily.com,inbound,001,World,0 -fotocasa.es,fotocasa.es,inbound,001,World,0 -fotolivro.com.br,fotolivro.com.br,inbound,001,World,0 -fotostrana.ru,fotocdn.net,inbound,001,World,3.4e-05 -foursquare.com,foursquare.com,inbound,001,World,1 -foxnews.com,foxnews.com,inbound,001,World,0.0139 -fpmailerbr.com,fpmailerbr.com,inbound,001,World,0 -fragrancenet.com,fragrancenet.com,inbound,001,World,0.000427 -francescas.com,bronto.com,inbound,001,World,0 -free-lance.ru,free-lance.ru,inbound,001,World,0 -free.fr,free.fr,inbound,001,World,0.984012 -free.fr,free.fr,outbound,001,World,6.9e-05 -freeadsmailer.com,zoothost.com,inbound,001,World,0 -freebeesafelist.com,zoothost.com,inbound,001,World,0 -freebizmag.com,delivery.net,inbound,001,World,0 -freebsd.org,freebsd.org,inbound,001,World,0.999835 -freecycle.org,freecycle.org,inbound,001,World,0.999927 -freedesktop.org,freedesktop.org,inbound,001,World,0 -freeflys.com,freeflys.com,inbound,001,World,0 -freelancer.com,freelancer.com,inbound,001,World,0 -freelancer.com,freelancernotify.com,inbound,001,World,0 -freelancer.com,getafreelancer.com,inbound,001,World,0 -freelists.org,iquest.net,inbound,001,World,0 -freelotto.com,plasmanetinc.com,inbound,001,World,0 -freemail.hu,freemail.hu,outbound,001,World,0 -freeml.com,gmo-media.jp,inbound,001,World,0 -freepeople.com,freepeople.com,inbound,001,World,0 -freesafelistking.com,zoothost.com,inbound,001,World,0 -freesafelistmailer.com,waters-advertising.com,inbound,001,World,0 -freshdesk.com,freshdesk.com,inbound,001,World,1 -freshers2015.com,secureserver.net,inbound,001,World,0 -freshlatesave.com,freshlatesave.com,inbound,001,World,1 -freshmail.pl,freshmail.pl,inbound,001,World,0 -fridays.com,fridays.com,inbound,001,World,0 -friskone.com,mailurja.com,inbound,001,World,0 -frk.com,frk.com,inbound,001,World,0.999995 -frontdoor.com,frontdoor.com,inbound,001,World,0 -frontgate-email.com,frontgate-email.com,inbound,001,World,0 -frontsight.com,frontsight.com,inbound,001,World,0 -frys.com,frys.com,inbound,001,World,0.00388 -frysmail.com,frysmail.com,inbound,001,World,0 -fspeletters.com,agorapub.co.uk,inbound,001,World,0 -ftchinese.com,ftchinese.com,inbound,001,World,0 -fubonshop.com,fubonshop.com,inbound,001,World,0 -fuckbooknet.net,infinitypersonals.com,inbound,001,World,0 -fuelrewards.com,britecast.com,inbound,001,World,0 -fundplaza.co.in,arrowsignindia.com,inbound,001,World,0 -fundplaza.in,fundplaza.in,inbound,001,World,0 -funonthenet.in,funonthenet.in,inbound,001,World,1 -futureshop.com,futureshop.com,inbound,001,World,0 -futurmailer.pt,futurmailer.pt,inbound,001,World,0 -gabbar.info,gabbar.info,inbound,001,World,1 -gaiaonline.com,gaiaonline.com,inbound,001,World,0 -gamecity.ne.jp,gamecity.ne.jp,inbound,001,World,0 -gamefly.com,gamefly.com,inbound,001,World,0.013381 -gamehouse.com,gamehouse.com,inbound,001,World,0 -gamingmails.com,gamingmails.com,inbound,001,World,0 -gap.com,gap.com,inbound,001,World,0 -gap.eu,gap.eu,inbound,001,World,0 -gapcanada.ca,gapcanada.ca,inbound,001,World,0 -garanti.com.tr,euromsg.net,inbound,001,World,0 -garanti.com.tr,garanti.com.tr,inbound,001,World,0.421936 -gardeningclubmail.co.uk,msgfocus.com,inbound,001,World,0 -garnethill-email.com,garnethill-email.com,inbound,001,World,0 -gaylordalert.com,gaylordalert.com,inbound,001,World,0 -gbyguess.com,guess.com,inbound,001,World,0.001787 -gcast.com.au,systemsserver.net,inbound,001,World,0 -gdtsuccess.com,groupdealtools.com,inbound,001,World,1 -geico.com,geico.com,inbound,001,World,0.096879 -gemoney.com,rsys1.com,inbound,001,World,0 -gene.com,roche.com,inbound,001,World,1 -generalmills.com,boxtops4education.com,inbound,001,World,0 -generalmills.com,pillsbury.com,inbound,001,World,0 -gentoo.org,gentoo.org,inbound,001,World,1 -geocaching.com,groundspeak.com,inbound,001,World,1 -geojit.com,geojit.com,inbound,001,World,0.203748 -get-me-jobs.com,get-me-jobs.com,inbound,001,World,0 -gethired.com,gethired.com,inbound,001,World,1 -getinbox.net,getinbox.net,inbound,001,World,0 -getitfree.us,getitfree.us,inbound,001,World,0 -getkeepsafe.com,getkeepsafe.com,inbound,001,World,1 -getmein.com,getmein.com,inbound,001,World,0 -getpaidsolutions.com,getpaidsolutions.com,inbound,001,World,1 -getpocket.com,bronto.com,inbound,001,World,0 -getresponse.com,getresponse.com,inbound,001,World,0 -gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,001,World,0 -ghin.com,ghinconnect.com,inbound,001,World,0 -ghup.in,mgenie.in,inbound,001,World,0 -giffgaff.com,giffgaff.com,inbound,001,World,0 -gillyhicks-email.com,abercrombie-email.com,inbound,001,World,0 -gilt.com,gilt.com,inbound,001,World,1e-06 -gilt.jp,gilt.jp,inbound,001,World,0 -github.com,github.com,inbound,001,World,1 -github.com,github.net,inbound,001,World,1 -github.com,postini.com,inbound,001,World,0.872186 -glassdoor.com,glassdoor.com,inbound,001,World,0.662834 -glasses.com,glasses.com,inbound,001,World,0 -gliq.com,gliq.com,inbound,001,World,0.99852 -globalmembersupport.com,globalmembersupport.com,inbound,001,World,0 -globalsafelist.com,globalsafelist.com,inbound,001,World,0 -globalsources.com,globalsources.com,inbound,001,World,0.007546 -globalspec.com,globalspec.com,inbound,001,World,0 -globaltestmarket.com,globaltestmarket.com,inbound,001,World,0 -globasemail.com,globasemail.com,inbound,001,World,0.951088 -globetel.com.ph,globetel.com.ph,inbound,001,World,1 -gmail.com,02.net,inbound,001,World,0.998007 -gmail.com,amazonaws.com,inbound,001,World,0.988441 -gmail.com,anteldata.net.uy,inbound,001,World,0.998653 -gmail.com,as13285.net,inbound,001,World,0.999943 -gmail.com,asianet.co.th,inbound,001,World,0.998473 -gmail.com,au-net.ne.jp,inbound,001,World,1 -gmail.com,bbox.fr,inbound,001,World,0.919868 -gmail.com,bbtec.net,inbound,001,World,1 -gmail.com,belgacom.be,inbound,001,World,0.822281 -gmail.com,bell.ca,inbound,001,World,0.992482 -gmail.com,bellsouth.net,inbound,001,World,0.9996 -gmail.com,bezeqint.net,inbound,001,World,0.974444 -gmail.com,bigpond.net.au,inbound,001,World,0.999907 -gmail.com,blackberry.com,inbound,001,World,0.996337 -gmail.com,bluewin.ch,inbound,001,World,0.9337 -gmail.com,brasiltelecom.net.br,inbound,001,World,0.99995 -gmail.com,btcentralplus.com,inbound,001,World,0.999969 -gmail.com,centurytel.net,inbound,001,World,0.999415 -gmail.com,cgocable.net,inbound,001,World,0.998291 -gmail.com,charter.com,inbound,001,World,0.999111 -gmail.com,chello.nl,inbound,001,World,0.999951 -gmail.com,claro.net.br,inbound,001,World,1 -gmail.com,comcast.net,inbound,001,World,0.999616 -gmail.com,comcastbusiness.net,inbound,001,World,0.985464 -gmail.com,cox.net,inbound,001,World,0.962636 -gmail.com,data-hotel.net,inbound,001,World,0.000605 -gmail.com,emailsrvr.com,inbound,001,World,1 -gmail.com,embarqhsd.net,inbound,001,World,0.999554 -gmail.com,fastwebnet.it,inbound,001,World,0.984708 -gmail.com,franchiseindia.com,inbound,001,World,1 -gmail.com,frontiernet.net,inbound,001,World,0.995233 -gmail.com,gvt.net.br,inbound,001,World,0.999513 -gmail.com,hinet.net,inbound,001,World,0.97312 -gmail.com,iinet.net.au,inbound,001,World,0.966984 -gmail.com,jazztel.es,inbound,001,World,0.999701 -gmail.com,lorexddns.net,inbound,001,World,0 -gmail.com,majesticmoneymailer.com,inbound,001,World,1 -gmail.com,mchsi.com,inbound,001,World,0.999951 -gmail.com,movistar.cl,inbound,001,World,0.999361 -gmail.com,mtnl.net.in,inbound,001,World,0.999527 -gmail.com,mycingular.net,inbound,001,World,0.999918 -gmail.com,myvzw.com,inbound,001,World,0.999889 -gmail.com,naukri.com,inbound,001,World,0.000998 -gmail.com,net24.it,inbound,001,World,0.999964 -gmail.com,netcabo.pt,inbound,001,World,0.998164 -gmail.com,netvigator.com,inbound,001,World,0.818637 -gmail.com,numericable.fr,inbound,001,World,0.999726 -gmail.com,ocn.ne.jp,inbound,001,World,0.99466 -gmail.com,ono.com,inbound,001,World,0.995991 -gmail.com,optonline.net,inbound,001,World,0.999776 -gmail.com,optusnet.com.au,inbound,001,World,0.992856 -gmail.com,orange.es,inbound,001,World,0.998743 -gmail.com,orange.fr,inbound,001,World,0 -gmail.com,otenet.gr,inbound,001,World,0.957964 -gmail.com,panda-world.ne.jp,inbound,001,World,1 -gmail.com,postini.com,inbound,001,World,0.776029 -gmail.com,proxad.net,inbound,001,World,0.998094 -gmail.com,qwest.net,inbound,001,World,0.997575 -gmail.com,rcn.com,inbound,001,World,0.986768 -gmail.com,rima-tde.net,inbound,001,World,0.99915 -gmail.com,rogers.com,inbound,001,World,0.999917 -gmail.com,rr.com,inbound,001,World,0.967896 -gmail.com,sbcglobal.net,inbound,001,World,0.998817 -gmail.com,secureserver.net,inbound,001,World,0.272513 -gmail.com,seed.net.tw,inbound,001,World,0.992252 -gmail.com,sfr.net,inbound,001,World,0.999878 -gmail.com,shawcable.net,inbound,001,World,0.999998 -gmail.com,singnet.com.sg,inbound,001,World,0.955461 -gmail.com,skybroadband.com,inbound,001,World,0.999854 -gmail.com,spcsdns.net,inbound,001,World,0.999998 -gmail.com,suddenlink.net,inbound,001,World,0.961594 -gmail.com,t-ipconnect.de,inbound,001,World,0.999841 -gmail.com,tdc.net,inbound,001,World,0.999591 -gmail.com,telecom.net.ar,inbound,001,World,0.999664 -gmail.com,telecomitalia.it,inbound,001,World,0.996998 -gmail.com,telekom.hu,inbound,001,World,0.999977 -gmail.com,telenet.be,inbound,001,World,1 -gmail.com,telepac.pt,inbound,001,World,0.999584 -gmail.com,telesp.net.br,inbound,001,World,0.999743 -gmail.com,telia.com,inbound,001,World,1 -gmail.com,telkomadsl.co.za,inbound,001,World,0.999989 -gmail.com,telus.com,inbound,001,World,1 -gmail.com,telus.net,inbound,001,World,0.974677 -gmail.com,threembb.co.uk,inbound,001,World,1 -gmail.com,tmodns.net,inbound,001,World,0.999979 -gmail.com,totbb.net,inbound,001,World,0.999876 -gmail.com,tpgi.com.au,inbound,001,World,0.999649 -gmail.com,tpnet.pl,inbound,001,World,0.999761 -gmail.com,veloxzone.com.br,inbound,001,World,0.999969 -gmail.com,verizon.net,inbound,001,World,0.990214 -gmail.com,videotron.ca,inbound,001,World,0.967213 -gmail.com,virginm.net,inbound,001,World,0.996611 -gmail.com,vodacom.co.za,inbound,001,World,1 -gmail.com,vodafone-ip.de,inbound,001,World,1 -gmail.com,vodafone.pt,inbound,001,World,0.999563 -gmail.com,vodafonedsl.it,inbound,001,World,0.999006 -gmail.com,vtr.net,inbound,001,World,0.999072 -gmail.com,wanadoo.fr,inbound,001,World,0.999753 -gmail.com,websitewelcome.com,inbound,001,World,1 -gmail.com,wideopenwest.com,inbound,001,World,0.999729 -gmail.com,windstream.net,inbound,001,World,0.951847 -gmail.com,yahoo.{...},inbound,001,World,0.999147 -gmail.com,ziggo.nl,inbound,001,World,1 -gmail.com,zoothost.com,inbound,001,World,0.033858 -gmo.jp,gmo-media.jp,inbound,001,World,0 -gmoes.jp,gmoes.jp,inbound,001,World,0 -gmsend.com,gmsend.com,inbound,001,World,0 -gmt.ne.jp,gmt.ne.jp,inbound,001,World,0 -gmx.de,gmx.net,inbound,001,World,1 -gmx.de,gmx.net,outbound,001,World,1 -gmx.net,gmx.net,inbound,001,World,1 -gnavi.co.jp,gnavi.co.jp,inbound,001,World,0.002441 -go.com,starwave.com,inbound,001,World,0.006276 -go4worldbusiness.com,go4worldbusiness.com,inbound,001,World,1 -goalunited.org,ccmdcampaigns.net,inbound,001,World,0 -gob.ar,gob.ar,inbound,001,World,0.159673 -gob.ec,gob.ec,inbound,001,World,0.618006 -godaddy.com,secureserver.net,inbound,001,World,0 -godtubemail.com,godtubemail.com,inbound,001,World,0 -godvinemail.com,godvinemail.com,inbound,001,World,0 -gog.com,gog.com,inbound,001,World,0 -gogecapital.com,rsys1.com,inbound,001,World,0 -gogroopie.com,gogroopie.com,inbound,001,World,0.000283 -gohappy.com.tw,gohappy.com.tw,inbound,001,World,0.000104 -goldenbrands.gr,goldenbrands.gr,inbound,001,World,1 -goldenline.pl,goldenline.pl,inbound,001,World,1 -goldenopsafelist.com,zoothost.com,inbound,001,World,0 -goldstar.com,goldstar.com,inbound,001,World,1 -golfmnb.com,golfmnb.com,inbound,001,World,0 -golfnow.com,email-golfnow.com,inbound,001,World,0 -gomaji.com,gomaji.com,inbound,001,World,0 -goodgame.com,emsmtp.com,inbound,001,World,0 -goodlife.pt,emv8.com,inbound,001,World,0 -google.com,postini.com,inbound,001,World,0.703706 -googlegroups.com,postini.com,inbound,001,World,0.674075 -googlemail.com,t-ipconnect.de,inbound,001,World,0.999957 -gop.com,gop.com,inbound,001,World,0 -gopusamedia.com,gopusamedia.com,inbound,001,World,0 -govdelivery.com,govdelivery.com,inbound,001,World,0 -govdelivery.com,postini.com,inbound,001,World,0.089736 -governmentjobs.com,governmentjobs.com,inbound,001,World,0 -gpmailer.com.br,parperfeito.com,inbound,001,World,0 -grabone-mail-ie.com,grabone-mail-ie.com,inbound,001,World,0 -grabone-mail.com,grabone-mail.com,inbound,001,World,0 -grassrootsaction.com,grassfire.net,inbound,001,World,0 -gratka.pl,gratka.pl,inbound,001,World,0 -greatergood.com,greatergood.com,inbound,001,World,0 -gree.jp,gree.jp,inbound,001,World,0 -grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,001,World,0 -groopdealz.com,groopdealz.com,inbound,001,World,1 -groupalia.es,groupalia.es,inbound,001,World,0 -groupalia.it,groupalia.it,inbound,001,World,0 -groupon.jp,data-hotel.net,inbound,001,World,1e-06 -groupon.{...},chtah.net,inbound,001,World,0 -groupon.{...},groupon.{...},inbound,001,World,0.989844 -groupon.{...},postini.com,inbound,001,World,0.887392 -grouponmail.{...},grouponmail.{...},inbound,001,World,0 -grubhubmail.com,grubhubmail.com,inbound,001,World,0 -grupanya.com,euromsg.net,inbound,001,World,0 -grupos.com.br,grupos.com.br,inbound,001,World,0 -gtbank.com,gtbank.com,inbound,001,World,0.056121 -guess.ca,guess.com,inbound,001,World,0.000807 -guess.com,guess.com,inbound,001,World,0.003805 -guessfactory.com,guess.com,inbound,001,World,0.00164 -gumtree.com,marktplaats.nl,inbound,001,World,0 -gumtree.com.au,kijiji.com,inbound,001,World,0 -gunosy.com,gunosy.com,inbound,001,World,0.998682 -guruin.info,guru.net.in,inbound,001,World,1 -gurunavi.jp,gurunavi.jp,inbound,001,World,0 -gustazos.com,cityoferta.com,inbound,001,World,1 -gymglish.com,gymglish.com,inbound,001,World,0.000788 -habitaclia.com,splio.es,inbound,001,World,0.951406 -hallmark.com,hallmark.com,inbound,001,World,0 -hannaandersson.com,hannaandersson.com,inbound,001,World,0.013533 -harborfreightemail.com,harborfreightemail.com,inbound,001,World,0 -harristeetermail.com,harristeetermail.com,inbound,001,World,0.99673 -harvard.edu,harvard.edu,inbound,001,World,0.1751 -haskell.org,haskell.org,inbound,001,World,0.000703 -hautelook.com,hautelook.com,inbound,001,World,0.000459 -hayneedle.com,hayneedle.com,inbound,001,World,0 -hazteoir.org,hazteoir.org,inbound,001,World,0.31478 -hdfcbank.com,powerelay.com,inbound,001,World,1 -hdfcbank.net,powerelay.com,inbound,001,World,1 -hdfcbank.net,quickvmail.com,inbound,001,World,0 -helpareporter.net,helpareporter.com,inbound,001,World,0 -hepsiburada.com,euromsg.net,inbound,001,World,0 -herbalifemail.com,herbalifemail.com,inbound,001,World,0 -herculist.com,herculist.com,inbound,001,World,0 -heteml.jp,heteml.jp,inbound,001,World,0.950766 -hgtv.com,hgtv.com,inbound,001,World,0 -hh.ru,hh.ru,inbound,001,World,0.996236 -hhgreggemail.com,hhgreggemail.com,inbound,001,World,0 -hilton.com,hiltonemail.com,inbound,001,World,0 -hinet.net,hinet.net,inbound,001,World,0.007093 -hinet.net,hinet.net,outbound,001,World,0.00565 -hipchat.com,hipchat.com,inbound,001,World,1 -hipmunk.com,hipmunk.com,inbound,001,World,0 -hispavista.com,hispavista.com,inbound,001,World,0 -hln.be,persgroep-ops.net,inbound,001,World,0 -hm-f.jp,hm-f.jp,inbound,001,World,0 -hobsonsmail.com,hobsonsmail.com,inbound,001,World,0 -hollister-email.com,abercrombie-email.com,inbound,001,World,0 -home.ne.jp,zaq.ne.jp,inbound,001,World,9e-06 -homeaway.com,haspf.com,inbound,001,World,0 -homebaselife.com,ec-cluster.com,inbound,001,World,0 -homechoice.co.za,homechoice.co.za,inbound,001,World,0 -homedecorators.com,homedecorators.com,inbound,001,World,0 -homedepot.com,homedepot.com,inbound,001,World,1 -homedepotemail.com,homedepotemail.com,inbound,001,World,0 -honto.jp,honto.jp,inbound,001,World,0 -hootsuite.com,hootsuite.com,inbound,001,World,1 -horchowemail.com,horchowemail.com,inbound,001,World,0 -horoscope.com,center.com,inbound,001,World,2e-06 -hostelworld.com,bronto.com,inbound,001,World,0 -hostgator.com,hostgator.com,inbound,001,World,0.976131 -hostgator.com,websitewelcome.com,inbound,001,World,0.999822 -hotel.de,emp-mail.de,inbound,001,World,0 -hotels.com,hotels.com,inbound,001,World,0 -hotelurbano.com.br,allin.com.br,inbound,001,World,0 -hotmail.{...},hotmail.{...},inbound,001,World,0.999968 -hotmail.{...},hotmail.{...},outbound,001,World,1 -hotmail.{...},postini.com,inbound,001,World,0.837967 -hotornot.com,monopost.com,inbound,001,World,0.99472 -hotschedules.com,hotschedules.com,inbound,001,World,0 -hotspotmailer.com,hotspotmailer.com,inbound,001,World,1 -hotukdeals.com,hotukdeals.com,inbound,001,World,1 -hotwire.com,hotwire.com,inbound,001,World,0 -house.gov,house.gov,inbound,001,World,0.999966 -houseoffraser.co.uk,houseoffraser.co.uk,inbound,001,World,0 -houzz.com,houzz.com,inbound,001,World,1 -hp.com,hp.com,inbound,001,World,0.202841 -hpnotifier.nl,hpnotifier.nl,inbound,001,World,0 -hsbc.co.in,hsbc.com.hk,inbound,001,World,1 -hsbc.com.hk,hsbc.com.hk,inbound,001,World,1 -hsn.com,hsn.com,inbound,001,World,0 -htcampusmailer.com,eccluster.com,inbound,001,World,0 -hubspot.com,hubspot.com,inbound,001,World,1 -huinforma.com.br,huinforma.com.br,inbound,001,World,0 -hulumail.com,hulumail.com,inbound,001,World,0 -hungry-girl.com,hungry-girl.com,inbound,001,World,0 -hungryhouse.co.uk,mxmfb.com,inbound,001,World,0 -huntington.com,huntington.com,inbound,001,World,0.987351 -i-part.com.tw,i-part.com.tw,inbound,001,World,0.001865 -i-say.com,ipsos-interactive.com,inbound,001,World,1 -iamlgnd2.com,iamlgnd2.com,inbound,001,World,1 -ibm.com,ibm.com,inbound,001,World,0.94413 -ibps.in,sify.net,inbound,001,World,0 -ibpsorg.org,sify.net,inbound,001,World,0 -ibsys.com,ibsys.com,inbound,001,World,9e-05 -icbc.com.ar,clickexperts.net,inbound,001,World,0 -icbc.com.ar,standardbank.com.ar,inbound,001,World,0 -icelandmail.co.uk,emsg-live.co.uk,inbound,001,World,0 -icicibank.com,icicibank.com,inbound,001,World,0.009835 -icicisecurities.com,icicibank.com,inbound,001,World,0.000803 -icims.com,icims.com,inbound,001,World,0.999939 -icloud.com,apple.com,inbound,001,World,1 -icloud.com,icloud.com,outbound,001,World,1 -icloud.com,mac.com,inbound,001,World,1 -icloud.com,me.com,inbound,001,World,0.999995 -icors.org,lsoft.us,inbound,001,World,0 -icpbounce.com,icpbounce.com,inbound,001,World,0 -idc.email,nmsrv.com,inbound,001,World,1 -idealista.com,idealista.com,inbound,001,World,0.001627 -ideascost.com,ramcorp.in,inbound,001,World,0 -idgconnect-resources.com,idgconnect-resources.com,inbound,001,World,0 -ieee.org,ieee.org,inbound,001,World,0.999912 -ifttt.com,ifttt.com,inbound,001,World,1 -ig.com.br,ig.com.br,inbound,001,World,0 -ig.com.br,ig.com.br,outbound,001,World,0 -ign.com,ign.com,inbound,001,World,0 -ignitionsender.com,ignitionsender.com,inbound,001,World,0 -igot-mails.com,zoothost.com,inbound,001,World,0 -iheart.com,iheart.com,inbound,001,World,0 -iimjobs.com,iimjobs.com,inbound,001,World,1 -ikmultimedianews.com,ikmultimedianews.com,inbound,001,World,0.993319 -illinois.edu,illinois.edu,inbound,001,World,0.866481 -imageshost.ca,imageshost.ca,inbound,001,World,0 -imakenews.net,imakenews.com,inbound,001,World,0 -imi.ne.jp,lifemedia.jp,inbound,001,World,0 -immobilienscout24.de,immobilienscout24.de,inbound,001,World,1 -imo.im,imo.im,inbound,001,World,1 -imodules.com,imodules.com,inbound,001,World,0 -imvu.com,imvu.com,inbound,001,World,7e-06 -in-boxpays.com,in-boxpays.com,inbound,001,World,0 -inboxair.com,inboxair.com,inbound,001,World,0 -inboxdollars.com,inboxdollars.com,inbound,001,World,0 -inboxfirst.com,inboxfirst.com,inbound,001,World,0 -inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,001,World,0.999877 -inboxpays.com,inboxpays.com,inbound,001,World,0 -inboxpounds.co.uk,inboxpounds.co.uk,inbound,001,World,0 -indeed.com,indeed.com,inbound,001,World,0.000121 -indeedemail.com,indeedemail.com,inbound,001,World,0 -independentlivingbullion.com,independentlivingbullion.com,inbound,001,World,0 -indiamart.com,indiamart.com,inbound,001,World,1 -indiaproperty.com,indiaproperty.com,inbound,001,World,0 -indiatimes.com,speakingtree.in,inbound,001,World,0 -indiatimeshop.com,sendpal.in,inbound,001,World,0 -indieroyale.com,desura.com,inbound,001,World,1 -infibeam.com,eccluster.com,inbound,001,World,0 -infobradesco.com.br,infobradesco.com.br,inbound,001,World,0 -infoempleo.com,infoempleo.com,inbound,001,World,0 -infojobs.com.br,anuntis.com,inbound,001,World,0 -infojobs.it,infojobs.it,inbound,001,World,0 -infojobs.net,infojobs.net,inbound,001,World,0 -infomoney.com.br,infomoney.com.br,inbound,001,World,1 -infopanel.jp,mailds.jp,inbound,001,World,0 -infopraca.pl,careesma.com,inbound,001,World,0 -informz.net,informz.net,inbound,001,World,0 -infos-micromania.com,infos-micromania.com,inbound,001,World,0 -infosephora.com,splio.com,inbound,001,World,0.962167 -infoworld.com,infoworld.com,inbound,001,World,0 -infradead.org,infradead.org,inbound,001,World,1 -infusionmail.com,infusionmail.com,inbound,001,World,0 -ingdirect.es,ingdirect.es,inbound,001,World,1 -inman.com,inman.com,inbound,001,World,0 -inmotionhosting.com,inmotionhosting.com,inbound,001,World,0.990051 -innovyx.net,innovyx.net,inbound,001,World,0 -ino.com,ino.com,inbound,001,World,0.999981 -insidehook.com,sailthru.com,inbound,001,World,0 -instagram.com,facebook.com,inbound,001,World,1 -instantprofitlist.com,screenshotads.com,inbound,001,World,0 -intage.co.jp,intage.co.jp,inbound,001,World,0.00557 -inter-chat.com,inter-chat.com,inbound,001,World,0 -interac.ca,certapay.com,inbound,001,World,0 -interactivebrokers.com,interactivebrokers.com,inbound,001,World,1 -interactiverealtyservices.com,interactiverealtyservices.com,inbound,001,World,0 -intercom.io,mailgun.info,inbound,001,World,1 -interdatesa.com,fagms.net,inbound,001,World,0 -interealty.net,interealty.net,inbound,001,World,1 -internations.org,internations.org,inbound,001,World,1 -interweave.com,interweave.com,inbound,001,World,0 -interwell.gr,interwell.gr,inbound,001,World,0 -intliv2.net,internationalliving.com,inbound,001,World,0 -intuit.com,intuit.com,inbound,001,World,0.89114 -invalidemail.com,taleo.net,inbound,001,World,1 -investopedia.com,vclk.net,inbound,001,World,0.999999 -investorplace.com,investorplace.com,inbound,001,World,0.995093 -inx1and1.de,1and1.com,inbound,001,World,1 -inxserver.com,inxserver.de,inbound,001,World,0.952863 -inxserver.de,inxserver.de,inbound,001,World,0.980838 -ipcmedia.co.uk,ipcmedia.co.uk,inbound,001,World,0 -iqelite.com,iqelite.com,inbound,001,World,0 -irctcshopping.com,chtah.net,inbound,001,World,0 -iridium.com,iridium.com,inbound,001,World,0.997882 -isbank.com.tr,isbank.com.tr,inbound,001,World,0.009655 -isendservice.com.br,isendservice.com.br,inbound,001,World,0 -itau-unibanco.com.br,itau.com.br,inbound,001,World,0 -itms.in.ua,itms.in.ua,inbound,001,World,0 -itsmyascent.com,itsmyascent.com,inbound,001,World,0 -ittoolbox.com,ittoolbox.com,inbound,001,World,0 -ittoolbox.com,toolbox.com,inbound,001,World,0 -itunes.com,apple.com,inbound,001,World,0.082921 -itwhitepapers.com,itwhitepapers.com,inbound,001,World,0 -iwantoneofthose.com,thehut.com,inbound,001,World,0 -ixs1.net,ixs1.net,inbound,001,World,0.005131 -jackwills.com,jackwills.com,inbound,001,World,0 -jalag.de,jalag.de,inbound,001,World,1 -jane.com,jane.com,inbound,001,World,1 -jango.com,jango.com,inbound,001,World,0 -jared.com,jared.com,inbound,001,World,0 -jcity.com,jcity.com,inbound,001,World,0 -jcpenney.com,jcpenney.com,inbound,001,World,9e-06 -jdate.com,postdirect.com,inbound,001,World,0 -jeevansathi.com,jeevansathi.com,inbound,001,World,0 -jetprivilege.com,jetprivilege.com,inbound,001,World,0 -jetsetter.com,smartertravelmedia.com,inbound,001,World,0.041362 -jeuxvideo.com,jeuxvideo.com,inbound,001,World,0.148921 -jeweloscoemail.com,email-mywebgrocer2.com,inbound,001,World,0 -jibjab.com,storybots.com,inbound,001,World,0 -jira.com,uc-inf.net,inbound,001,World,1 -jiscmail.ac.uk,lsoft.se,inbound,001,World,0 -joann-mail.com,joann-mail.com,inbound,001,World,0 -jobinsider.com,jobinsider.com,inbound,001,World,0 -jobinthailand.com,jobinthailand.com,inbound,001,World,1 -jobisjob.com,jobisjob.com,inbound,001,World,0 -jobmaster.co.il,jobmaster1.co.il,inbound,001,World,0 -jobomas.com,jobomas.com,inbound,001,World,1 -jobrapidoalert.com,jobrapidoalert.com,inbound,001,World,0 -jobs2web.com,ondemand.com,inbound,001,World,1 -jobscentral.com.sg,mailgun.net,inbound,001,World,1 -jobsdbalert.co.id,jobsdbalert.co.id,inbound,001,World,0 -jobsdbalert.com,jobsdbalert.com,inbound,001,World,0 -jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,001,World,0 -jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,001,World,0 -jobserve.com,jobserve.com,inbound,001,World,0 -jobsindubai.com,jobsindubai.ca,inbound,001,World,0 -jobsite.co.uk,jobsite.co.uk,inbound,001,World,0.000509 -jobson.com,jobsonmail.com,inbound,001,World,0 -jobsradar.com,jobsradar.com,inbound,001,World,0 -jobstreet.com,jobstreet.com,inbound,001,World,0 -jockeycomfort.com,jockeycomfort.com,inbound,001,World,0 -johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,001,World,0 -jomashop.com,lstrk.net,inbound,001,World,1 -joobmailer.com,joobmailer.com,inbound,001,World,0 -josbank.com,josbank.com,inbound,001,World,0 -joshin.co.jp,joshin.co.jp,inbound,001,World,8e-06 -jossandmain.com,jossandmain.com,inbound,001,World,0 -jpcycles.com,jpcycles.com,inbound,001,World,0.001716 -jtv.com,jtv.com,inbound,001,World,0 -jumia.com.ng,fagms.de,inbound,001,World,0 -jungleerummy.com,jungleerummy.com,inbound,001,World,1 -juno.com,untd.com,inbound,001,World,0 -juno.com,untd.com,outbound,001,World,0 -jusbrasil.com.br,jusbrasil.com.br,inbound,001,World,0 -just-eat.co.uk,ec-cluster.com,inbound,001,World,0 -justclick.ru,justclick.ru,inbound,001,World,0 -justdial.com,iaires.com,inbound,001,World,0 -justdial.com,mailurja.com,inbound,001,World,0 -justfab.com,bronto.com,inbound,001,World,0 -justfab.fr,bronto.com,inbound,001,World,0 -k1speed.com,k1speed.com,inbound,001,World,1 -kagoya.net,kagoya.net,inbound,001,World,0.013097 -kalunga.com.br,kalunga.com.br,inbound,001,World,0 -karvy.com,karvy.com,inbound,001,World,0.045186 -kasikornbank.com,kasikornbank.com,inbound,001,World,0 -kaskusnetworks.com,kaskus.com,inbound,001,World,0 -kay.com,kay.com,inbound,001,World,0 -keek.com,keek.com,inbound,001,World,1 -kernel.org,kernel.org,inbound,001,World,0 -kgbdeals.co.uk,email1-kgbdeals.com,inbound,001,World,0 -kgstores.com,kgstores.com,inbound,001,World,0 -kiabi.com,dms-02.net,inbound,001,World,0 -kickstarter.com,kickstarter.com,inbound,001,World,1 -kidsfootlocker.com,footlocker.com,inbound,001,World,0 -kijiji.ca,kijiji.com,inbound,001,World,0 -kik.com,kik.com,inbound,001,World,1 -kimblegroup.com,kimblegroup.com,inbound,001,World,1 -kintera.com,kintera.com,inbound,001,World,0 -kismia.com,kismia.com,inbound,001,World,1 -kiwari.com,kiwari.com,inbound,001,World,9e-06 -klaviyomail.com,klaviyomail.com,inbound,001,World,1 -kliksa.net,euromsg.net,inbound,001,World,0 -kliktoday.com,kliktoday.com,inbound,001,World,0 -klm-mail.com,klm-mail.com,inbound,001,World,0 -klove.com,emfbroadcasting.com,inbound,001,World,0 -kohls.com,kohls.com,inbound,001,World,0 -komando.com,komando.com,inbound,001,World,0 -kongregate.com,kongregate.com,inbound,001,World,0 -kotak.com,kotak.com,inbound,001,World,0.115651 -kp.org,kp.org,inbound,001,World,0.999975 -krogermail.com,bigfootinteractive.com,inbound,001,World,0 -krs.bz,tricorn.net,inbound,001,World,0 -kubra.com,kubra.com,inbound,001,World,1.8e-05 -kundenserver.de,kundenserver.de,inbound,001,World,1 -kvbmail.com,kvbmail.com,inbound,001,World,0 -la-meteo-mail.fr,splio.com,inbound,001,World,1 -laaptuemail.com,laaptuemail.com,inbound,001,World,0 -lakewoodchurch.com,lakewoodchurch.com,inbound,001,World,0 -lancers.jp,lancers.jp,inbound,001,World,0 -landmarketingmailer.com,zoothost.com,inbound,001,World,0 -landofnod.com,landofnod.com,inbound,001,World,0.002525 -landsend.com,email-landsend.com,inbound,001,World,0 -landsend.com,postdirect.com,inbound,001,World,0 -languagepod101.com,eclient10.com,inbound,001,World,0 -languagepod101.com,eddlvr.com,inbound,001,World,0 -languagepod101.com,ednwsltr3.com,inbound,001,World,0 -languagepod101.com,ednwsltr8.com,inbound,001,World,0 -languagepod101.com,emaildirect.net,inbound,001,World,0 -laposte.net,laposte.net,inbound,001,World,0.271722 -laposte.net,laposte.net,outbound,001,World,0 -laptuinvite.com,laptuinvite.com,inbound,001,World,0 -laredoute.fr,laredoute.fr,inbound,001,World,0 -lasenza.com,lasenza.com,inbound,001,World,0 -lastcallemail.com,lastcallemail.com,inbound,001,World,0 -lastminute.com,lastminute.com,inbound,001,World,0.00831 -laterooms.com,laterooms.com,inbound,001,World,0.014746 -latimes.com,latimes.com,inbound,001,World,0 -lauraashley.com,lauraashley.com,inbound,001,World,0 -lazerhits.com,lazerhits.com,inbound,001,World,1 -leadercontato.com.br,leadercontato.com.br,inbound,001,World,0 -leboncoin.fr,leboncoin.fr,inbound,001,World,0 -lefigaro.fr,splio.com,inbound,001,World,1 -leftlanesports.com,auspient.com,inbound,001,World,0.00705 -leftlanesports.com,leftlanesports.com,inbound,001,World,0 -legalshieldassociate.com,legalshield.com,inbound,001,World,0 -lelong.my,lelong.com.my,inbound,001,World,1 -lelong.my,lelong.net.my,inbound,001,World,1 -lemonde.fr,lemonde.fr,inbound,001,World,0.000111 -leparisien.fr,leparisien.fr,inbound,001,World,0 -lexico.com,lexico.com,inbound,001,World,0 -lexpress.fr,bp06.net,inbound,001,World,0 -libero.it,libero.it,inbound,001,World,0.00024 -libero.it,libero.it,outbound,001,World,0 -life360.com,life360.com,inbound,001,World,1 -lifecare-news.com,email-lifecare.com,inbound,001,World,0 -lifecooler.com,1-hostingservice.com,inbound,001,World,0 -lifemiles.com,bigfootinteractive.com,inbound,001,World,0 -lifescript.com,ilinkmd.com,inbound,001,World,0 -lindenlab.com,lindenlab.com,inbound,001,World,0.999444 -line.me,naver.com,inbound,001,World,1 -line6.com,line6.com,inbound,001,World,2.7e-05 -linkedin.com,linkedin.com,inbound,001,World,0.998882 -linkedin.com,postini.com,inbound,001,World,0.835872 -linkedin.com,yahoo.{...},inbound,001,World,0.999927 -linkshare.com,linksynergy.com,inbound,001,World,0 -liquidation.com,liquidation.com,inbound,001,World,6e-06 -listadventure.com,adlabsinc.com,inbound,001,World,0 -listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,001,World,0.00023 -listeneremail.net,listeneremail.net,inbound,001,World,0 -listia.com,listia.com,inbound,001,World,1 -listjoe.com,adlabsinc.com,inbound,001,World,0 -listnerds.com,listnerds.com,inbound,001,World,0 -listreturn.com,zoothost.com,inbound,001,World,0 -listserve.com,listserve.com,inbound,001,World,0 -listvolta.com,listvolta.com,inbound,001,World,0 -listwire.com,listwire.com,inbound,001,World,0 -litres.ru,litres.ru,inbound,001,World,1 -live.{...},hotmail.{...},inbound,001,World,0.999954 -live.{...},hotmail.{...},outbound,001,World,1 -livedoor.com,livedoor.com,inbound,001,World,0 -livefyre.com,andbit.net,inbound,001,World,1 -livejournal.com,livejournal.com,inbound,001,World,0 -livemailservice.com,livemailservice.com,inbound,001,World,0 -livenation.com,exacttarget.com,inbound,001,World,0 -livescribe.com,bronto.com,inbound,001,World,0 -livingsocial.com,livingsocial.com,inbound,001,World,0 -livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,001,World,0 -lmlmgv.com.br,gvarev.com.br,inbound,001,World,0 -localhires.com,localhires.com,inbound,001,World,1 -loccitane.com,neolane.net,inbound,001,World,0 -loft.com,anntaylor.com,inbound,001,World,0 -logentries.com,logentries.com,inbound,001,World,0 -logitech.com,dvsops.com,inbound,001,World,0 -logmein.com,logmein.com,inbound,001,World,0.031467 -lojasmarisa.com.br,lojasmarisa.com.br,inbound,001,World,0 -lolsolos.com,ultimateadsites.net,inbound,001,World,0.999996 -lombardipublishing.com,lombardipublishing.com,inbound,001,World,0 -lonelywifehookup.com,iverificationsystems.com,inbound,001,World,0 -lookout.com,lookout.com,inbound,001,World,1 -lordandtaylor.com,lordandtaylor.com,inbound,001,World,0 -loveaholics.com,ropot.net,inbound,001,World,0 -lovelywholesale.com,lovelywholesale.com,inbound,001,World,1 -loveplanet.ru,pochta.ru,inbound,001,World,0.640035 -lowcostholidays.co.uk,communicatoremail.com,inbound,001,World,0 -lrsmail.com,lrsmail.com,inbound,001,World,0 -lsi.com,postini.com,inbound,001,World,0.981121 -lt02.net,listrak.com,inbound,001,World,1 -lt02.net,lstrk.net,inbound,001,World,1 -ltdcommodities.com,ltdcomm.net,inbound,001,World,0 -lua.org,pepperfish.net,inbound,001,World,1 -luckymag.com,mkt4500.com,inbound,001,World,0 -ludokados.com,ludokado.com,inbound,001,World,0 -lulu.com,bronto.com,inbound,001,World,0 -lulus.com,lstrk.net,inbound,001,World,1 -lumosity.com,lumosity.com,inbound,001,World,1 -luxa.jp,luxa.jp,inbound,001,World,0 -lynxmail.in,iaires.com,inbound,001,World,0 -lyris.net,lyris.net,inbound,001,World,0 -lyst.com,lyst.com,inbound,001,World,1 -m1e.net,m1e.net,inbound,001,World,0.000342 -m3.com,m3.com,inbound,001,World,0 -mac.com,icloud.com,outbound,001,World,1 -mac.com,mac.com,inbound,001,World,1 -maccosmetics.com,esteelauder.com,inbound,001,World,0 -macromill.com,macromill.com,inbound,001,World,1e-06 -macupdate.com,mailgun.info,inbound,001,World,1 -macys.com,macys.com,inbound,001,World,0 -madmels.info,ultimateadsites.net,inbound,001,World,1 -madmimi.com,madmimi.com,inbound,001,World,0 -mag2.com,tandem-m.com,inbound,001,World,0 -magicbricks.com,tbsl.in,inbound,001,World,0 -magicjack.com,magicjack.com,inbound,001,World,1 -magix.net,magix.net,inbound,001,World,0.673176 -magnetdev.com,magnetmail.net,inbound,001,World,0 -mail-backcountry.com,email-bcmarketing.com,inbound,001,World,0 -mail-boss.com,mail-boss.com,inbound,001,World,0 -mail-cdiscount.com,mail-cdiscount.com,inbound,001,World,0 -mail-mbank.pl,mail-mbank.pl,inbound,001,World,0 -mail-route.com,mail-route.com,inbound,001,World,0 -mail-thestreet.com,mail-thestreet.com,inbound,001,World,0 -mail.mil,mail.mil,inbound,001,World,0 -mail.ru,mail.ru,inbound,001,World,0.987506 -mail.ru,mail.ru,outbound,001,World,0.00674 -mailaccurate.com,mgenie.in,inbound,001,World,0 -mailchimp.com,mailchimp.com,inbound,001,World,0.967415 -maileclipse.com,emce2.in,inbound,001,World,0 -mailengine1.com,mailengine1.com,inbound,001,World,0 -mailer-service.de,mailer-service.de,inbound,001,World,4.9e-05 -mailer4u.in,elabs10.com,inbound,001,World,0 -mailersend.com,mailersend.com,inbound,001,World,0 -mailfacil.com.br,md02.com,inbound,001,World,0 -mailfeast.com,mgenie.in,inbound,001,World,0 -mailgun.org,mailgun.info,inbound,001,World,1 -mailgun.org,mailgun.net,inbound,001,World,1 -mailgun.org,mailgun.us,inbound,001,World,1 -mailing-list.it,mailing-list.it,inbound,001,World,0 -mailingathome.net,mailingathome.net,inbound,001,World,0.999995 -mailjayde.com,mailjayde.com,inbound,001,World,0 -mailjet.com,mailjet.com,inbound,001,World,0.436025 -mailmachine1050.com,mailmachine1050.com,inbound,001,World,0 -mailmailmail.net,mailmailmail.net,inbound,001,World,0 -mailoct.in,tcmailer14.in,inbound,001,World,0 -mailoct1.in,mailoct1.in,inbound,001,World,0 -mailoct1.in,myntramail2.in,inbound,001,World,0 -mailorama.fr,mailorama.fr,inbound,001,World,0 -mailplus.nl,brightbase.net,inbound,001,World,1 -mailpost.in,iaires.com,inbound,001,World,0 -mailpv.net,pvmailer.net,inbound,001,World,1 -mailquant.com,iaires.com,inbound,001,World,0 -mailsend1.com,mailsend6.com,inbound,001,World,0 -mailsender.com.br,mailsender.com.br,inbound,001,World,0 -maisonsdumonde.com,bp06.net,inbound,001,World,0 -makro.nl,srv2.de,inbound,001,World,0.88256 -manager.com.br,manager.com.br,inbound,001,World,0 -mandrillapp.com,backpage.com,inbound,001,World,1 -mandrillapp.com,mandrillapp.com,inbound,001,World,1 -mandrillapp.com,mcsignup.com,inbound,001,World,1 -mandrillapp.com,myjobhelperalerts.com,inbound,001,World,1 -mango.com,emstechnology2.net,inbound,001,World,0 -manipal.edu,iaires.com,inbound,001,World,0 -manta.com,exacttarget.com,inbound,001,World,0 -mapfre.com,emv5.com,inbound,001,World,0 -mar0.net,mar0.net,inbound,001,World,0.976409 -marcustheatres.com,movio.co,inbound,001,World,0 -markandgraham.com,markandgraham.com,inbound,001,World,0 -markavip.com,markavip.com,inbound,001,World,0 -marketer-safelist.com,jsalfianmarketing.com,inbound,001,World,1 -marketinghq.net,elabs8.com,inbound,001,World,0 -marketingprofs.com,marketingprofs.com,inbound,001,World,0.004412 -marketingstudio.com,marketingstudio.com,inbound,001,World,0 -marksandspencer.com,marksandspencer.com,inbound,001,World,0 -marktplaats.nl,marktplaats.nl,inbound,001,World,0 -marlboro.com,marlboro.com,inbound,001,World,0 -maropost.com,biotrustnews.com,inbound,001,World,0 -maropost.com,mailing-truthaboutabs.com,inbound,001,World,0 -maropost.com,maropost.com,inbound,001,World,0 -maropost.com,mp2200.com,inbound,001,World,0 -maropost.com,mp2201.com,inbound,001,World,0 -maropost.com,survivallife.com,inbound,001,World,0 -marykay.com,marykay.com,inbound,001,World,0 -masivapp.com,masivapp.com,inbound,001,World,1 -massageenvyclinics.com,massageenvyclinics.com,inbound,001,World,0 -masterbase.com,masterbase.com,inbound,001,World,0 -mastercard-email.com,mastercard-email.com,inbound,001,World,0 -match.com,match.com,inbound,001,World,0 -matchwereld.nl,matchwereld.nl,inbound,001,World,0 -mate1.net,mate1.net,inbound,001,World,0 -matrixemailer.com,matrixemailer.com,inbound,001,World,0 -maxpark.com,gidepark.ru,inbound,001,World,1 -mbga.jp,mbga.jp,inbound,001,World,0 -mbna.co.uk,ec-cluster.com,inbound,001,World,0 -mbounces.com,emdbms.com,inbound,001,World,0 -mbstrm.com,mobilestorm.com,inbound,001,World,0 -mcafee.com,mcafee.com,inbound,001,World,0.963535 -mcarthurglen.com,mcarthurglen.com,inbound,001,World,0 -mcdlv.net,mcdlv.net,inbound,001,World,0 -mcdlv.net,postini.com,inbound,001,World,0.082163 -mckinsey.com,bigfootinteractive.com,inbound,001,World,0 -mcsv.net,mcsv.net,inbound,001,World,0 -mcsv.net,postini.com,inbound,001,World,0.101883 -mdirector.com,mdrctr.com,inbound,001,World,0 -mdlinx.com,mdlinx.com,inbound,001,World,0 -me.com,icloud.com,outbound,001,World,1 -me.com,mac.com,inbound,001,World,1 -mec.gov.br,mec.gov.br,inbound,001,World,0 -mecumauction.com,mecumauction.com,inbound,001,World,0 -medallia.com,medallia.com,inbound,001,World,0.999682 -mediabistro.com,iworld.com,inbound,001,World,0.006428 -mediapost.com,mediapost.com,inbound,001,World,0 -medium.com,messagebus.com,inbound,001,World,0 -medpagetoday.com,wc09.net,inbound,001,World,0 -medscape.com,medscape.com,inbound,001,World,0 -meetic.com,meetic.com,inbound,001,World,0 -meetmemail.com,meetmemail.com,inbound,001,World,0 -meetup.com,meetup.com,inbound,001,World,5e-06 -megasenders.com,megasenders.com,inbound,001,World,0.07662 -melaleuca.com,melaleuca.com,inbound,001,World,0.003493 -memberdealsusa.com,memberdealsusa.com,inbound,001,World,0 -menswearhouse.com,menswearhouse.com,inbound,001,World,0 -mequedouno.com,mequedouno.com,inbound,001,World,0 -mercadojobs.com,sendgrid.net,inbound,001,World,1 -mercadolibre.com,mercadolibre.com,inbound,001,World,0 -mercadolivre.com,mercadolibre.com,inbound,001,World,0 -merceworld.com,merceworld.com,inbound,001,World,0.996738 -mercola.com,mercola.com,inbound,001,World,0.000369 -merodea.me,sendgrid.net,inbound,001,World,1 -messagegears.net,messagegears.net,inbound,001,World,0 -met-art.com,hydentra.com,inbound,001,World,1 -metro.co.in,srv2.de,inbound,001,World,0.966947 -metrodeal.com,fagms.de,inbound,001,World,0 -mgmresorts.com,mgmresorts.com,inbound,001,World,0 -mgo.com,bronto.com,inbound,001,World,0 -michaels.com,chtah.net,inbound,001,World,0 -michaels.com,michaels.com,inbound,001,World,0 -microcentermedia.com,bfi0.com,inbound,001,World,0 -microsoft.com,hotmail.{...},inbound,001,World,1 -microsoft.com,msn.com,inbound,001,World,1 -microsoftemail.com,microsoftemail.com,inbound,001,World,0 -microsoftemail.com,microsoftstoreemail.com,inbound,001,World,0 -midnightsunsafelist.com,zoothost.com,inbound,001,World,0 -mightydeals.co.uk,mightydeals.co.uk,inbound,001,World,0 -mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,001,World,0 -milfaholic.com,iverificationsystems.com,inbound,001,World,0 -miltnews.com,miltnews.com,inbound,001,World,0 -mindbodyonline.com,mindbodyonline.com,inbound,001,World,1 -mindfieldonline.com,mindfieldonline.com,inbound,001,World,0 -mindmoviesmail.com,mindmoviesmail.com,inbound,001,World,0.004239 -mindvalleymail3.com,mindvalleymail3.com,inbound,001,World,0 -minhavida.com.br,minhavida.com.br,inbound,001,World,0 -mint.com,mint.com,inbound,001,World,0 -minted.com,messagelabs.com,inbound,001,World,0.999669 -mirtesen.ru,mtml.ru,inbound,001,World,0 -missselfridge.com,wallis-fashion.com,inbound,001,World,0 -mistersafelist.com,zoothost.com,inbound,001,World,0 -mit.edu,mit.edu,inbound,001,World,0.868705 -mitula.net,mitula.org,inbound,001,World,0 -mitula.org,mitula.org,inbound,001,World,0 -mixcloudmail.com,mixcloudmail.com,inbound,001,World,0.995482 -mixi.jp,mixi.jp,inbound,001,World,0 -mjinn.com,mailurja.com,inbound,001,World,0 -mkt015.com,mkt015.com,inbound,001,World,0 -mkt022.com,mkt022.com,inbound,001,World,0 -mkt063.com,mkt063.com,inbound,001,World,0 -mkt1136.com,mkt1136.com,inbound,001,World,0 -mkt1985.com,fmlinks.net,inbound,001,World,0 -mkt2010.com,mkt2010.com,inbound,001,World,0 -mkt2106.com,mkt2106.com,inbound,001,World,0 -mkt2170.com,mkt2170.com,inbound,001,World,0 -mkt2181.com,mkt2181.com,inbound,001,World,0 -mkt2615.com,mkt2615.com,inbound,001,World,0 -mkt2813.com,mkt2813.com,inbound,001,World,0 -mkt2944.com,mkt2944.com,inbound,001,World,0 -mkt3134.com,mkt3134.com,inbound,001,World,0 -mkt3142.com,mkt3142.com,inbound,001,World,0 -mkt3156.com,mkt3156.com,inbound,001,World,0 -mkt3203.com,mkt3203.com,inbound,001,World,0 -mkt346.com,mkt346.com,inbound,001,World,0 -mkt3544.com,mkt3544.com,inbound,001,World,0 -mkt3622.com,mkt3622.com,inbound,001,World,0 -mkt3682.com,mkt3682.com,inbound,001,World,0 -mkt3690.com,mkt3690.com,inbound,001,World,0 -mkt3695.com,mkt3695.com,inbound,001,World,0 -mkt3804.com,mkt3804.com,inbound,001,World,0 -mkt3815.com,mkt3815.com,inbound,001,World,0 -mkt3952.com,xoom.com,inbound,001,World,0 -mkt4355.com,mkt4355.com,inbound,001,World,0 -mkt4364.com,mkt4364.com,inbound,001,World,0 -mkt459.com,mkt459.com,inbound,001,World,0 -mkt4701.com,mkt4701.com,inbound,001,World,0 -mkt4728.com,mkt4728.com,inbound,001,World,0 -mkt4731.com,mkt4731.com,inbound,001,World,0 -mkt4738.com,mkt4738.com,inbound,001,World,0 -mkt5071.com,mkt5071.com,inbound,001,World,0 -mkt5098.com,mkt5098.com,inbound,001,World,0 -mkt5131.com,mkt5131.com,inbound,001,World,0 -mkt5144.com,mkt5144.com,inbound,001,World,0 -mkt5144.com,mkt5980.com,inbound,001,World,0 -mkt5144.com,mkt5981.com,inbound,001,World,0 -mkt5181.com,mkt5181.com,inbound,001,World,0 -mkt5269.com,mkt5269.com,inbound,001,World,0 -mkt529.com,mkt529.com,inbound,001,World,0 -mkt5297.com,mkt5297.com,inbound,001,World,0 -mkt5297.com,mkt5309.com,inbound,001,World,0 -mkt5371.com,mkt5371.com,inbound,001,World,0 -mkt5806.com,mkt5806.com,inbound,001,World,0 -mkt5934.com,mkt5934.com,inbound,001,World,0 -mkt5937.com,mkt5937.com,inbound,001,World,0 -mkt5970.com,mkt5970.com,inbound,001,World,0 -mkt6100.com,mkt6098.com,inbound,001,World,0 -mkt6276.com,mkt6276.com,inbound,001,World,0 -mkt6323.com,mkt6323.com,inbound,001,World,0 -mkt746.com,mkt746.com,inbound,001,World,0 -mkt824.com,mkt869.com,inbound,001,World,0 -mktdillards.com,mktdillards.com,inbound,001,World,0 -mktid10.com,1-hostingservice.com,inbound,001,World,0 -mktomail.com,mktdns.com,inbound,001,World,0 -mktomail.com,mktomail.com,inbound,001,World,0 -mktomail.com,mktroute.com,inbound,001,World,0 -ml.com,bankofamerica.com,inbound,001,World,1 -mlgns.com,mlgns.com,inbound,001,World,0 -mlgnserv.com,mlgnserv.com,inbound,001,World,0 -mlsend.com,mlsend.com,inbound,001,World,0 -mlsend2.com,mlsend2.com,inbound,001,World,0 -mlssoccer.com,mlssoccer.com,inbound,001,World,0 -mmaco.net,mmaco.net,inbound,001,World,0.999998 -mmagic.jp,mmagic.jp,inbound,001,World,0.000531 -mmks.it,mail-maker.it,inbound,001,World,0 -mmorpg.com,mmorpg.com,inbound,001,World,0.002979 -mmsecure.nl,donenad.nl,inbound,001,World,0 -mo1send.com,mo1send.com,inbound,001,World,0 -mobile01.com,mobile01.com,inbound,001,World,0 -mobly.com.br,mobly.com.br,inbound,001,World,0 -mocospace.com,mocospace.com,inbound,001,World,0 -modellsemail.com,n-email.net,inbound,001,World,0 -modnakasta.ua,emv5.com,inbound,001,World,0 -monex.co.jp,monex.co.jp,inbound,001,World,0.019424 -moneycontrol.com,active18.com,inbound,001,World,0 -moneyforward.com,moneyforward.com,inbound,001,World,0 -moneymorning.com,moneymappress.com,inbound,001,World,0 -moneysupermarketmail.com,moneysupermarketmail.com,inbound,001,World,0 -monipla.jp,aainc.co.jp,inbound,001,World,0 -monografias.com,elistas.net,inbound,001,World,0 -monster.co.in,monster.co.in,inbound,001,World,0 -monster.com,monster.com,inbound,001,World,0.000231 -monster.com,tmpw.net,inbound,001,World,0.000503 -monsterindia.com,monster.co.in,inbound,001,World,0 -moon-ray.com,moon-ray.com,inbound,001,World,0.001892 -mooply.co,mailendo.com,inbound,001,World,0 -mooresclothing.com,mooresclothing.com,inbound,001,World,0 -morhipo.com,euromsg.net,inbound,001,World,0 -morningstar.net,morningstar.net,inbound,001,World,0 -mothercaregroup.com,neolane.net,inbound,001,World,0 -motosnap.com,motosnap.com,inbound,001,World,0.994795 -moveon.org,moveon.org,inbound,001,World,0.99577 -moviestarplanet.com,moviestarplanet.com,inbound,001,World,0 -mozilla.org,mozilla.com,inbound,001,World,0.633237 -mpme.jp,mpme.jp,inbound,001,World,0 -mpse.jp,emsaqua.jp,inbound,001,World,0 -mpse.jp,emsbeige.jp,inbound,001,World,0 -mpse.jp,emsbrown.jp,inbound,001,World,0 -mpse.jp,emscyan.jp,inbound,001,World,0 -mpse.jp,emsgold.jp,inbound,001,World,0 -mpse.jp,emslime.jp,inbound,001,World,0 -mpse.jp,emsnavy.jp,inbound,001,World,0 -mpse.jp,emspink.jp,inbound,001,World,0 -mpse.jp,emssnow.jp,inbound,001,World,0 -mpse.jp,mpme.jp,inbound,001,World,0 -mpse.jp,yahoo.co.jp,inbound,001,World,0 -mpsnd.ch,agenceweb.net,inbound,001,World,0 -mrc.org,msgfocus.com,inbound,001,World,0 -mrmlsmatrix.com,mrmlsmatrix.com,inbound,001,World,0 -ms.com,ms.com,inbound,001,World,1 -ms00.net,ms00.net,inbound,001,World,0 -msdp1.com,msdp1.com,inbound,001,World,0 -msgfocus.com,msgfocus.com,inbound,001,World,0.011658 -msn.com,hotmail.{...},inbound,001,World,0.999964 -msn.com,hotmail.{...},outbound,001,World,1 -mta.info,ealert.com,inbound,001,World,0 -mtasv.net,mtasv.net,inbound,001,World,0.999999 -musiciansfriend.com,musiciansfriend.com,inbound,001,World,0 -musicnotes-alerts.com,mybuys.com,inbound,001,World,0 -mustanglist.com,mustanglist.com,inbound,001,World,0 -mxmfb.com,mxmfb.com,inbound,001,World,0 -mycheapoair.com,mycheapoair.com,inbound,001,World,0.957931 -mycolorscreen.com,mta4.net,inbound,001,World,0 -mydailymoment.biz,mydailymoment.biz,inbound,001,World,0 -mydailymoment.info,mydailymoment.info,inbound,001,World,0 -mydailymoment.net,mydailymoment.net,inbound,001,World,0 -mydailymoment.us,mydailymoment.us,inbound,001,World,0 -myfedloan.org,aessuccess.org,inbound,001,World,0.328917 -myfitnesspal.com,messagebus.com,inbound,001,World,0 -myfxbook.com,myfxbook.com,inbound,001,World,0 -mygreatlakes.org,glhec.org,inbound,001,World,0.000247 -mygroupon.co.th,grouponmail.{...},inbound,001,World,0 -myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,001,World,0 -myheritage.com,myheritage.com,inbound,001,World,0 -myideeli.com,myideeli.com,inbound,001,World,0 -mymeijer.com,mymeijer.com,inbound,001,World,0 -mymms.com,fagms.de,inbound,001,World,0 -mynavi.jp,mynavi.jp,inbound,001,World,3.5e-05 -myngp.com,ngpweb.com,inbound,001,World,0 -myntramail.com,iaires.com,inbound,001,World,0 -myntramail.com,myntramail.com,inbound,001,World,0 -myntramails.in,icubes.in,inbound,001,World,0 -myoutlets.in,trustmailer.com,inbound,001,World,0 -myperfectsale.com,myperfectsale.com,inbound,001,World,0.999988 -mypoints.com,mypoints.com,inbound,001,World,0 -myprotein.com,thehut.com,inbound,001,World,0 -mysafelistmailer.com,mysafelistmailer.com,inbound,001,World,0.00033 -mysale.my,mysale.my,inbound,001,World,0 -mysale.ph,mysale.ph,inbound,001,World,0 -mysmartprice.com,itzwow.com,inbound,001,World,1 -mysupermarket.co.uk,mysupermarket.co.uk,inbound,001,World,0 -mysurvey.com,mysurvey.com,inbound,001,World,0 -mysurvey.eu,mysurvey.com,inbound,001,World,0 -myvegas.com,myvegas.com,inbound,001,World,1 -myzamanamail.com,myzamanamail.com,inbound,001,World,0 -n-email.net,n-email.net,inbound,001,World,0 -n-email1.net,n-email1.net,inbound,001,World,0 -n-email4.net,n-email4.net,inbound,001,World,0 -naaptoldeals.com,eccluster.com,inbound,001,World,0 -namorico.me,namorico.me,inbound,001,World,1 -nanomail.com.br,araie.com.br,inbound,001,World,1 -napitipp.hu,napitipp.hu,inbound,001,World,0 -nasa.gov,nasa.gov,inbound,001,World,0.082443 -nascar.com,nascar.com,inbound,001,World,0 -nastygal.com,bronto.com,inbound,001,World,0 -nasza-klasa.pl,nasza-klasa.pl,inbound,001,World,0 -nationalexpress.com,nationalexpress.com,inbound,001,World,0 -nationbuilder.com,nationbuilder.com,inbound,001,World,1 -nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,001,World,0 -nature.com,nature.com,inbound,001,World,0 -naukri.com,naukri.com,inbound,001,World,0.002488 -nauta.cu,etecsa.net,inbound,001,World,0 -naver.com,naver.com,inbound,001,World,1 -naver.com,naver.com,outbound,001,World,1 -navy.mil,navy.mil,inbound,001,World,0.152107 -nba.com,nba.com,inbound,001,World,0.005828 -nbaa.org,nbaa.org,inbound,001,World,0 -ncl.com,ncl.com,inbound,001,World,0 -neimanmarcusemail.com,neimanmarcusemail.com,inbound,001,World,0 -nend.net,postini.com,inbound,001,World,0 -neolane.net,neolane.net,inbound,001,World,0.01682 -nesinemail.com,euromsg.net,inbound,001,World,0 -net-a-porter.com,net-a-porter.com,inbound,001,World,0 -net-empregos.com,net-empregos.com,inbound,001,World,0 -net-survey.jp,net-survey.jp,inbound,001,World,0 -netatlantic.com,netatlantic.com,inbound,001,World,0.001085 -netbk.co.jp,netbk.co.jp,inbound,001,World,0.010994 -netcommunity1.com,blackbaud.com,inbound,001,World,0 -netflix.com,amazonses.com,inbound,001,World,0.999999 -netflix.com,netflix.com,inbound,001,World,1 -netlogmail.com,netlogmail.com,inbound,001,World,0 -netopia.pt,netopia.pt,inbound,001,World,0.017294 -netprosoftmail.com,netprosoftmail.com,inbound,001,World,0 -netshoes.com.br,netshoes.com.br,inbound,001,World,0.078768 -netsuite.com,netsuite.com,inbound,001,World,0.57723 -networkworld.com,networkworld.com,inbound,001,World,0 -newegg.com,newegg.com,inbound,001,World,1e-06 -newgrounds.com,newgrounds.com,inbound,001,World,0 -newmarkethealth.com,newmarkethealth.com,inbound,001,World,0 -newrelic.com,sendlabs.com,inbound,001,World,0 -news-h5g.com,news-h5g.com,inbound,001,World,0 -newsletter-verychic.com,splio.es,inbound,001,World,0.9804 -newsmax.com,newsmax.com,inbound,001,World,0.004105 -newspaperdirect.com,newspaperdirect.com,inbound,001,World,0.000177 -newyorktimesinfo.com,newyorktimesinfo.com,inbound,001,World,0 -nexcess.net,nexcess.net,inbound,001,World,0.214803 -next-engine.org,next-engine.org,inbound,001,World,1 -nextdoor.com,mailgun.info,inbound,001,World,1 -nextdoor.com,mailgun.net,inbound,001,World,1 -nextdoor.com,nextdoor.com,inbound,001,World,1 -nfl.com,bfi0.com,inbound,001,World,0 -nflshop.com,nflshop.com,inbound,001,World,0 -nhs.jobs,nhscareersjobs.co.uk,inbound,001,World,0 -nic.in,relayout.nic.in,inbound,001,World,0 -nicovideo.jp,nicovideo.jp,inbound,001,World,0 -nieuwsblad.be,vummail.be,inbound,001,World,0 -nifty.com,nifty.com,inbound,001,World,0.762068 -nih.gov,nih.gov,inbound,001,World,8.8e-05 -nike.com,nike.com,inbound,001,World,0 -nikkei.com,nikkei.co.jp,inbound,001,World,0 -nikkeibp.co.jp,nikkeibp.co.jp,inbound,001,World,4.9e-05 -ninewestmail.com,ninewestmail.com,inbound,001,World,0 -ning.com,ning.com,inbound,001,World,0 -nissen.jp,nissen.jp,inbound,001,World,0 -nixle.com,nixle.com,inbound,001,World,0 -nl00.net,netline.com,inbound,001,World,5e-06 -nl00.net,nl00.net,inbound,001,World,0 -nmp1.com,nmp1.net,inbound,001,World,0 -nokia.com,nokia.com,inbound,001,World,0.001256 -nongnu.org,gnu.org,inbound,001,World,1 -nordstrom.com,taleo.net,inbound,001,World,1 -nortonfromsymantec.com,rsys1.com,inbound,001,World,0 -nos.pt,netcabo.pt,inbound,001,World,6.3e-05 -noticiasaominuto.com,ccmdcampaigns.net,inbound,001,World,0 -noticiasaominuto.com,noticiasaominuto.com,inbound,001,World,0 -novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,001,World,0 -npr.org,npr.org,inbound,001,World,0 -nrholding.net,nrholding.net,inbound,001,World,0 -ns.nl,tripolis.com,inbound,001,World,0 -nsandi.com,mxmfb.com,inbound,001,World,0 -numbersusa.com,numbersusa.com,inbound,001,World,0 -nyandcompany.com,nyandcompany.com,inbound,001,World,0 -nytimes.com,nytimes.com,inbound,001,World,0.00472 -nzsale.co.nz,nzsale.co.nz,inbound,001,World,0 -oakley.com,oakley.com,inbound,001,World,0.000904 -ocadomail.com,ocadomail.com,inbound,001,World,0 -ocmail1.in,tcmail.in,inbound,001,World,0 -ocmail14.in,tcmailer5.in,inbound,001,World,0 -ocmail22.in,tcmailer15.in,inbound,001,World,0 -ocmail22.in,tcmailer4.in,inbound,001,World,0 -ocmail40.in,tcmailer15.in,inbound,001,World,0 -ocmail40.in,tcmailer4.in,inbound,001,World,0 -ocn.ad.jp,ocn.ad.jp,inbound,001,World,0 -ocn.ne.jp,ocn.ad.jp,inbound,001,World,0 -ocnmail.in,ocmail6.in,inbound,001,World,0 -ocnmail.in,tcmail3.in,inbound,001,World,0 -odisseias.com,emv4.net,inbound,001,World,0 -odnoklassniki.ru,odnoklassniki.ru,inbound,001,World,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,001,World,0 -ofertasefacil.com.br,ofertasefacil.com.br,inbound,001,World,0 -ofertix.com,ofertix.com,inbound,001,World,0 -ofertop.pe,icommarketing.com,inbound,001,World,0 -offers.com,offers.com,inbound,001,World,1 -offerum.com,cccampaigns.com,inbound,001,World,0 -offerum.com,ccemails.com,inbound,001,World,0 -officedepot.com,officedepot.com,inbound,001,World,0.012483 -officemax.com,officemax.com,inbound,001,World,0 -officemax.com,officemaxworkplace.com,inbound,001,World,0 -ofsys.com,bulletin-metro.ca,inbound,001,World,0 -oknotify2.com,oknotify2.com,inbound,001,World,0 -oldnavy.ca,oldnavy.ca,inbound,001,World,0 -oldnavy.com,oldnavy.com,inbound,001,World,0 -olx.pt,fixeads.com,inbound,001,World,0 -olympiaedge.net,olympiaedge.net,inbound,001,World,0 -omahasteaks.com,omahasteaks.com,inbound,001,World,0.006139 -oneindia.in,infimail.com,inbound,001,World,0 -oneindia.in,mailurja.com,inbound,001,World,0 -onekingslane.com,onekingslane.com,inbound,001,World,0 -onepatriotplace.com,britecast.com,inbound,001,World,0 -onestopplus.com,neolane.net,inbound,001,World,0 -onetravelspecials.com,onetravelspecials.com,inbound,001,World,0.365386 -online.com,cnet.com,inbound,001,World,0 -onlive.com,ipost.com,inbound,001,World,0 -onmicrosoft.com,outlook.com,inbound,001,World,1 -onthecitymail.org,onthecitymail.org,inbound,001,World,1 -oo155.com,bsftransmit7.com,inbound,001,World,0 -oo155.com,oo155.com,inbound,001,World,0 -openstack.org,openstack.org,inbound,001,World,0.996884 -openstackmail.com,infimail.com,inbound,001,World,0 -opentable.com,opentable.com,inbound,001,World,0.000662 -opinionoutpost.com,opinionoutpost.com,inbound,001,World,0 -opinionsquare.com,opinionsquare.com,inbound,001,World,0 -oprah.com,oprah.com,inbound,001,World,0 -opticsplanet.com,opticsplanet.com,inbound,001,World,0 -optimusmail.in,iaires.com,inbound,001,World,0 -optonline.net,cv.net,inbound,001,World,0 -optonline.net,optonline.net,outbound,001,World,0 -orange.fr,orange.fr,inbound,001,World,0 -orange.fr,orange.fr,outbound,001,World,0 -orderscatalog.com,orderscatalog.com,inbound,001,World,0 -oriental-trading.com,oriental-trading.com,inbound,001,World,0 -oroscopofree.com,adsender.us,inbound,001,World,0 -oroscopofree.com,oroscopofree.com,inbound,001,World,0 -orsay.com,emp-mail.de,inbound,001,World,0 -os-email.com,os-email.com,inbound,001,World,0 -oshkoshbgosh.com,oshkoshbgosh.com,inbound,001,World,6e-06 -osu.edu,outlook.com,inbound,001,World,1 -otto.de,eccluster.com,inbound,001,World,1 -ouffer.com,ouffer.com,inbound,001,World,0.01022 -ourtime.com,seniorpeoplemeet.com,inbound,001,World,0 -outback.com,outback.com,inbound,001,World,0 -outlook.com,hotmail.{...},inbound,001,World,0.999929 -outlook.com,hotmail.{...},outbound,001,World,1 -outspot.be,teneo.be,inbound,001,World,0 -outspot.nl,teneo.be,inbound,001,World,0 -ovenmail.com,iaires.com,inbound,001,World,0 -overnightprints.com,chtah.net,inbound,001,World,0 -overstock.com,overstock.com,inbound,001,World,0 -ovh.net,ovh.net,inbound,001,World,0.207527 -ovuline.com,ovuline.com,inbound,001,World,1 -oxfam.org.uk,msgfocus.com,inbound,001,World,0 -ozsale.com.au,ozsale.com.au,inbound,001,World,0.000192 -p-world.co.jp,p-world.co.jp,inbound,001,World,0 -pagoda.com,zales.com,inbound,001,World,0 -pagseguro.com.br,uol.com.br,inbound,001,World,0 -pair.com,pair.com,inbound,001,World,0.880779 -palmscasinoresort.com,palmscasinoresort.com,inbound,001,World,0 -pampers.com,bfi0.com,inbound,001,World,0 -panasonic.jp,panasonic.jp,inbound,001,World,0 -pandaresearch.com,pandaresearch.com,inbound,001,World,0 -pandora.com,pandora.com,inbound,001,World,1 -pandora.net,pandora.net,inbound,001,World,0.999666 -panelplace.com,smtp.com,inbound,001,World,0 -panerabreadnews.com,panerabreadnews.com,inbound,001,World,0 -pantaloondirect.net,iaires.com,inbound,001,World,0 -papajohns-specials.com,papajohns-specials.com,inbound,001,World,0 -paradisepublishers.com,paradisepublishers.com,inbound,001,World,0.999626 -parents.com,meredith.com,inbound,001,World,0 -parkmobileglobal.com,parkmobile.us,inbound,001,World,0 -path.com,path.com,inbound,001,World,1 -patriotupdate.com,inboxfirst.com,inbound,001,World,0 -payback.de,artegic.net,inbound,001,World,0.999986 -payback.in,chtah.net,inbound,001,World,0 -payback.in,eccluster.com,inbound,001,World,0 -payback.in,ecm-cluster.com,inbound,001,World,0 -payback.in,ecmcluster.com,inbound,001,World,0 -paypal.co.uk,paypal.com,inbound,001,World,1 -paypal.com,paypal.com,inbound,001,World,0.608856 -paypal.com.au,paypal.com,inbound,001,World,1 -paypal.de,paypal.com,inbound,001,World,1 -paytm.com,paytm.com,inbound,001,World,1 -pbteen.com,pbteen.com,inbound,001,World,0 -pccomponentes.com,pccomponentes.com,inbound,001,World,0 -pch.com,ed10.com,inbound,001,World,0 -pchfrontpage.com,ed10.com,inbound,001,World,0 -pchlotto.com,ed10.com,inbound,001,World,0 -pchplayandwin.com,ed10.com,inbound,001,World,0 -pchsearch.com,ed10.com,inbound,001,World,0 -pcmag.com,ittoolbox.com,inbound,001,World,0 -pcworld.com,pcworld.com,inbound,001,World,0 -pd25.com,pd25.com,inbound,001,World,1 -pd25.com,pd27.com,inbound,001,World,1 -peanuthome.info,adopterc.info,inbound,001,World,0 -peanuthome.info,aguitytr.info,inbound,001,World,0 -peanuthome.info,bevest.info,inbound,001,World,0 -peanuthome.info,bluester.info,inbound,001,World,0 -peanuthome.info,burror.info,inbound,001,World,0 -peanuthome.info,bursion.info,inbound,001,World,0 -peanuthome.info,cantexi.info,inbound,001,World,0 -peanuthome.info,caserhi.info,inbound,001,World,0 -peanuthome.info,celect.info,inbound,001,World,0 -peanuthome.info,chintone.info,inbound,001,World,0 -peanuthome.info,citery.info,inbound,001,World,0 -peanuthome.info,cleathal.info,inbound,001,World,0 -peanuthome.info,coherentrequittal.info,inbound,001,World,0 -peanuthome.info,colicom.info,inbound,001,World,0 -peanuthome.info,complec.info,inbound,001,World,0 -peanuthome.info,cyprinoidkaiserdom.info,inbound,001,World,0 -peanuthome.info,deciarc.info,inbound,001,World,0 -peanuthome.info,declaws.info,inbound,001,World,0 -peanuthome.info,dewest.info,inbound,001,World,0 -peanuthome.info,epconce.info,inbound,001,World,0 -peanuthome.info,fnotec.info,inbound,001,World,0 -peanuthome.info,folkswor.info,inbound,001,World,0 -peanuthome.info,forepert.info,inbound,001,World,0 -peanuthome.info,gurgaro.info,inbound,001,World,0 -peanuthome.info,heallyps.info,inbound,001,World,0 -peanuthome.info,holeph.info,inbound,001,World,0 -peanuthome.info,homewor.info,inbound,001,World,0 -peanuthome.info,hydroni.info,inbound,001,World,0 -peanuthome.info,ingenbu.info,inbound,001,World,0 -peanuthome.info,kinklybotaurus.info,inbound,001,World,0 -peanuthome.info,ninetiethwhiffet.info,inbound,001,World,0 -peanuthome.info,unglibshudder.info,inbound,001,World,0 -peanutwebmaster.info,adcrent.info,inbound,001,World,0 -peanutwebmaster.info,addmiel.info,inbound,001,World,0 -peanutwebmaster.info,agilhe.info,inbound,001,World,0 -peanutwebmaster.info,allegap.info,inbound,001,World,0 -peanutwebmaster.info,andvore.info,inbound,001,World,0 -peanutwebmaster.info,angogl.info,inbound,001,World,0 -peanutwebmaster.info,animass.info,inbound,001,World,0 -peanutwebmaster.info,arettery.info,inbound,001,World,0 -peanutwebmaster.info,aribank.info,inbound,001,World,0 -peanutwebmaster.info,arkefoc.info,inbound,001,World,0 -peanutwebmaster.info,avenog.info,inbound,001,World,0 -peanutwebmaster.info,barrave.info,inbound,001,World,0 -peanutwebmaster.info,bindowmo.info,inbound,001,World,0 -peanutwebmaster.info,bitravit.info,inbound,001,World,0 -peanutwebmaster.info,borsand.info,inbound,001,World,0 -peanutwebmaster.info,branti.info,inbound,001,World,0 -peanutwebmaster.info,breaserp.info,inbound,001,World,0 -peanutwebmaster.info,bredogly.info,inbound,001,World,0 -peanutwebmaster.info,briantra.info,inbound,001,World,0 -peanutwebmaster.info,bridea.info,inbound,001,World,0 -peanutwebmaster.info,carial.info,inbound,001,World,0 -peanutwebmaster.info,castac.info,inbound,001,World,0 -peanutwebmaster.info,chedoner.info,inbound,001,World,0 -peanutwebmaster.info,chiquent.info,inbound,001,World,0 -peanutwebmaster.info,cinchoi.info,inbound,001,World,0 -peanutwebmaster.info,cliate.info,inbound,001,World,0 -peanutwebmaster.info,cognn.info,inbound,001,World,0 -peanutwebsite.info,abjibbin.info,inbound,001,World,0 -peanutwebsite.info,audiette.info,inbound,001,World,0 -peanutwebsite.info,blancer.info,inbound,001,World,0 -peanutwebsite.info,bluester.info,inbound,001,World,0 -peanutwebsite.info,burror.info,inbound,001,World,0 -peanutwebsite.info,bursion.info,inbound,001,World,0 -peanutwebsite.info,caserhi.info,inbound,001,World,0 -peanutwebsite.info,celect.info,inbound,001,World,0 -peanutwebsite.info,cleathal.info,inbound,001,World,0 -peanutwebsite.info,coherentrequittal.info,inbound,001,World,0 -peanutwebsite.info,complec.info,inbound,001,World,0 -peanutwebsite.info,condost.info,inbound,001,World,0 -peanutwebsite.info,cyprinoidkaiserdom.info,inbound,001,World,0 -peanutwebsite.info,deciarc.info,inbound,001,World,0 -peanutwebsite.info,declaws.info,inbound,001,World,0 -peanutwebsite.info,epconce.info,inbound,001,World,0 -peanutwebsite.info,ferrayer.info,inbound,001,World,0 -peanutwebsite.info,fnotec.info,inbound,001,World,0 -peanutwebsite.info,folkswor.info,inbound,001,World,0 -peanutwebsite.info,forepert.info,inbound,001,World,0 -peanutwebsite.info,gurgaro.info,inbound,001,World,0 -peanutwebsite.info,holeph.info,inbound,001,World,0 -peanutwebsite.info,homewor.info,inbound,001,World,0 -peanutwebsite.info,hydroni.info,inbound,001,World,0 -peanutwebsite.info,ingenbu.info,inbound,001,World,0 -peanutwebsite.info,kinklybotaurus.info,inbound,001,World,0 -peanutwebsite.info,ninetiethwhiffet.info,inbound,001,World,0 -pearlsofwealth.com,pearlsofwealth.com,inbound,001,World,1 -peartreegreetings.com,rexcraft.com,inbound,001,World,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,001,World,0 -pennwell.com,pennwell.com,inbound,001,World,0 -pepabo.com,pepabo.com,inbound,001,World,0 -pepboys.com,pepboys.com,inbound,001,World,0 -peperoni.de,peperoni.de,inbound,001,World,1 -pepperfry.com,epidm.net,inbound,001,World,0 -perfectpriceindia.com,infimail.com,inbound,001,World,0 -perfectworld.com,perfectworld.com,inbound,001,World,0.000162 -perfora.net,perfora.net,inbound,001,World,0.920896 -permissionresearch.com,permissionresearch.com,inbound,001,World,0 -personalliberty.com,personalliberty.com,inbound,001,World,1 -personare.com.br,personare.com.br,inbound,001,World,0 -peytz.dk,peytz.dk,inbound,001,World,4e-06 -pga.com,pga.com,inbound,001,World,0 -pge.com,pge.com,inbound,001,World,0.149792 -pgeveryday.com,bfi0.com,inbound,001,World,0 -philosophy.com,philosophy.com,inbound,001,World,0 -phoenix.edu,phoenix.edu,inbound,001,World,0 -photobox.com,photobox.com,inbound,001,World,0 -photoprintit.com,photoprintit.com,inbound,001,World,0 -phpclasses.org,phpclasses.org,inbound,001,World,0 -phsmtpbox.com,phsmtpbox.com,inbound,001,World,0 -pia.jp,pia.jp,inbound,001,World,0 -pinger.com,pinger.com,inbound,001,World,0 -pinterest.com,pinterest.com,inbound,001,World,1 -pivotaltracker.com,pivotaltracker.com,inbound,001,World,1 -pixable.com,pixable.com,inbound,001,World,1 -pixum.com,pixum.com,inbound,001,World,0 -pizzahut.com,quikorder.com,inbound,001,World,0 -pizzahutoffers.com,pizzahutoffers.com,inbound,001,World,0 -placedestendances.com,placedestendances.com,inbound,001,World,0 -plaisio.gr,fagms.de,inbound,001,World,0 -planeo.com,planeo.com,inbound,001,World,0 -planeo.pt,planeo.pt,inbound,001,World,0 -playstation.com,playstation.com,inbound,001,World,0 -playstationmail.net,playstationmail.net,inbound,001,World,0 -playtika.com,emv8.com,inbound,001,World,0 -plexapp.com,plex.tv,inbound,001,World,1 -plumdistrict.com,plumdistrict.com,inbound,001,World,1 -pmailus.com,patrontechnology.com,inbound,001,World,0 -pnc.com,messagelabs.com,inbound,001,World,0.997194 -pnetweb.co.za,hosting.co.za,inbound,001,World,0 -pnetweb.co.za,salesnet.co.za,inbound,001,World,0 -pobox.com,pobox.com,inbound,001,World,0.975372 -pof.com,plentyoffish.co.uk,inbound,001,World,0 -pogo.com,pogo.com,inbound,001,World,0 -pointtown.com,gmo-media.jp,inbound,001,World,0 -poinx.com,poinx.com,inbound,001,World,0 -pokerstars.com,pokerstars.eu,inbound,001,World,0 -pokerstars.eu,pokerstars.eu,inbound,001,World,0 -pokupon.by,mailersend.com,inbound,001,World,0 -pokupon.ua,mailersend.com,inbound,001,World,0 -politicoemail.com,politicoemail.com,inbound,001,World,0 -polyvore.com,polyvore.com,inbound,001,World,1 -pontofrio.com.br,emv8.com,inbound,001,World,0 -popsugar.com,popsugar.com,inbound,001,World,0 -postcardfromhell.com,cyberthugs.com,inbound,001,World,1 -postgresql.org,postgresql.org,inbound,001,World,0.996805 -potterybarn.com,potterybarn.com,inbound,001,World,0 -potterybarnkids.com,potterybarnkids.com,inbound,001,World,0 -praca.pl,praca.pl,inbound,001,World,1 -pracuj.pl,pracuj.pl,inbound,001,World,0 -preferredpetclub.com,preferredpetclub.com,inbound,001,World,0 -presslaff.net,dat-e-baseonline.com,inbound,001,World,0 -pressmartmail.com,pressmartmail.com,inbound,001,World,0 -priceline.com,priceline.com,inbound,001,World,1 -princess.com,princess.com,inbound,001,World,0 -printvenue.com,fagms.de,inbound,001,World,0 -priorityoneemail.com,priorityoneemail.com,inbound,001,World,0 -privalia.com,privalia.com,inbound,001,World,3.94913202027326e-07 -private-elist.com,private-elist.com,inbound,001,World,0 -privoscite.si,privoscite.si,inbound,001,World,0 -profitcenteronline.com,groupdealtools.com,inbound,001,World,1 -progressive.com,progressive.com,inbound,001,World,0.897291 -progressiveagent.com,progressive.com,inbound,001,World,1 -promedmail.org,childrenshospital.org,inbound,001,World,1 -promod-news.fr,promod-news.com,inbound,001,World,0 -propertysolutions.com,propertysolutions.com,inbound,001,World,1 -prospectgeysercoop.com,prospectgeysercoop.com,inbound,001,World,1 -protopmail.com,protopmail.com,inbound,001,World,0 -providesupport.com,providesupport.com,inbound,001,World,0.000103 -proxyvote.com,adp-ics.com,inbound,001,World,0.908635 -psu.edu,psu.edu,inbound,001,World,0.073843 -pttplc.com,pttgrp.com,inbound,001,World,0 -publicators.com,publicators.com,inbound,001,World,0 -publix.com,publix.com,inbound,001,World,0.002567 -pucp.edu.pe,pucp.edu.pe,inbound,001,World,0.225541 -puffinmailer.com,zoothost.com,inbound,001,World,0 -pur3.net,pur3.net,inbound,001,World,0.001111 -purewow.com,purewow.com,inbound,001,World,0 -puritan.com,email-nbtyinc.com,inbound,001,World,0 -purlsmail.com,purlsmail.com,inbound,001,World,0 -pxsmail.com,pxsmail.com,inbound,001,World,0 -python.org,python.org,inbound,001,World,1 -q.com,synacor.com,inbound,001,World,0.999729 -qemailserver.com,qemailserver.com,inbound,001,World,0 -qoinpro.com,qoinpro.com,inbound,001,World,1 -qoo10.jp,qoo10.jp,inbound,001,World,2e-06 -qoo10.sg,qoo10.co.id,inbound,001,World,1.9e-05 -qoo10.sg,qoo10.com,inbound,001,World,0 -qoo10.sg,qoo10.my,inbound,001,World,2.2e-05 -qoo10.sg,qoo10.sg,inbound,001,World,8e-06 -qq.com,qq.com,inbound,001,World,0.999978 -qq.com,qq.com,outbound,001,World,1 -qtropnews.com,qtropnews.com,inbound,001,World,1.7e-05 -qualicorp.com.br,qualicorp.com.br,inbound,001,World,1 -quality.net.ua,quality.net.ua,inbound,001,World,0 -qualitysafelist.com,zoothost.com,inbound,001,World,0 -queopinas.com,confirmit.com,inbound,001,World,1 -quickbooks.com,intuit.com,inbound,001,World,0.99908 -quickrewards.net,quickrewards.net,inbound,001,World,0 -quikr.com,quikr.com,inbound,001,World,0.026599 -quinstreet.com,neoquin.com,inbound,001,World,0 -quora.com,quora.com,inbound,001,World,1 -quoramail.com,quoramail.com,inbound,001,World,1 -qvcemail.com,qvcemail.com,inbound,001,World,0 -r-project.org,ethz.ch,inbound,001,World,0.99999 -r51.it,musvc.com,inbound,001,World,0.092498 -r52.it,musvc.com,inbound,001,World,0.000221 -r57.it,musvc.com,inbound,001,World,0.139425 -r67.it,musvc.com,inbound,001,World,0.199845 -r70.it,musvc.com,inbound,001,World,0.311983 -rabota.ua,rabota.ua,inbound,001,World,0 -rackroom-email.com,rackroom-email.com,inbound,001,World,0 -radarsystems.net,radarsystems.net,inbound,001,World,1 -radiantretailapps.com,radiantretailapps.com,inbound,001,World,0 -radioshack.com,radioshack.com,inbound,001,World,0 -railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,001,World,0 -rakuten.co.jp,rakuten.co.jp,inbound,001,World,0 -rakuten.co.jp,shareee.jp,inbound,001,World,0 -rakuten.co.jp,yahoo.co.jp,inbound,001,World,0 -rakuten.com,rakuten.com,inbound,001,World,0 -rakuten.ne.jp,rakuten.co.jp,inbound,001,World,0 -rambler.ru,rambler.ru,inbound,001,World,0.08173 -randomhouse.com,randomhouse.com,inbound,001,World,0 -rapattoni.com,rapmls.com,inbound,001,World,0 -ratedpeople.com,ratedpeople.com,inbound,001,World,0.000979 -rax.ru,rax.ru,inbound,001,World,0.000258 -razerzone.com,chtah.net,inbound,001,World,0 -rbc.com,rbc.com,inbound,001,World,0.001897 -rdio.com,rdio.com,inbound,001,World,1 -reactiveadz.com,downlinebuilderdirect.com,inbound,001,World,0 -realage-mail.com,postdirect.com,inbound,001,World,0 -realestate.com.au,realestate.com.au,inbound,001,World,0 -realtor.org,realtor.org,inbound,001,World,0 -realtytrac.com,realtytrac.com,inbound,001,World,0.001681 -realus.co.jp,realus.co.jp,inbound,001,World,0 -recipe.com,meredith.com,inbound,001,World,0 -recochoku.jp,recochoku.jp,inbound,001,World,0 -recruit.net,recruit.net,inbound,001,World,0 -redbox.com,exacttarget.com,inbound,001,World,0 -redbox.com,redbox.com,inbound,001,World,4e-06 -redcross.org.uk,redcross.org.uk,inbound,001,World,0 -redfin.com,redfin.com,inbound,001,World,0 -rediffmail.com,akadns.net,outbound,001,World,0 -rediffmail.com,rediffmail.com,inbound,001,World,0 -redtri.com,redtri.com,inbound,001,World,0 -reebokusnews.com,reebokusnews.com,inbound,001,World,0 -reebonz.com,ed10.com,inbound,001,World,0 -reebonz.com,reebonz.com,inbound,001,World,0.857615 -reed.co.uk,reed.co.uk,inbound,001,World,0 -regie11.net,odiso.net,inbound,001,World,0 -regionalhelpwanted.com,regionalhelpwanted.com,inbound,001,World,1 -registrar-servers.com,registrar-servers.com,inbound,001,World,0.053619 -registro.br,registro.br,inbound,001,World,0 -relax7.hu,gruppi.hu,inbound,001,World,0 -relianceada.com,relianceada.com,inbound,001,World,0.276515 -rent.com,rent.com,inbound,001,World,0.000397 -rentalcars.com,rentalcars.com,inbound,001,World,1 -renweb.com,renweb.com,inbound,001,World,0 -repica.jp,kakaku.com,inbound,001,World,0 -repica.jp,repica.jp,inbound,001,World,0 -republicwireless.com,republicwireless.com,inbound,001,World,0.033564 -research-panel.jp,research-panel.jp,inbound,001,World,0 -researchgate.net,researchgate.net,inbound,001,World,0 -responder.co.il,responder.co.il,inbound,001,World,1 -restorationhardware.com,restorationhardware.com,inbound,001,World,0 -retailjobinsider.com,retailjobinsider.com,inbound,001,World,0 -retailmenot.com,retailmenot.com,inbound,001,World,3.54930373308668e-07 -reverbnation.com,reverbnation.com,inbound,001,World,0 -revolutiongolf.com,revolutiongolf.com,inbound,001,World,0 -rewardme.in,bfi0.com,inbound,001,World,0 -reyrey.net,reyrey.net,inbound,001,World,0.946402 -ricardoeletro.com.br,allin.com.br,inbound,001,World,0 -richersoundsvip.com,ibwmail.com,inbound,001,World,0 -richyrichmailer.com,maddog-productions.info,inbound,001,World,1 -rightmove.com,rightmove.com,inbound,001,World,0 -rigzonemail.com,rigzonemail.com,inbound,001,World,0 -rikunabi.com,rikunabi.com,inbound,001,World,1e-05 -ringcentral.com,ringcentral.com,inbound,001,World,1 -ripleyperu.com.pe,icommarketing.com,inbound,001,World,0 -riseup.net,riseup.net,inbound,001,World,1 -rivamail.com,mailurja.com,inbound,001,World,0 -rmtr.de,rapidmail.de,inbound,001,World,0 -rnmk.com,rnmk.com,inbound,001,World,0 -roadrunner.com,rr.com,inbound,001,World,0.005479 -roamans.com,neolane.net,inbound,001,World,0 -rocketmail.com,yahoo.{...},inbound,001,World,1 -rocketmail.com,yahoodns.net,outbound,001,World,1 -rockpath.info,rockpath.info,inbound,001,World,1 -rockwellcollins.com,rockwellcollins.com,inbound,001,World,1 -rogers.com,yahoo.{...},inbound,001,World,1 -rogers.com,yahoodns.net,outbound,001,World,1 -rookiestewsemails.com,rookiestewsemails.com,inbound,001,World,0 -roulartamail.be,roulartamail.be,inbound,001,World,0 -royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,001,World,0 -rpinow.org,app-info.net,inbound,001,World,0 -rpo9usa.email,rpo9usa.email,inbound,001,World,0 -rr.com,rr.com,inbound,001,World,0.008342 -rr.com,rr.com,outbound,001,World,0 -rsgsv.net,postini.com,inbound,001,World,0.093513 -rsgsv.net,rsgsv.net,inbound,001,World,0 -rsvpsv.net,rsvpsv.net,inbound,001,World,0 -rsvpsv.net,send.esp.br,inbound,001,World,0 -rsys2.com,amfam.com,inbound,001,World,0 -rsys2.com,cheaptickets.com,inbound,001,World,0 -rsys2.com,dishnetworkmail.com,inbound,001,World,0 -rsys2.com,e-comms.net,inbound,001,World,0 -rsys2.com,eharmony.com,inbound,001,World,0 -rsys2.com,fathead.com,inbound,001,World,0 -rsys2.com,intuit.com,inbound,001,World,0 -rsys2.com,kmart.com,inbound,001,World,0 -rsys2.com,kohlernews.com,inbound,001,World,0 -rsys2.com,lego.com,inbound,001,World,0 -rsys2.com,lenovo.com,inbound,001,World,0 -rsys2.com,modcloth.com,inbound,001,World,0 -rsys2.com,moxieinteractive.com,inbound,001,World,0 -rsys2.com,orbitz.com,inbound,001,World,0 -rsys2.com,payless.com,inbound,001,World,0 -rsys2.com,petsathome.com,inbound,001,World,0 -rsys2.com,quizzle.com,inbound,001,World,0 -rsys2.com,robeez.com,inbound,001,World,0 -rsys2.com,rsys1.com,inbound,001,World,0 -rsys2.com,rsys2.com,inbound,001,World,0 -rsys2.com,rsys3.com,inbound,001,World,0 -rsys2.com,rsys4.com,inbound,001,World,0 -rsys2.com,saucony.com,inbound,001,World,0 -rsys2.com,sears.com,inbound,001,World,0 -rsys2.com,shopbop.com,inbound,001,World,0 -rsys2.com,southwest.com,inbound,001,World,0 -rsys2.com,speeddatemail.com,inbound,001,World,0 -rsys2.com,thecompanystore.com,inbound,001,World,0 -rsys2.com,theknot.com,inbound,001,World,0 -rsys5.com,alibris.com,inbound,001,World,0 -rsys5.com,allstate-email.com,inbound,001,World,0 -rsys5.com,beachmint.com,inbound,001,World,0 -rsys5.com,belleandclive.com,inbound,001,World,0 -rsys5.com,br.dk,inbound,001,World,0 -rsys5.com,charlotterusse.com,inbound,001,World,0 -rsys5.com,comixology.com,inbound,001,World,0 -rsys5.com,cottonon.com,inbound,001,World,0 -rsys5.com,ediblearrangements.com,inbound,001,World,0 -rsys5.com,emailworldmarket.com,inbound,001,World,0 -rsys5.com,farfetch.com,inbound,001,World,0 -rsys5.com,frhiemailcommunications.com,inbound,001,World,0 -rsys5.com,harryanddavid.com,inbound,001,World,0 -rsys5.com,hollandandbarrett.com,inbound,001,World,0 -rsys5.com,icing.com,inbound,001,World,0 -rsys5.com,indigo.ca,inbound,001,World,0 -rsys5.com,jabong.com,inbound,001,World,0 -rsys5.com,jcrew.com,inbound,001,World,0 -rsys5.com,jjill.com,inbound,001,World,0 -rsys5.com,kanui.com.br,inbound,001,World,0 -rsys5.com,kirklands.com,inbound,001,World,0 -rsys5.com,lanebryant.com,inbound,001,World,0 -rsys5.com,lazada.com,inbound,001,World,0 -rsys5.com,leapfrog.com,inbound,001,World,0 -rsys5.com,llbean.com,inbound,001,World,0 -rsys5.com,lojascolombo.com.br,inbound,001,World,0 -rsys5.com,madewell.com,inbound,001,World,0 -rsys5.com,magazineluiza.com.br,inbound,001,World,0 -rsys5.com,missguided.co.uk,inbound,001,World,0 -rsys5.com,moma.org,inbound,001,World,0 -rsys5.com,nationalgeographic.com,inbound,001,World,0 -rsys5.com,neat.com,inbound,001,World,0 -rsys5.com,newbalance.com,inbound,001,World,0 -rsys5.com,news-voeazul.com.br,inbound,001,World,0 -rsys5.com,nordstrom.com,inbound,001,World,0 -rsys5.com,normthompson.com,inbound,001,World,0 -rsys5.com,novomundo.com.br,inbound,001,World,0 -rsys5.com,ourdeal.com.au,inbound,001,World,0 -rsys5.com,pier1.com,inbound,001,World,0 -rsys5.com,postini.com,inbound,001,World,0.0697 -rsys5.com,productmadness.com,inbound,001,World,0 -rsys5.com,rainbowshops.com,inbound,001,World,0 -rsys5.com,rei.com,inbound,001,World,0 -rsys5.com,roadrunnersports.com,inbound,001,World,0 -rsys5.com,rosettastone.com,inbound,001,World,0 -rsys5.com,seamless.com,inbound,001,World,0 -rsys5.com,serenaandlily.com,inbound,001,World,0 -rsys5.com,smiles.com.br,inbound,001,World,0 -rsys5.com,soubarato.com.br,inbound,001,World,0 -rsys5.com,strava.com,inbound,001,World,0 -rsys5.com,submarino.com.br,inbound,001,World,0 -rsys5.com,thewalkingcompany.com,inbound,001,World,0 -rsys5.com,tigerdirect.com,inbound,001,World,0 -rsys5.com,udemy.com,inbound,001,World,0 -rsys5.com,vitaminshoppe.com,inbound,001,World,0 -rsys5.com,vpusa.com,inbound,001,World,0 -rsys5.com,walmart.com.br,inbound,001,World,0 -rsys5.com,worldofwatches.com,inbound,001,World,0 -rsys5.com,xfinity.com,inbound,001,World,0 -rue21email.com,rue21email.com,inbound,001,World,0 -rueducommerce.com,groupe-rueducommerce.fr,inbound,001,World,0 -rummycirclemails.com,eccluster.com,inbound,001,World,0 -runkeeper.com,runkeeper.com,inbound,001,World,0 -runnet.jp,runnet.jp,inbound,001,World,0 -runtastic.com,runtastic.com,inbound,001,World,0 -rutenmail.com.tw,rutenmail.com.tw,inbound,001,World,0 -ruum.com,ruum.com,inbound,001,World,0 -ryanairmail.com,ryanairmail.com,inbound,001,World,0 -rzone.de,rzone.de,inbound,001,World,1 -s3s-br1.net,splio.com.br,inbound,001,World,0.908187 -s3s-main.net,splio.com,inbound,001,World,0.970428 -s4s-pl1.pl,splio.com,inbound,001,World,0.939032 -saavn.com,saavn.com,inbound,001,World,1 -safe-sender.net,safe-sender.net,inbound,001,World,0 -safelistextreme.com,quantumsafelist.com,inbound,001,World,0 -safelistpro.com,safelistpro.com,inbound,001,World,0.00014 -safeway.com,chtah.com,inbound,001,World,0 -safeway.com,safeway.com,inbound,001,World,0.000382 -sahibinden.com,sahibinden.com,inbound,001,World,0 -sailthru.com,sailthru.com,inbound,001,World,0 -saimails.in,infimail.com,inbound,001,World,0 -sainsburys.co.uk,emv5.com,inbound,001,World,0 -saisoncard.co.jp,saisoncard.co.jp,inbound,001,World,0 -saks.com,saks.com,inbound,001,World,0 -saksoff5th.com,saksoff5th.com,inbound,001,World,0 -sakura.ne.jp,sakura.ne.jp,inbound,001,World,0.640465 -salememail.net,salememail.net,inbound,001,World,0 -salesforce.com,postini.com,inbound,001,World,0.866057 -salesforce.com,salesforce.com,inbound,001,World,0.983322 -salesforce.com,salesforce.com,outbound,001,World,1 -salesmanago.pl,salesmanago.pl,inbound,001,World,0 -salliemae.com,salliemae.com,inbound,001,World,1 -salsalabs.net,salsalabs.net,inbound,001,World,0 -samashmusic.com,wc09.net,inbound,001,World,0 -samsclub.com,m0.net,inbound,001,World,0 -samsung.com,samsung.com,inbound,001,World,0.043941 -samsung.ru,samsung.ru,inbound,001,World,0 -samsungusa.com,samsungusa.com,inbound,001,World,0 -sanmina-sci.com,postini.com,inbound,001,World,0.999991 -sanmina.com,postini.com,inbound,001,World,0.9996 -sans.org,sans.org,inbound,001,World,0.497629 -santander.cl,santander.cl,inbound,001,World,0.999992 -santander.cl,santandersantiago.cl,inbound,001,World,1 -sapnetworkmail.com,sap-ag.de,inbound,001,World,1 -sapo.pt,sapo.pt,inbound,001,World,0.242839 -sapo.pt,sapo.pt,outbound,001,World,0 -saramin.co.kr,saramin.co.kr,inbound,001,World,0 -sassieshop.com,sassieshop.com,inbound,001,World,0.025887 -saturday.com,saturday.com,inbound,001,World,0 -savelivefresh.com,livesavemail.com,inbound,001,World,1 -savingdeals.in,infimail.com,inbound,001,World,0 -savingstar.com,savingstar.com,inbound,001,World,1 -sbcglobal.net,yahoo.{...},inbound,001,World,0.999991 -sbcglobal.net,yahoodns.net,outbound,001,World,1 -sbi.co.in,sbi.co.in,inbound,001,World,0 -sbr-inc.co.jp,hdemail.jp,inbound,001,World,0.000528 -sc.com,messagelabs.com,inbound,001,World,0.996156 -sc.com,sc.com,inbound,001,World,0.99683 -schwab.com,schwab.com,inbound,001,World,0.025055 -scmp.com,emarsys.net,inbound,001,World,0 -scoop.it,scoop.it,inbound,001,World,0 -scoopon.com.au,inxserver.de,inbound,001,World,1 -screwfix.info,fwdto.net,inbound,001,World,0 -sears.ca,sears.ca,inbound,001,World,0.005447 -searscard.com,searscard.com,inbound,001,World,1 -seaworld.com,seaworld.com,inbound,001,World,0 -secretescapes.com,secretescapes.com,inbound,001,World,0 -secure.ne.jp,secure.ne.jp,inbound,001,World,0.000356 -securence.com,securence.com,inbound,001,World,0.667957 -secureserver.net,secureserver.net,inbound,001,World,3.6e-05 -seek.com.au,seek.com.au,inbound,001,World,0 -seekingalpha.com,seekingalpha.com,inbound,001,World,1 -seekingalpha.com,sendgrid.net,inbound,001,World,1 -selectacast.net,selectacast.net,inbound,001,World,0.000561 -selection-priceminister.com,selection-priceminister.com,inbound,001,World,0 -semana.com,semana.com,inbound,001,World,0.005867 -senate.gov,senate.gov,inbound,001,World,0.992994 -sendearnings.com,sendearnings.com,inbound,001,World,0 -sender.lt,sritis.lt,inbound,001,World,0.000352 -sendgrid.info,sendgrid.net,inbound,001,World,0.999966 -sendgrid.me,sendgrid.net,inbound,001,World,1 -sendlane.com,sendlane.com,inbound,001,World,0 -sendpal.in,sendpal.in,inbound,001,World,0 -sendsmaily.info,sendsmaily.info,inbound,001,World,0 -seniorplanet.fr,seniorplanet.fr,inbound,001,World,0 -serpadres.es,chtah.net,inbound,001,World,0 -service-now.com,postini.com,inbound,001,World,0.988878 -service-now.com,service-now.com,inbound,001,World,0.998853 -serviciobancomer.com,serviciobancomer.com,inbound,001,World,0 -seznam.cz,seznam.cz,inbound,001,World,0.001781 -seznam.cz,seznam.cz,outbound,001,World,0.001741 -sfid01.com,sfid01.com,inbound,001,World,0 -sfimg.com,sfimarketing.com,inbound,001,World,0 -sfly.com,shutterfly.com,inbound,001,World,0 -sfr.fr,sfr.fr,inbound,001,World,0.236539 -sfr.fr,sfr.fr,outbound,001,World,0.004753 -shaadi.com,shaadi.com,inbound,001,World,0.000154 -shadowshopper.com,shadowshopper.com,inbound,001,World,5.6e-05 -shaw.ca,shaw.ca,inbound,001,World,0 -shaw.ca,shaw.ca,outbound,001,World,1 -sheplers.com,sheplers.com,inbound,001,World,0 -shiftplanning.com,shiftplanning.com,inbound,001,World,0 -shiksha.com,shiksha.com,inbound,001,World,0 -shinseibank.com,shinseibank.com,inbound,001,World,0 -shoedazzle.com,shoedazzle.com,inbound,001,World,0 -shoes.com,famousfootwear.com,inbound,001,World,0 -shop2gether.com.br,shop2gether.com.br,inbound,001,World,0 -shopbonton.com,shopbonton.com,inbound,001,World,0 -shopcluesemail.com,shopcluesemail.com,inbound,001,World,0 -shopcluesmail.com,shopcluesmail.com,inbound,001,World,0 -shophq.com,shophq.com,inbound,001,World,0 -shopjustice.com,shopjustice.com,inbound,001,World,0 -shopkick.com,shopkick.com,inbound,001,World,0 -shopko.com,shopko.com,inbound,001,World,0 -shopnineteenmails.in,iaires.com,inbound,001,World,0 -shoppersoptimum.ca,thindata.net,inbound,001,World,0 -shoppersstop.com,shoppersstop.com,inbound,001,World,0 -shoprite-email.com,email-mywebgrocer.com,inbound,001,World,0 -shoptime.com,shoptime.com,inbound,001,World,0 -shopto.net,shopto.net,inbound,001,World,1 -showingtime.com,showingtime.com,inbound,001,World,0 -showroomprive.com,showroomprive.be,inbound,001,World,0 -showroomprive.com,showroomprive.com,inbound,001,World,0.007102 -showroomprive.com,showroomprive.nl,inbound,001,World,0 -showroomprive.es,showroomprive.es,inbound,001,World,0 -showroomprive.it,showroomprive.pt,inbound,001,World,0 -showroomprive.pt,showroomprive.co.uk,inbound,001,World,0 -shtyle.fm,shtyle.fm,inbound,001,World,0 -shukatsu.jp,shukatsu.jp,inbound,001,World,0 -siella.jp,siella.jp,inbound,001,World,0.000199 -sierratradingpost.com,sierratradingpost.com,inbound,001,World,0.000885 -sigmabeauty.com,lstrk.net,inbound,001,World,1 -sii.cl,sii.cl,inbound,001,World,0 -simplesafelist.com,adminforfree.com,inbound,001,World,1 -simpletextadz.com,web-hosting.com,inbound,001,World,1 -simplyhired.com,simplyhired.com,inbound,001,World,0 -simplymarry.com,tbsl.in,inbound,001,World,0 -singsale.com.sg,singsale.com.sg,inbound,001,World,0 -siriusxm.com,xmradio.com,inbound,001,World,0 -sitecore-mailer.com,sendlabs.com,inbound,001,World,0 -sittercity.com,sittercity.com,inbound,001,World,0 -sixflags.com,sixflags.com,inbound,001,World,0 -skillpages-mailer.com,dynect.net,inbound,001,World,0 -skillpages-mailer.com,sendlabs.com,inbound,001,World,0 -skillpages-mailer.com,skillpagesmail.com,inbound,001,World,0 -sky.com,sky.com,inbound,001,World,8.8e-05 -skymall.com,skymall.com,inbound,001,World,0.001135 -skynet.be,belgacom.be,inbound,001,World,0.005903 -skype.com,delivery.net,inbound,001,World,0 -skype.com,skype.com,inbound,001,World,0 -skyscanner.net,skyscanner.net,inbound,001,World,1 -sld.cu,sld.cu,inbound,001,World,1 -slickdeals.net,slickdeals.net,inbound,001,World,4.9e-05 -slidesharemail.com,newslettergrid.com,inbound,001,World,1 -slidesharemail.com,slideshare.net,inbound,001,World,1 -slidesharemail.com,slidesharemail.com,inbound,001,World,1 -smartbrief.com,smartbrief.com,inbound,001,World,0 -smartdraw.com,smartdraw.com,inbound,001,World,0 -smartertravel.com,smartertravelmedia.com,inbound,001,World,0.021388 -smartphoneexperts.com,mailgun.net,inbound,001,World,1 -smartresponder.ru,smartresponder.ru,inbound,001,World,1 -smp.ne.jp,smp.ne.jp,inbound,001,World,0 -snagajob-email.com,snagajob-email.com,inbound,001,World,0 -snapdeal.com,snapdeal.com,inbound,001,World,0 -snapdealmail.in,snapdealmail.in,inbound,001,World,0 -snaphire.com,snaphire.com,inbound,001,World,0 -snapretail.com,snapretail.com,inbound,001,World,1 -socialappsmail.com,socialappsmail.com,inbound,001,World,1 -socialsex.biz,infinitypersonals.com,inbound,001,World,0 -sofmap.com,sofmap.com,inbound,001,World,0.0092 -softbank.jp,softbank.jp,inbound,001,World,0 -softbank.jp,softbank.jp,outbound,001,World,0 -softbank.ne.jp,softbank.ne.jp,inbound,001,World,0 -softbank.ne.jp,softbank.ne.jp,outbound,001,World,0 -solesociety.com,bronto.com,inbound,001,World,0 -solosenders.com,megasenders.com,inbound,001,World,8.2e-05 -solosenders.com,traxweb.org,inbound,001,World,0 -solveerrors.com,infimail.com,inbound,001,World,0 -soma.com,soma.com,inbound,001,World,0 -someecards.com,someecards.com,inbound,001,World,0 -songkick.com,songkick.com,inbound,001,World,1 -sony-latin.com,sony-latin.com,inbound,001,World,0.01735 -sony.com,sony.com,inbound,001,World,0.020225 -sony.jp,sony.jp,inbound,001,World,0.000654 -sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,001,World,0 -sonyrewards.com,sonyrewards.com,inbound,001,World,0 -soundcloudmail.com,soundcloudmail.com,inbound,001,World,0.999996 -sourceforge.net,sourceforge.net,inbound,001,World,1 -sourcenext.info,sourcenext.info,inbound,001,World,0 -southwest.com,southwest.com,inbound,001,World,0 -sp.gov.br,sp.gov.br,inbound,001,World,0.485842 -spanishdict.com,spanishdict.com,inbound,001,World,0.002698 -spareroom.co.uk,spareroom.co.uk,inbound,001,World,1 -sparklist.com,sparklist.com,inbound,001,World,0 -sparkpeople.com,sparkpeople.com,inbound,001,World,0 -spartoo.com,spartoo.com,inbound,001,World,0 -spectersoft.com,spectersoft.com,inbound,001,World,0 -speedyrewards-email.com,speedyrewards-email.com,inbound,001,World,0 -spencersonline.com,spencersonline.com,inbound,001,World,0 -spiritairlines.com,ctd004.net,inbound,001,World,0 -spiritairlines.com,ctd005.net,inbound,001,World,0 -splitwise.com,splitwise.com,inbound,001,World,1 -sportlobster.com,sportlobster.com,inbound,001,World,1 -sportsdirect.com,sportsdirect.com,inbound,001,World,0 -sportsline.com,cbsig.net,inbound,001,World,1 -sportsmansguide.com,sportsmansguide.com,inbound,001,World,0.736903 -spotifymail.com,spotifymail.com,inbound,001,World,1 -sprint.com,m0.net,inbound,001,World,0 -sqlservercentral.com,sqlservercentral.com,inbound,001,World,0 -square-enix.com,messagelabs.com,inbound,001,World,0.004493 -squareup.com,squareup.com,inbound,001,World,0.986452 -ssgadm.com,ssg.com,inbound,001,World,0 -staffeazymailers.com,iaires.com,inbound,001,World,0 -stakemail.com,iaires.com,inbound,001,World,0 -stampmail.in,iaires.com,inbound,001,World,0 -standaard.be,vummail.be,inbound,001,World,0 -standardbank.co.za,standardbank.co.za,inbound,001,World,0 -stanford.edu,highwire.org,inbound,001,World,1 -stanford.edu,stanford.edu,inbound,001,World,0.93025 -stansberryresearch.com,stansberry-re.net,inbound,001,World,0 -stansberryresearch.com,stansberryresearch.com,inbound,001,World,0.001008 -staples-pt.com,1-hostingservice.com,inbound,001,World,0 -staples.co.uk,ncrwebhost.de,inbound,001,World,0 -staples.com,staples.com,inbound,001,World,0.00638 -starbucks.com,iphmx.com,inbound,001,World,0.999476 -starbucks.com,starbucks.com,inbound,001,World,0 -stardockcorporation.com,stardockcorporation.com,inbound,001,World,0 -stardockentertainment.info,stardockentertainment.info,inbound,001,World,0 -starsports.com,eccluster.com,inbound,001,World,0 -startribune.com,startribune.com,inbound,001,World,0.001728 -startwire.com,jobsreport.com,inbound,001,World,1 -startwire.com,startwire.com,inbound,001,World,1 -starwoodhotels.com,outlook.com,inbound,001,World,1 -state-of-the-art-mailer.com,futurebanners.net,inbound,001,World,0 -state.gov,state.gov,inbound,001,World,0 -statefarm.com,statefarm.com,inbound,001,World,1 -stayfriends.de,stayfriends.de,inbound,001,World,0 -steampowered.com,steampowered.com,inbound,001,World,1 -steinmart.com,steinmart.com,inbound,001,World,0 -stelladot.com,stelladot.com,inbound,001,World,0 -stepstone.de,stepstone.com,inbound,001,World,0.001689 -stevemadden.com,stevemadden.com,inbound,001,World,0 -stjobs.sg,st701.com,inbound,001,World,1 -stjude.org,stjude.org,inbound,001,World,0.006716 -strava.com,strava.com,inbound,001,World,1 -streetauthoritydaily.com,streetauthoritydaily.com,inbound,001,World,0 -streeteasy.com,streeteasy.com,inbound,001,World,0 -striata.com,striata.com,inbound,001,World,0 -stubhub.com,stubhub.com,inbound,001,World,1 -studentbeans.com,emv8.com,inbound,001,World,0 -stumblemail.com,stumblemail.com,inbound,001,World,1 -stylecareers.com,stylecareers.com,inbound,001,World,0 -suafaturanet.com.br,suafaturanet.com.br,inbound,001,World,0.995846 -subito.it,subito.it,inbound,001,World,0 -subscribe.ru,subscribe.ru,inbound,001,World,0 -subscribermail.com,subscribermail.com,inbound,001,World,0 -subtend.info,subtend.info,inbound,001,World,1 -subway.com,subway.com,inbound,001,World,0 -sungard.com,postini.com,inbound,001,World,0.499946 -sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,001,World,1 -superbalist.com,sailthru.com,inbound,001,World,0 -superdeal.com.ua,mailersend.com,inbound,001,World,0 -superdrug.com,superdrug.com,inbound,001,World,0 -superjob.ru,superjob.ru,inbound,001,World,0 -supersafemailer.com,zoothost.com,inbound,001,World,0 -support-love.com,support-love.com,inbound,001,World,0 -supremelist.com,onlinehome-server.com,inbound,001,World,1 -surfmandelivery.com,surfmandelivery.com,inbound,001,World,0 -surlatable.com,surlatable.com,inbound,001,World,0 -surveyhelpcenter.com,jsmtp.net,inbound,001,World,0 -surveyjobopportunities.com,surveyjobopportunities.com,inbound,001,World,3.7e-05 -surveymonkey.com,surveymonkey.com,inbound,001,World,0 -surveysavvy.com,surveysavvy.com,inbound,001,World,0 -surveyspot.com,ssisurveys.com,inbound,001,World,0 -sut1.co.uk,sut1.co.uk,inbound,001,World,0.001789 -sut5.co.uk,sut5.co.uk,inbound,001,World,0 -swanson-vitamins.com,emv5.com,inbound,001,World,0 -sweepstakesalerts.com,sweepstakesalerts.com,inbound,001,World,0 -swimoutlet.com,isport.com,inbound,001,World,0 -sylectus.com,sylectus.com,inbound,001,World,0.361297 -sympatico.ca,hotmail.{...},inbound,001,World,1 -sympatico.ca,hotmail.{...},outbound,001,World,1 -synchronyfinancial.com,bigfootinteractive.com,inbound,001,World,0 -synergy360.jp,crmstyle.com,inbound,001,World,0 -t-online.de,t-online.de,inbound,001,World,1 -t-online.de,t-online.de,outbound,001,World,0.999939 -tadtopmails.com,tadtopmails.com,inbound,001,World,0 -taggedmail.com,taggedmail.com,inbound,001,World,0 -taipeifubon.com.tw,taipeifubon.com.tw,inbound,001,World,0 -taishinbank.com.tw,taishinbank.com.tw,inbound,001,World,0 -take2games.com,take2games.com,inbound,001,World,0 -talkmatch.com,talkmatch.com,inbound,001,World,0 -tamu.edu,tamu.edu,inbound,001,World,0.249656 -tanga.com,tanga.com,inbound,001,World,0 -tangeroutletsusa.com,bronto.com,inbound,001,World,0 -tanningmail.com,tanningmail.com,inbound,001,World,0 -tappingsolutionemail.com,tappingsolutionemail.com,inbound,001,World,0 -target-safelist.com,safelistpro.com,inbound,001,World,0.000215 -target.com,bigfootinteractive.com,inbound,001,World,0 -targetproblaster.com,targetproblaster.com,inbound,001,World,0 -targetx.com,targetx.com,inbound,001,World,0 -tarot.com,tarot.com,inbound,001,World,0 -tastefullysimpleparty.com,bigfootinteractive.com,inbound,001,World,0 -tasteofhome.com,tasteofhome.com,inbound,001,World,0 -tastingtable.com,tastingtable.com,inbound,001,World,0 -tatrabanka.sk,tatrabanka.sk,inbound,001,World,0 -taxi4sure.net,infimail.com,inbound,001,World,0 -tchibo.com.tr,euromsg.net,inbound,001,World,0 -tchibo.de,srv2.de,inbound,001,World,0.980445 -teach12.net,teach12.net,inbound,001,World,0 -teambuymail.com,teambuymail.com,inbound,001,World,0 -teamo.ru,teamo.ru,inbound,001,World,0 -teamsnap.com,teamsnap.com,inbound,001,World,1 -teamviewer.com,teamviewer.com,inbound,001,World,0 -teapartyinfo.org,teapartyinfo.org,inbound,001,World,0 -techgig.com,tbsl.in,inbound,001,World,0 -technolutions.net,technolutions.net,inbound,001,World,1 -techtarget.com,techtarget.com,inbound,001,World,0.001771 -telefonica.com,telefonica.com,inbound,001,World,0.998662 -telegraph.co.uk,telegraph.co.uk,inbound,001,World,0 -telenet.be,telenet-ops.be,inbound,001,World,0.000128 -teleportmyjob.com,clara.net,inbound,001,World,0 -telus.com,telus.com,inbound,001,World,0.000118 -telus.net,telus.net,inbound,001,World,0.006226 -templeandwebster.com.au,templeandwebster.com.au,inbound,001,World,5e-06 -ten24mail.com,ten24mail.com,inbound,001,World,0 -terra.com,terra.com,inbound,001,World,0.000463 -terra.com.br,terra.com,inbound,001,World,0 -terra.com.br,terra.com,outbound,001,World,0 -tesco.com,tesco.com,inbound,001,World,0 -testfunda.com,testfunda.com,inbound,001,World,0 -texasjobdepartment.com,texasjobdepartment.com,inbound,001,World,0 -textnow.me,textnow.me,inbound,001,World,0 -tgw.com,tgw.com,inbound,001,World,0 -theanimalrescuesite.com,theanimalrescuesite.com,inbound,001,World,0 -theatermania.com,wc09.net,inbound,001,World,0 -thebay.com,thebay.com,inbound,001,World,0 -thebodyshop-usa.com,email-bodyshop.com,inbound,001,World,0 -thebodyshop-usa.com,postdirect.com,inbound,001,World,0 -thecarousell.com,thecarousell.com,inbound,001,World,1 -thegrommet.com,lstrk.net,inbound,001,World,1 -theguardian.com,theguardian.com,inbound,001,World,0 -thehut.com,thehut.com,inbound,001,World,0 -theleadmagnet.com,your-server.de,inbound,001,World,1 -thelimited.com,thelimited.com,inbound,001,World,0 -themailbagsafelist.com,thomas-j-brown.com,inbound,001,World,0 -theoutnet.com,theoutnet.com,inbound,001,World,0 -thepamperedchef.com,thepamperedchef.com,inbound,001,World,0 -thephonehouse.es,splio.com,inbound,001,World,0.983578 -thephonehouse.es,splio.es,inbound,001,World,0.983266 -therealreal.com,email-realreal.com,inbound,001,World,0 -theskimm.com,theskimm.com,inbound,001,World,0 -thesource.ca,thesource.ca,inbound,001,World,0.00923 -thesovereigninvestor.com,sovereignsociety.com,inbound,001,World,0 -thewarehouse.co.nz,thewarehouse.co.nz,inbound,001,World,0 -thinkgeek.com,thinkgeek.com,inbound,001,World,1e-06 -thinkvidya.com,thinkvidya.com,inbound,001,World,0 -thirtyonegifts.com,thirtyonegifts.com,inbound,001,World,0 -thomascook.com,eccluster.com,inbound,001,World,0 -thoughtful-mind.com,thoughtful-mind.com,inbound,001,World,0 -thumbtack.com,thumbtack.com,inbound,001,World,1 -ticketmaster.com,ticketmaster.com,inbound,001,World,0.251337 -ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,001,World,0 -ticketmonster.co.kr,ticketmonster.co.kr,inbound,001,World,0 -timehop.com,timehop.com,inbound,001,World,1 -timeout.com,ec-cluster.com,inbound,001,World,0 -timesjobs.com,tbsl.in,inbound,001,World,0 -timesjobsmail.com,tbsl.in,inbound,001,World,0 -timesofindia.com,indiatimes.com,inbound,001,World,0 -timewarnercable.com,bigfootinteractive.com,inbound,001,World,0 -timeweb.ru,timeweb.ru,inbound,001,World,1 -tinyletterapp.com,tinyletterapp.com,inbound,001,World,0 -tiscali.it,tiscali.it,outbound,001,World,0 -tmart.com,chtah.net,inbound,001,World,0 -tmp.com,tmpw.com,inbound,001,World,0 -tobi.com,messagebus.com,inbound,001,World,0 -tobizaru.jp,tobizaru.jp,inbound,001,World,0 -tocoo.jp,aics.ne.jp,inbound,001,World,0 -toluna.com,toluna.com,inbound,001,World,0 -tomtommailer.com,tomtommailer.com,inbound,001,World,0 -topface.com,topface.com,inbound,001,World,1e-06 -topica.com,topica-silver-y.com,inbound,001,World,0 -topqpon.si,topqpon.si,inbound,001,World,0 -topspin.net,topspin.net,inbound,001,World,1 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 -touchbase2.com,infimail.com,inbound,001,World,0 -touchbase2.com,mailurja.com,inbound,001,World,0 -touchbasepro.com,touchbasepro.com,inbound,001,World,0 -tower.jp,tower.jp,inbound,001,World,0 -townhallmail.com,townhallmail.com,inbound,001,World,0 -townnews-mail.com,townnews-mail.com,inbound,001,World,0 -townsquaremedia.info,sailthru.com,inbound,001,World,0 -toysrus.com,epsl1.com,inbound,001,World,0 -trabajar.com,trabajo.org,inbound,001,World,0.999997 -trabalhar.com,trabajo.org,inbound,001,World,0.999994 -tradeloop.com,tradeloop.com,inbound,001,World,1 -trademe.co.nz,trademe.co.nz,inbound,001,World,0.991916 -trafficboostermailer.com,trafficboostermailer.com,inbound,001,World,1 -trafficleads2incomevm.com,zoothost.com,inbound,001,World,0 -trafficprolist.com,thomas-j-brown.com,inbound,001,World,0 -trafficwave.net,trafficwave.net,inbound,001,World,0 -transittraveljobinsider.com,transittraveljobinsider.com,inbound,001,World,0 -transportexchangegroup.com,transportexchangegroup.com,inbound,001,World,0.998703 -transversal.net,transversal.net,inbound,001,World,1 -travelchannel.com,travelchannel.com,inbound,001,World,0 -travelocity.com,travelocity.com,inbound,001,World,0.000194 -travelzoo.com,travelzoo.com,inbound,001,World,0 -travisperkins.co.uk,travisperkins.co.uk,inbound,001,World,0 -trclient.com,trclient.com,inbound,001,World,0 -treemall.com.tw,symphox.com,inbound,001,World,0 -trello.com,mandrillapp.com,inbound,001,World,1 -trialsmith.com,membercentral.com,inbound,001,World,0 -tricaemidia.com.br,tricaemidia.com.br,inbound,001,World,0 -triongames.com,triongames.com,inbound,001,World,0.085718 -tripadvisor.com,tripadvisor.com,inbound,001,World,0.000588 -tripit.com,tripit.com,inbound,001,World,0 -tripolis.com,tripolis.com,inbound,001,World,0 -trovit.com,trovit.com,inbound,001,World,0 -trulia.com,trulia.com,inbound,001,World,0 -tsite.jp,tsite.jp,inbound,001,World,0 -tsmmail.com,tsmmail.com,inbound,001,World,0 -tsutaya.co.jp,tsutaya.co.jp,inbound,001,World,0 -tucasa.com,grupodtm.com,inbound,001,World,0 -tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,001,World,0 -tumblr.com,tumblr.com,inbound,001,World,1 -turbine.com,turbine.com,inbound,001,World,0.013592 -turkcell.com.tr,turkcell.com.tr,inbound,001,World,0.043492 -turner.com,cnn.com,inbound,001,World,0 -twe-safelist.com,adminforfree.com,inbound,001,World,1 -twitch.tv,justin.tv,inbound,001,World,0 -twitter.com,postini.com,inbound,001,World,0.765163 -twitter.com,twitter.com,inbound,001,World,0.999969 -twoomail.com,netlogmail.com,inbound,001,World,0 -twoomail.com,twoomail.com,inbound,001,World,0 -type.jp,type.jp,inbound,001,World,0 -u-shopping.com.tw,u-shopping.com.tw,inbound,001,World,0 -uber.com,uber.com,inbound,001,World,1 -ubi.com,ubi.com,inbound,001,World,0 -ubivox.com,ubivox.com,inbound,001,World,0.956275 -ubuntu.com,canonical.com,inbound,001,World,0.000129 -ucla.edu,ucla.edu,inbound,001,World,0.806204 -udnpaper.com,udnpaper.com,inbound,001,World,0.998858 -udnshopping.com,udnshopping.com,inbound,001,World,0 -uga.edu,outlook.com,inbound,001,World,1 -uhaul.com,uhaul.com,inbound,001,World,0.997947 -uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,001,World,0 -uiuc.edu,illinois.edu,inbound,001,World,0.999815 -ukr.net,fwdcdn.com,inbound,001,World,1 -ukr.net,ukr.net,outbound,001,World,0.999992 -ulta.com,exacttarget.com,inbound,001,World,0 -ulta.com,ulta.com,inbound,001,World,0.034861 -ulteem.com,ulteem.com,inbound,001,World,0 -ultimateadsites.net,ultimateadsites.net,inbound,001,World,1 -umd.edu,umd.edu,inbound,001,World,0.021709 -umn.edu,umn.edu,inbound,001,World,0.966992 -umpiredigital.com,inboxmarketer.com,inbound,001,World,0 -unilever.com,mxlogic.net,inbound,001,World,1 -uniqlo-usa.com,uniqlo-usa.com,inbound,001,World,0 -united.com,coair.com,inbound,001,World,1 -united.com,united.com,inbound,001,World,0 -unitedrepublic.org,unitedrepublic.org,inbound,001,World,1 -universalorlando.com,universalorlando.com,inbound,001,World,0 -unosinsidersclub.com,unosinsidersclub.com,inbound,001,World,0 -unrollmail.com,unrollmail.com,inbound,001,World,1 -uol.com.br,uol.com.br,inbound,001,World,0 -uol.com.br,uol.com.br,outbound,001,World,0 -upenn.edu,upenn.edu,inbound,001,World,0.16521 -upromise.com,delivery.net,inbound,001,World,0 -ups.com,ups.com,inbound,001,World,0.997955 -urbanoutfitters.com,freepeople.com,inbound,001,World,0 -urx.com.br,urx.com.br,inbound,001,World,0 -usaa.com,usaa.com,inbound,001,World,0.639434 -usafisnews.org,usafisnews.org,inbound,001,World,0 -usahockey-email.com,usahockey-email.com,inbound,001,World,0 -usajobs.gov,opm.gov,inbound,001,World,0.931948 -usc.edu,usc.edu,inbound,001,World,0.300314 -uscourts.gov,uscourts.gov,inbound,001,World,0 -uslargestsafelist.com,zoothost.com,inbound,001,World,0 -usndr.com,unisender.com,inbound,001,World,1 -usndr.com,usndr.com,inbound,001,World,1 -usp.br,usp.br,inbound,001,World,0.044176 -usps.com,usps.gov,inbound,001,World,0.909588 -usps.gov,usps.gov,inbound,001,World,0.940315 -ustream.tv,mailgun.net,inbound,001,World,1 -ustream.tv,mailgun.us,inbound,001,World,1 -usx.com.br,uqx.com.br,inbound,001,World,0 -usx.com.br,usx.com.br,inbound,001,World,0 -usx.com.br,utx.com.br,inbound,001,World,0 -utfsm.cl,utfsm.cl,inbound,001,World,0.002318 -utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,001,World,0 -utx.com.br,utx.com.br,inbound,001,World,0 -uvarosa.com.br,uvarosa.com.br,inbound,001,World,0 -vacationstogo.com,vacationstogo.com,inbound,001,World,0 -vagas.com.br,vagas.com.br,inbound,001,World,0 -vakifbank.com.tr,vakifbank.com.tr,inbound,001,World,1 -valuedopinions.co.uk,researchnow-usa.com,inbound,001,World,1 -vanheusenrewards.com,vanheusenrewards.com,inbound,001,World,0 -vd.nl,emsecure.net,inbound,001,World,0 -velocityfrequentflyer.com,virginaustralia.com,inbound,001,World,0.003442 -venca.es,eccluster.com,inbound,001,World,0 -venere.com,kiwari.com,inbound,001,World,0 -vente-exclusive.com,vente-exclusive.com,inbound,001,World,0.000463 -venteprivee.com,venteprivee.com,inbound,001,World,0 -venusswimwear.net,venusswimwear.net,inbound,001,World,0 -verabradleymail.com,verabradleymail.com,inbound,001,World,0 -verizon.com,verizon.com,inbound,001,World,0.999793 -verizon.net,verizon.net,inbound,001,World,0.00713 -verizon.net,verizon.net,outbound,001,World,0 -verizon.net,yahoo.{...},inbound,001,World,0.999435 -verizonwireless.com,verizonwireless.com,inbound,001,World,0.972245 -vfoutletvip.com,vfoutletvip.com,inbound,001,World,0 -vhmnetworkemail.com,jobdiagnosis.com,inbound,001,World,0 -viagogo.com,chtah.net,inbound,001,World,0 -viajanet.com.br,viajanet.com.br,inbound,001,World,0.001345 -vicinity.nl,picsrv.net,inbound,001,World,0.022107 -victoriassecret.com,victoriassecret.com,inbound,001,World,0 -videotron.ca,videotron.ca,inbound,001,World,0.005886 -vietcombank.com.vn,vietcombank.com.vn,inbound,001,World,1 -vikingrivercruises.com,bfi0.com,inbound,001,World,0 -vimeo.com,vimeo.com,inbound,001,World,0 -vindale.com,vindale.com,inbound,001,World,0 -vipvoice.com,npdor.com,inbound,001,World,0 -viraladmagnet.com,viraladmagnet.com,inbound,001,World,1 -viralsender.com,viralsender.com,inbound,001,World,0 -virgilio.it,virgilio.net,inbound,001,World,0 -virginia.edu,virginia.edu,inbound,001,World,0.055153 -virtualtarget.com.br,virtualtarget.com.br,inbound,001,World,0 -vistaprint.com,vistaprint.com,inbound,001,World,0 -vistaprint.com.au,vistaprint.com.au,inbound,001,World,0 -visualsoft.co.uk,visualsoft.co.uk,inbound,001,World,0 -vitacost.com,vitacost.com,inbound,001,World,0 -vitaladviews.com,zoothost.com,inbound,001,World,0 -vitaminworld.com,email-nbtyinc.com,inbound,001,World,0 -vivastreet.com,viwii.net,inbound,001,World,0 -vk.com,vkontakte.ru,inbound,001,World,0 -vocus.com,vocus.com,inbound,001,World,0 -vodafone.com,vodafone.in,inbound,001,World,0.617518 -vonage.com,vonagenetworks.net,inbound,001,World,0.006531 -votervoice.net,votervoice.net,inbound,001,World,0 -vouchercloud.com,vouchercloud.com,inbound,001,World,0 -vovici.com,vovici.com,inbound,001,World,0 -voyageprive.com,cccampaigns.net,inbound,001,World,0 -voyageprive.es,ccmdcampaigns.net,inbound,001,World,0 -voyageprive.it,ccmdcampaigns.net,inbound,001,World,0 -voyages-sncf.com,neolane.net,inbound,001,World,0 -vpass.ne.jp,clickmailer.jp,inbound,001,World,0 -vpcontact.com,vpcontact.com,inbound,001,World,0 -vresp.com,verticalresponse.com,inbound,001,World,0 -vt.edu,vt.edu,inbound,001,World,0.978548 -vtext.com,vtext.com,inbound,001,World,0 -vtext.com,vtext.com,outbound,001,World,1 -vudu.com,vudu.com,inbound,001,World,0 -vueling.com,vueling.com,inbound,001,World,0 -vuezone.com,vuezone.com,inbound,001,World,0 -vzwpix.com,vtext.com,inbound,001,World,0 -wagjag.com,wagjag.com,inbound,001,World,0 -walgreens.com,walgreens.com,inbound,001,World,0.006128 -wallst.com,wallst.com,inbound,001,World,0 -wallstreetdaily.com,wallstreetdaily.com,inbound,001,World,0 -walmart.ca,walmart.ca,inbound,001,World,0 -walmart.com,walmart.com,inbound,001,World,0.315059 -wanadoo.fr,orange.fr,inbound,001,World,0 -wanadoo.fr,orange.fr,outbound,001,World,0 -warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,001,World,0 -waterstones.com,waterstones.com,inbound,001,World,0 -wattpad.com,wattpad.com,inbound,001,World,1 -waves-audio.com,emv8.com,inbound,001,World,0 -way2movies.net,way2movies.net,inbound,001,World,0 -way2sms.biz,way2sms.biz,inbound,001,World,0 -way2sms.in,way2sms.in,inbound,001,World,0 -way2smsemail.com,way2smsemail.com,inbound,001,World,0 -way2smsemails.com,way2smsemails.com,inbound,001,World,0 -way2smsmail.in,way2smsmail.in,inbound,001,World,0 -way2smsmails.com,way2smsmails.com,inbound,001,World,0 -wayfair.com,csnstores.com,inbound,001,World,0 -wealthyaffiliate.com,wealthyaffiliate.com,inbound,001,World,1 -web.de,web.de,inbound,001,World,0.999986 -web.de,web.de,outbound,001,World,1 -webcas.net,webcas.net,inbound,001,World,0 -webmd.com,webmd.com,inbound,001,World,2e-06 -webs.com,epsl1.com,inbound,001,World,0 -websaver.ca,websaver.ca,inbound,001,World,0 -websitesettings.com,stabletransit.com,inbound,001,World,0 -websitewelcome.com,websitewelcome.com,inbound,001,World,1 -webstars2k.com,webstars2k.com,inbound,001,World,1 -wechat.com,qq.com,inbound,001,World,1 -weebly.com,weeblymail.com,inbound,001,World,0 -wegottickets.com,wegottickets.com,inbound,001,World,0 -weheartit.com,weheartit.com,inbound,001,World,1 -wehkamp.nl,wehkamp.nl,inbound,001,World,0 -wellsfargo.com,wellsfargo.com,inbound,001,World,1.0 -wellsfargoadvisors.com,wellsfargo.com,inbound,001,World,1 -wemakeprice.com,wemakeprice.com,inbound,001,World,0 -westelm.com,westelm.com,inbound,001,World,0 -westmarine.com,westmarine.com,inbound,001,World,0 -westwing.com.br,cust-cluster.com,inbound,001,World,0 -westwing.es,ecm-cluster.com,inbound,001,World,0 -westwing.ru,ecm-cluster.com,inbound,001,World,0 -wetransfer.com,wetransfer.com,inbound,001,World,0.999751 -wetsealnewsletter.com,wetsealnewsletter.com,inbound,001,World,0 -wgbh.org,wgbh.org,inbound,001,World,0 -whaakky.com,whaakky.com,inbound,001,World,0 -whatcounts.com,wc09.net,inbound,001,World,0 -whentowork.com,whentowork.com,inbound,001,World,0 -whereareyounow.com,wayn.net,inbound,001,World,0 -whitehouse.gov,whitehouse.gov,inbound,001,World,0 -whitelabelpros.com,whitelabelpros.com,inbound,001,World,0 -wiggle.com,wiggle.com,inbound,001,World,0 -wikia.com,wikia.com,inbound,001,World,0.550228 -wikimedia.org,wikimedia.org,inbound,001,World,0 -william-reed.com,neolane.net,inbound,001,World,0 -williamhill.com,williamhill.com,inbound,001,World,0 -williams-sonoma.com,williams-sonoma.com,inbound,001,World,0 -windstream.net,windstream.net,inbound,001,World,0.002172 -wine.com,wine.com,inbound,001,World,0 -winkalmail.com,fnbox.com,inbound,001,World,0 -wisdomitservices.com,infimail.com,inbound,001,World,0 -wldemail-mailer.com,wldemail.com,inbound,001,World,0 -wldemail.com,emarsys.net,inbound,001,World,0 -wmtransfer.com,wmtransfer.com,inbound,001,World,0 -wnd.com,emv4.net,inbound,001,World,0 -wnd.com,worldnetdaily.com,inbound,001,World,0 -wolfmedia.us,wolfmedia.us,inbound,001,World,0 -womanwithin.com,womanwithin.com,inbound,001,World,0 -woodforest.com,woodforest.com,inbound,001,World,1 -wordfly.com,wordfly.com,inbound,001,World,0 -work.ua,work.ua,inbound,001,World,1 -workcircle.com,workcircle.net,inbound,001,World,0 -workhunter.net,workhunter.net,inbound,001,World,1 -workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,001,World,0 -worldsingles.co,worldsingles.com,inbound,001,World,0 -worldwinner.com,worldwinner.com,inbound,001,World,0.112191 -wotif.com,whatcounts.com,inbound,001,World,0 -wowcher.co.uk,wowcher.co.uk,inbound,001,World,0.000373 -wp.com,wordpress.com,inbound,001,World,0 -wp.pl,wp.pl,inbound,001,World,0.998027 -wp.pl,wp.pl,outbound,001,World,1 -wpengine.com,wpengine.com,inbound,001,World,1 -writers-community.com,writers-community.com,inbound,001,World,0 -writersstore.com,writersstore.com,inbound,001,World,0 -wsjemail.com,wsjemail.com,inbound,001,World,0 -wuaki.tv,chtah.net,inbound,001,World,0 -wustl.edu,app-info.net,inbound,001,World,0 -wwe.com,wwe.com,inbound,001,World,8e-06 -www.gov.tw,hinet.net,inbound,001,World,8e-05 -wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,001,World,0 -xbox.com,xbox.com,inbound,001,World,0 -xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,001,World,0 -xen.org,xen.org,inbound,001,World,1 -xing.com,xing.com,inbound,001,World,0 -xmailix.com,xmailix.com,inbound,001,World,0 -xmeeting.com,xmeeting.com,inbound,001,World,1 -xmr3.com,messagereach.com,inbound,001,World,0.999933 -xoom.com,xoom.com,inbound,001,World,0 -xpnews.com.br,xpnews.com.br,inbound,001,World,1 -xxxconnect.com,infinitypersonals.com,inbound,001,World,0 -yahoo-inc.com,yahoo.{...},inbound,001,World,0.999974 -yahoo.co.jp,yahoo.co.jp,inbound,001,World,1.5e-05 -yahoo.co.jp,yahoo.co.jp,outbound,001,World,0 -yahoo.{...},postini.com,inbound,001,World,0.767368 -yahoo.{...},yahoo.co.jp,inbound,001,World,0 -yahoo.{...},yahoo.{...},inbound,001,World,0.999416 -yahoo.{...},yahoodns.net,outbound,001,World,1 -yahoogroups.com,yahoodns.net,outbound,001,World,1 -yakala.co,euromsg.net,inbound,001,World,0 -yammer.com,yammer.com,inbound,001,World,1 -yandex.ru,yandex.net,inbound,001,World,0.999698 -yandex.ru,yandex.ru,outbound,001,World,1 -yapikredi.com.tr,yapikredi.com.tr,inbound,001,World,1 -yapstone.com,yapstone.com,inbound,001,World,0 -yelp.com,yelpcorp.com,inbound,001,World,0 -yesbank.in,yesbank.in,inbound,001,World,0.108676 -yipit.com,yipit.com,inbound,001,World,1 -ymail.com,yahoo.{...},inbound,001,World,1 -ymail.com,yahoodns.net,outbound,001,World,1 -ymlpserver.net,ymlpserver.net,inbound,001,World,0 -ymlpsrv.net,ymlpsrv.net,inbound,001,World,0 -yodobashi.com,yodobashi.com,inbound,001,World,0 -yoox.com,yoox.com,inbound,001,World,0 -youmail.com,youmail.com,inbound,001,World,0 -youravon.com,email-avonglobal.com,inbound,001,World,0 -youreletters3.com,equitymaster.com,inbound,001,World,0 -yourezads.com,yourezads.com,inbound,001,World,0.002974 -yourezlist.com,simplicityads.com,inbound,001,World,9.5e-05 -yourhostingaccount.com,yourhostingaccount.com,inbound,001,World,0 -yournewsletters.net,everydayhealth.com,inbound,001,World,0 -youversion.com,youversion.com,inbound,001,World,1 -zacks.com,zacks.com,inbound,001,World,0.999188 -zalando.be,fagms.de,inbound,001,World,0 -zalando.dk,fagms.de,inbound,001,World,0 -zalando.fi,fagms.de,inbound,001,World,0 -zalando.it,fagms.de,inbound,001,World,0 -zalando.nl,fagms.de,inbound,001,World,0 -zalando.pl,fagms.de,inbound,001,World,0 -zappos.com,zappos.com,inbound,001,World,0.626758 -zara.com,cheetahmail.com,inbound,001,World,0 -zattoo.com,sendnode.com,inbound,001,World,0 -zelonews.com.br,zelonews.com.br,inbound,001,World,0 -zendesk.com,zdsys.com,inbound,001,World,1 -zgalleriestyle.com,zgalleriestyle.com,inbound,001,World,0 -zibmail.info,zibmail.info,inbound,001,World,2e-06 -zillow.com,zillow.com,inbound,001,World,2.82542783334609e-07 -zinio.com,zinio.com,inbound,001,World,0.006193 -zinio.net,zinio.com,inbound,001,World,1 -zipalerts.com,sendgrid.net,inbound,001,World,1 -zipalerts.com,zipalerts.com,inbound,001,World,1 -ziprealty.com,ziprealty.com,inbound,001,World,0.994545 -ziprecruiter.com,ziprecruiter.com,inbound,001,World,1 -zivamewear.com,infimail.com,inbound,001,World,0 -zizigo.com,euromsg.net,inbound,001,World,0 -zlavadna.sk,zlavadna.sk,inbound,001,World,0 -zoom.com.br,zoom.com.br,inbound,001,World,0 -zoominternet.net,synacor.com,inbound,001,World,0 -zoosk.com,zoosk.com,inbound,001,World,0 -zoothost.com,zoothost.com,inbound,001,World,0.107168 -zorpia.com,zorpia.com,inbound,001,World,0.56104 -zovifashion.com,eccluster.com,inbound,001,World,0 -zulily.com,zulily.com,inbound,001,World,0 -zumzi.com,neogen.ro,inbound,001,World,0 -zumzi.com,zumzi.com,inbound,001,World,0 -zyngamail.com,zyngamail.com,inbound,001,World,0 -zzounds.com,zzounds.com,inbound,001,World,0 -careers24.com,careers24.com,inbound,002,Africa,0 -cpm.co.ma,cpm.co.ma,inbound,002,Africa,0 -fnb.co.za,fnb.co.za,inbound,002,Africa,0 -gmail.com,telkomadsl.co.za,inbound,002,Africa,0.999989 -gmail.com,vodacom.co.za,inbound,002,Africa,1 -gtbank.com,gtbank.com,inbound,002,Africa,0.056121 -pnetweb.co.za,hosting.co.za,inbound,002,Africa,0 -pnetweb.co.za,salesnet.co.za,inbound,002,Africa,0 -bigpond.com,bigpond.com,inbound,009,Oceania,0 -bigpond.com,bigpond.com,outbound,009,Oceania,1 -empoweredcomms.com.au,empoweredcomms.com.au,inbound,009,Oceania,0 -gmail.com,bigpond.net.au,inbound,009,Oceania,0.999907 -gmail.com,iinet.net.au,inbound,009,Oceania,0.966945 -gmail.com,optusnet.com.au,inbound,009,Oceania,0.992845 -gmail.com,tpgi.com.au,inbound,009,Oceania,0.999648 -mbounces.com,emdbms.com,inbound,009,Oceania,0 -realestate.com.au,realestate.com.au,inbound,009,Oceania,0 -snaphire.com,snaphire.com,inbound,009,Oceania,0 -trademe.co.nz,trademe.co.nz,inbound,009,Oceania,0.991916 -1-day.co.nz,1-day.co.nz,inbound,019,Americas,0 -12manage.com,netarrest.com,inbound,019,Americas,0.99998 -1800petmeds.com,1800petmeds.com,inbound,019,Americas,0.00599 -2touchbase.com,infimail.com,inbound,019,Americas,0 -4wheelparts.com,4wheelparts.com,inbound,019,Americas,0 -99acres.com,99acres.com,inbound,019,Americas,0 -aafes.com,aafes.com,inbound,019,Americas,0 -abercrombie-email.com,abercrombie-email.com,inbound,019,Americas,0 -abercrombiekids-email.com,abercrombie-email.com,inbound,019,Americas,0 -about.com,about.com,inbound,019,Americas,0 -about.com,sailthru.com,inbound,019,Americas,0 -acemserv.com,acemserv.com,inbound,019,Americas,0 -activesafelist.com,zoothost.com,inbound,019,Americas,0 -adidasusnews.com,adidasusnews.com,inbound,019,Americas,0 -adjockeys.com,thomas-j-brown.com,inbound,019,Americas,0 -admastersafelist.com,zoothost.com,inbound,019,Americas,0 -adorama.com,adorama.com,inbound,019,Americas,0 -adpirate.net,thomas-j-brown.com,inbound,019,Americas,0 -adtpulse.com,adtpulse.com,inbound,019,Americas,0 -ae.com,ae.com,inbound,019,Americas,0 -aexp.com,aexp.com,inbound,019,Americas,1 -affairalert.com,iverificationsystems.com,inbound,019,Americas,0 -agorafinancial.com,agorafinancial.com,inbound,019,Americas,0 -airbnb.com,airbnb.com,inbound,019,Americas,0.867975 -airbrake.io,mailgun.net,inbound,019,Americas,1 -airfarewatchdog.com,smartertravelmedia.com,inbound,019,Americas,0.012002 -airmiles.ca,bigfootinteractive.com,inbound,019,Americas,0 -akcijatau.lt,akcijatau.lt,inbound,019,Americas,0 -alarm.com,alarm.com,inbound,019,Americas,0 -alarmnet.com,alarmnet.com,inbound,019,Americas,0 -alaskaair.com,alaskaair.com,inbound,019,Americas,0 -alertid.com,alertid.com,inbound,019,Americas,0 -allheart.com,allheart.com,inbound,019,Americas,0 -allmodern.com,allmodern.com,inbound,019,Americas,0 -allout.org,allout.org,inbound,019,Americas,1 -allrecipes.com,allrecipes.com,inbound,019,Americas,0 -allstate.com,rsys1.com,inbound,019,Americas,0 -alm.com,sailthru.com,inbound,019,Americas,0 -alumniclass.com,alumniclass.com,inbound,019,Americas,0 -alumniconnections.com,alumniconnections.com,inbound,019,Americas,0 -amazon.{...},postini.com,inbound,019,Americas,0.658136 -amazonses.com,postini.com,inbound,019,Americas,0.733998 -americanas.com,americanas.com,inbound,019,Americas,0 -americanbar.org,abanet.org,inbound,019,Americas,0.00574 -amubm.com,amubm.com,inbound,019,Americas,1 -ancestry.com,ancestry.com,inbound,019,Americas,0 -angelbroking.in,infimail.com,inbound,019,Americas,0 -anghami.com,mailgun.net,inbound,019,Americas,1 -anthropologie.com,freepeople.com,inbound,019,Americas,0 -aol.com,aol.com,inbound,019,Americas,0.999528 -aol.com,aol.com,outbound,019,Americas,0.999984 -aol.com,sailthru.com,inbound,019,Americas,0 -aol.net,aol.com,inbound,019,Americas,1 -apache.org,apache.org,inbound,019,Americas,0 -apnacomplex.com,apnacomplex.com,inbound,019,Americas,0.999981 -apply-4-jobs.com,apply-4-jobs.com,inbound,019,Americas,0 -aptmail.in,mailurja.com,inbound,019,Americas,0 -arcamax.com,arcamax.com,inbound,019,Americas,1.4e-05 -aritzia.com,aritzia.com,inbound,019,Americas,0 -armaniexchange.com,bronto.com,inbound,019,Americas,0 -asadventure.com,asadventure.com,inbound,019,Americas,0 -asana.com,asana.com,inbound,019,Americas,1 -ashleymadison.com,ashleymadison.com,inbound,019,Americas,1 -asos.com,asos.com,inbound,019,Americas,0 -assembla.com,assembla.com,inbound,019,Americas,1 -astrology.com,astrology.com,inbound,019,Americas,0 -astrology.com,hsnlmailsvc.com,inbound,019,Americas,0 -astrology.com,webstakes.com,inbound,019,Americas,0 -astrology.com,wsafmailsvc.com,inbound,019,Americas,0 -atlassian.net,uc-inf.net,inbound,019,Americas,1 -atrapalo.cl,atrapalo.com,inbound,019,Americas,0 -atrapalo.com,atrapalo.com,inbound,019,Americas,0 -att-mail.com,att-mail.com,inbound,019,Americas,1.9e-05 -att-mail.com,att.com,inbound,019,Americas,0.999966 -att.net,att.net,outbound,019,Americas,0.204629 -att.net,mycingular.net,inbound,019,Americas,0.00021 -att.net,yahoo.{...},inbound,019,Americas,1 -australiagsm.net,australiagsm.net,inbound,019,Americas,0 -autoloop.us,loop28.com,inbound,019,Americas,0 -avaaz.org,avaaz.org,inbound,019,Americas,0 -avalanchesafelist.com,zoothost.com,inbound,019,Americas,0 -aveda.com,esteelauder.com,inbound,019,Americas,0 -avenue.com,avenue.com,inbound,019,Americas,0 -avg.com,avg.com,inbound,019,Americas,0 -avomail.com,avomail.com,inbound,019,Americas,0 -b2b-mail.net,b2b-mail.net,inbound,019,Americas,0 -b2b-mail.net,contact-list.net,inbound,019,Americas,0 -babycenter.com,rsys3.com,inbound,019,Americas,0 -babyoye.com,babyoye.com,inbound,019,Americas,0.011564 -badoo.com,monopost.com,inbound,019,Americas,1 -baligam.co.il,baligam.co.il,inbound,019,Americas,1 -banamex.com,ibrands.es,inbound,019,Americas,0 -bancoahorrofamsa.com,avantel.net.mx,inbound,019,Americas,1 -bancomercorreo.com,bancomercorreo.com,inbound,019,Americas,0 -bandsintown.com,bandsintown.com,inbound,019,Americas,1 -barclaycard.co.uk,barclays.co.uk,inbound,019,Americas,0 -barenecessities.com,barenecessities.com,inbound,019,Americas,0 -barleyment.ca,barleyment.ca,inbound,019,Americas,0 -barneys.com,barneys.com,inbound,019,Americas,0 -baseballsavings.com,baseballsavings.com,inbound,019,Americas,0 -basspronews.com,basspronews.com,inbound,019,Americas,0 -bathandbodyworks.com,bathandbodyworks.com,inbound,019,Americas,0 -baublebar.com,baublebar.com,inbound,019,Americas,0.011219 -bayt.com,bayt.com,inbound,019,Americas,2e-06 -bcp.com.pe,bcp.com.pe,inbound,019,Americas,1 -bellsouth.net,att.net,outbound,019,Americas,0 -bellsouth.net,yahoo.{...},inbound,019,Americas,1 -bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -bespokeoffers.co.uk,chtah.net,inbound,019,Americas,0 -bestbuy.ca,bestbuy.ca,inbound,019,Americas,0 -bestdealsforyou.in,elabs5.com,inbound,019,Americas,0 -bevmo.com,bevmo.com,inbound,019,Americas,0.000967 -bhcosmetics.com,bronto.com,inbound,019,Americas,0 -bhg.com,meredith.com,inbound,019,Americas,0 -bigfishgames.com,bigfishgames.com,inbound,019,Americas,0 -biglist.com,biglist.com,inbound,019,Americas,0 -bigtent.com,carezen.net,inbound,019,Americas,0 -bionexo.com,bionexo.com.br,inbound,019,Americas,0.999594 -birthdayalarm.com,monkeyinferno.net,inbound,019,Americas,0 -bitbucket.org,bitbucket.org,inbound,019,Americas,0 -bitlysupport.com,mailgun.info,inbound,019,Americas,1 -bitlysupport.com,mailgun.us,inbound,019,Americas,1 -bitslane.email,bitslane.email,inbound,019,Americas,0 -bitstatement.org,bitstatement.org,inbound,019,Americas,1 -bizjournals.com,bizjournals.com,inbound,019,Americas,0 -bizmailtoday.com,bizmailtoday.com,inbound,019,Americas,0 -bjs.com,bjs.com,inbound,019,Americas,0 -blablacar.com,blablacar.com,inbound,019,Americas,1 -blackberry.com,blackberry.com,inbound,019,Americas,0 -blackboard.com,blackboard.com,inbound,019,Americas,1 -blissworld.com,lstrk.net,inbound,019,Americas,1 -bloomingdales.com,bloomingdales.com,inbound,019,Americas,0 -bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,019,Americas,0 -bluehost.com,bluehost.com,inbound,019,Americas,0.000971 -bluehost.com,hostmonster.com,inbound,019,Americas,0 -bluehost.com,unifiedlayer.com,inbound,019,Americas,0 -blueshellgames.com,blueshellgames.com,inbound,019,Americas,0 -bluestatedigital.com,bluestatedigital.com,inbound,019,Americas,0 -bluestonemx.com,bluestonemx.com,inbound,019,Americas,1 -bm23.com,bronto.com,inbound,019,Americas,0 -bm324.com,bronto.com,inbound,019,Americas,0 -bmdeda99.com,bmdeda99.com,inbound,019,Americas,0 -bn.com,bn.com,inbound,019,Americas,0 -bnetmail.com,bnetmail.com,inbound,019,Americas,0 -bol.com.br,bol.com.br,inbound,019,Americas,0 -bol.com.br,bol.com.br,outbound,019,Americas,0 -boletinrenuevo.com,boletinrenuevo.com,inbound,019,Americas,0 -bomnegocio.com,bomnegocio.com,inbound,019,Americas,0.588708 -bonobos.com,bronto.com,inbound,019,Americas,0 -bookbub.com,bookbub.com,inbound,019,Americas,1 -booking.com,booking.com,inbound,019,Americas,1 -bookingbuddy.com,smartertravelmedia.com,inbound,019,Americas,0.000487 -boomtownroi.com,boomtownroi.com,inbound,019,Americas,0 -boscovs.com,boscovs.com,inbound,019,Americas,0 -bostonproper.com,bostonproper.com,inbound,019,Americas,0 -boutiquesecret.com,chtah.net,inbound,019,Americas,0 -bradfordexchange.com,bradfordexchange.com,inbound,019,Americas,0 -bradsdeals.com,bradsdeals.com,inbound,019,Americas,0 -brassring.com,brassring.com,inbound,019,Americas,1 -briantracyintl.com,briantracyintl.com,inbound,019,Americas,0 -brijj.com,brijj.com,inbound,019,Americas,0 -brincltd.com,brincltd.com,inbound,019,Americas,0 -bronto.com,bronto.com,inbound,019,Americas,0 -bsf01.com,bsftransmit33.com,inbound,019,Americas,0 -buffalo.edu,buffalo.edu,inbound,019,Americas,0.001059 -bumeran.com,bumeran.com,inbound,019,Americas,0 -burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,019,Americas,0 -burton.co.uk,burton.co.uk,inbound,019,Americas,0 -buscojobs.com,amazonaws.com,inbound,019,Americas,0 -buy123.com.tw,buy123.com.tw,inbound,019,Americas,1 -bzm.mobi,nmsrv.com,inbound,019,Americas,1 -c21stores.com,c21stores.com,inbound,019,Americas,0 -ca.gov,ca.gov,inbound,019,Americas,0.702758 -cafepress.com,cafepress.com,inbound,019,Americas,0.000778 -caixa.gov.br,caixa.gov.br,inbound,019,Americas,0 -californiapsychicsemail.com,californiapsychicsemail.com,inbound,019,Americas,0 -calottery.com,calottery.com,inbound,019,Americas,0.999426 -camel.com,rjrsignup.com,inbound,019,Americas,0 -canadianvisaexpert.net,canadianvisaexpert.net,inbound,019,Americas,0 -cancer.org,delivery.net,inbound,019,Americas,0 -capillary.co.in,capillary.co.in,inbound,019,Americas,1 -capitalone.com,bigfootinteractive.com,inbound,019,Americas,0 -capitalone360.com,ingdirect.com,inbound,019,Americas,0 -care.com,care.com,inbound,019,Americas,0 -care2.com,care2.com,inbound,019,Americas,0 -careerage.com,careerage.com,inbound,019,Americas,0 -careerflash.net,careerflash.net,inbound,019,Americas,1 -carmamail.com,carmamail.com,inbound,019,Americas,0 -carnivalfunmail.com,carnivalfunmail.com,inbound,019,Americas,0 -carolsdaughter.com,carolsdaughter.com,inbound,019,Americas,0 -carwale.com,carwale.com,inbound,019,Americas,0 -case.edu,cwru.edu,inbound,019,Americas,0.999942 -castingnetworks.com,castingnetworks.com,inbound,019,Americas,0.000102 -causes.com,causes.com,inbound,019,Americas,1 -cb2.com,cb2.com,inbound,019,Americas,0 -cbsig.net,cbsig.net,inbound,019,Americas,1 -ccbchurch.com,ccbchurch.com,inbound,019,Americas,1 -ccialerts.com,ccialerts.com,inbound,019,Americas,0 -ccs.com,footlocker.com,inbound,019,Americas,2.5e-05 -cenlat.com,cenlat.com,inbound,019,Americas,0.064459 -cerberusapp.com,cerberusapp.com,inbound,019,Americas,1 -cfmvmail.com,cfmvmail.com,inbound,019,Americas,0 -chabad.org,chabad.org,inbound,019,Americas,0 -champssports.com,footlocker.com,inbound,019,Americas,0.000131 -change.org,change.org,inbound,019,Americas,1 -charlestyrwhitt.com,charlestyrwhitt.com,inbound,019,Americas,0 -charter.net,charter.net,inbound,019,Americas,0 -charter.net,charter.net,outbound,019,Americas,0 -chase.com,jpmchase.com,inbound,019,Americas,0.999999 -chatcitynotifications.com,chatcitynotifications.com,inbound,019,Americas,0 -chaturbate.com,chaturbate.com,inbound,019,Americas,1 -cheapairmailer.com,cheapairmailer.com,inbound,019,Americas,0 -check.me,check.me,inbound,019,Americas,0 -cheekylovers.com,ropot.net,inbound,019,Americas,0 -chess.com,chess.com,inbound,019,Americas,1 -chicagotribune.com,latimes.com,inbound,019,Americas,0 -chicos.com,chicos.com,inbound,019,Americas,0.000156 -childrensplace.com,childrensplace.com,inbound,019,Americas,0 -christianbook.com,christianbook.com,inbound,019,Americas,1 -christianmingle.com,christianmingle.com,inbound,019,Americas,0 -chtah.com,chtah.net,inbound,019,Americas,0 -chtah.net,chtah.net,inbound,019,Americas,0 -cincghq.com,searchhomesingta.com,inbound,019,Americas,1 -circleofmomsmail.com,circleofmomsmail.com,inbound,019,Americas,0 -citibank.com,bigfootinteractive.com,inbound,019,Americas,0 -ck.com,ck.com,inbound,019,Americas,0 -clarks.com,clarks.com,inbound,019,Americas,0 -clickdimensions.com,clickdimensions.com,inbound,019,Americas,0 -clickexperts.net,clickexperts.net,inbound,019,Americas,0 -clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,019,Americas,0 -climber.com,climber.com,inbound,019,Americas,0 -clinique.com,esteelauder.com,inbound,019,Americas,0 -clubcupon.com.ar,clubcupon.com.ar,inbound,019,Americas,0 -cmm01.com,coremotivesmarketing.com,inbound,019,Americas,0 -coach.com,delivery.net,inbound,019,Americas,0 -codeproject.com,codeproject.com,inbound,019,Americas,0 -coldwatercreek.com,coldwatercreek.com,inbound,019,Americas,0 -collectionsetc.com,collectionsetc.com,inbound,019,Americas,0 -columbia.edu,columbia.edu,inbound,019,Americas,0.762355 -comcast.net,comcast.net,inbound,019,Americas,0.919244 -comcast.net,comcast.net,outbound,019,Americas,0.999998 -comenity.net,alldata.net,inbound,019,Americas,1 -comenity.net,bigfootinteractive.com,inbound,019,Americas,0 -commonfloor.com,commonfloor.com,inbound,019,Americas,1 -compute.internal,amazonaws.com,inbound,019,Americas,0.875148 -computerworld.com,computerworld.com,inbound,019,Americas,0 -comunicacaodemkt.com,locaweb.com.br,inbound,019,Americas,0 -constantcontact.com,confirmedcc.com,inbound,019,Americas,0 -constantcontact.com,constantcontact.com,inbound,019,Americas,5.3e-05 -constantcontact.com,postini.com,inbound,019,Americas,0.042128 -containerstore.com,containerstore.com,inbound,019,Americas,0 -convio.net,convio.net,inbound,019,Americas,0 -coremotivesmarketing.com,coremotivesmarketing.com,inbound,019,Americas,0 -cornell.edu,cornell.edu,inbound,019,Americas,0.192285 -correosocc.com,correosocc.com,inbound,019,Americas,1 -costco.co.uk,costco.com,inbound,019,Americas,0 -costco.com,costco.com,inbound,019,Americas,7e-06 -costcophotocenter.com,wc09.net,inbound,019,Americas,0 -costcoservices.com,costco.com,inbound,019,Americas,0 -cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,019,Americas,0 -couchsurfing.org,couchsurfing.com,inbound,019,Americas,0 -couponamama.com,couponamama.com,inbound,019,Americas,1 -cox.com,cox.com,inbound,019,Americas,0.001665 -cox.net,cox.net,inbound,019,Americas,0.009187 -cox.net,cox.net,outbound,019,Americas,0 -coyotelogistics.com,postini.com,inbound,019,Americas,0 -cp20.com,cp20.com,inbound,019,Americas,0 -cpbnc.com,cpbnc.com,inbound,019,Americas,0 -cpbnc.com,fye.com,inbound,019,Americas,0 -crackle.com,crackle.com,inbound,019,Americas,0 -craigslist.org,craigslist.org,inbound,019,Americas,0 -craigslist.org,craigslist.org,outbound,019,Americas,1 -crashlytics.com,crashlytics.com,inbound,019,Americas,1 -crashlytics.com,sendgrid.net,inbound,019,Americas,1 -crateandbarrel.com,crateandbarrel.com,inbound,019,Americas,0 -creationsrewards.net,creationsrewards.net,inbound,019,Americas,0 -creditkarma.com,creditkarma.com,inbound,019,Americas,1 -crosswalkmail.com,crosswalkmail.com,inbound,019,Americas,0 -crowdcut.com,crowdcut.com,inbound,019,Americas,1 -crunchyroll.com,crunchyroll.com,inbound,019,Americas,0 -cumulusdist.net,cumulusdist.net,inbound,019,Americas,0 -cuponatic.com.pe,cuponatic.com.pe,inbound,019,Americas,1 -cuponicamail.com,fnbox.com,inbound,019,Americas,0 -curbednetwork.com,curbednetwork.com,inbound,019,Americas,1 -cuspemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -custombriefings.com,custombriefings.com,inbound,019,Americas,0 -customercenter.net,customercenter.net,inbound,019,Americas,0.996453 -customeriomail.com,customeriomail.com,inbound,019,Americas,1 -cxomedia.com,cxomedia.com,inbound,019,Americas,0 -cybercoders.com,cybercoders.com,inbound,019,Americas,0 -dafiti.cl,dafiti.cl,inbound,019,Americas,0 -dailyhoroscope.com,tarot.com,inbound,019,Americas,0 -datehookup.com,datehookup.com,inbound,019,Americas,0 -datingvipnotifications.com,datingvipnotifications.com,inbound,019,Americas,0 -daviacalendar.com,daviacalendar.com,inbound,019,Americas,1 -davidsbridal.com,davidsbridal.com,inbound,019,Americas,0 -davidstea.com,bronto.com,inbound,019,Americas,0 -daz3d.com,bronto.com,inbound,019,Americas,0 -dealersocket.com,dealersocket.com,inbound,019,Americas,0 -dealsaver.com,secondstreetmedia.com,inbound,019,Americas,0.999823 -debshops.com,lstrk.net,inbound,019,Americas,1 -deliasshopemail.com,deliasshopemail.com,inbound,019,Americas,0 -delivery.net,delivery.net,inbound,019,Americas,0 -delivery.net,m0.net,inbound,019,Americas,0 -dell.com,bfi0.com,inbound,019,Americas,0 -dentalsenders.com,dentalsenders.com,inbound,019,Americas,0 -descontos.pt,descontos.pt,inbound,019,Americas,0 -designerapparel.com,myperfectsale.com,inbound,019,Americas,1 -despegar.com,despegar.com,inbound,019,Americas,0 -dhgate.com,chtah.net,inbound,019,Americas,0 -dice.com,dice.com,inbound,019,Americas,0 -digitalmailer.com,digitalmailer.com,inbound,019,Americas,0 -digitalmedia-comunicacion.es,chtah.net,inbound,019,Americas,0 -digitalromanceinc.com,digitalromanceinc.com,inbound,019,Americas,1 -directv.com,directv.com,inbound,019,Americas,0.026649 -discover.com,discoverfinancial.com,inbound,019,Americas,1 -disparadordeemails.com,locaweb.com.br,inbound,019,Americas,0 -disqus.net,disqus.net,inbound,019,Americas,0 -dn.net,naukri.com,inbound,019,Americas,0 -docusign.net,docusign.net,inbound,019,Americas,0.985435 -donationnet.net,donationnet.net,inbound,019,Americas,0 -dorothyperkins.com,dorothyperkins.com,inbound,019,Americas,0 -dowjones.info,dowjones.info,inbound,019,Americas,0 -dptagent.biz,dptagent.biz,inbound,019,Americas,0 -dptagent.net,dptagent.net,inbound,019,Americas,0 -dreamhost.com,dreamhost.com,inbound,019,Americas,0 -dreamwidth.org,dreamwidth.org,inbound,019,Americas,0 -drhinternet.net,drhinternet.net,inbound,019,Americas,0 -driftem.com,emce2.in,inbound,019,Americas,0 -driftem.com,mailurja.com,inbound,019,Americas,0 -dropbox.com,dropbox.com,inbound,019,Americas,1 -dropboxmail.com,dropbox.com,inbound,019,Americas,1 -dsw.com,dsw.com,inbound,019,Americas,0 -ducks.org,uptilt.com,inbound,019,Americas,0 -duke.edu,duke.edu,inbound,019,Americas,0.308101 -dukecareers.com,dukecareers.com,inbound,019,Americas,1 -dvor.com,dvor.com,inbound,019,Americas,0 -dynamite-safelist.com,thomas-j-brown.com,inbound,019,Americas,0 -dynect-mailer.net,sendlabs.com,inbound,019,Americas,0 -e-activist.com,e-activist.com,inbound,019,Americas,0 -e-beallsonline.com,e-stagestores.com,inbound,019,Americas,0 -e-costco.mx,costco.com,inbound,019,Americas,0 -e-goodysonline.com,e-stagestores.com,inbound,019,Americas,0 -e-peebles.com,e-stagestores.com,inbound,019,Americas,0 -e-rewards.net,e-rewards.net,inbound,019,Americas,1 -e-stagestores.com,e-stagestores.com,inbound,019,Americas,0 -e-travelclub.es,e-travelclub.es,inbound,019,Americas,0 -e2ma.net,e2ma.net,inbound,019,Americas,1 -ea.com,ea.com,inbound,019,Americas,0.001232 -eaccess.net,postini.com,inbound,019,Americas,0 -earn-e-miles.com,earn-e-miles.com,inbound,019,Americas,0 -earnerslist.com,traxweb.net,inbound,019,Americas,9e-06 -earthfare-email.com,edclient2.com,inbound,019,Americas,0 -earthlink.net,earthlink.net,inbound,019,Americas,0.031648 -earthlink.net,earthlink.net,outbound,019,Americas,0 -eastbay.com,footlocker.com,inbound,019,Americas,0 -easyhealthoptions.com,easyhealthoptions.com,inbound,019,Americas,1 -easyroommate.com,easyroommate.com,inbound,019,Americas,0 -ebates.com,bfi0.com,inbound,019,Americas,0 -ebay.{...},ebay.{...},inbound,019,Americas,0.99941 -ebizac2.com,ebizac2.com,inbound,019,Americas,0 -eblastengine.com,secondstreetmedia.com,inbound,019,Americas,0.999827 -ec2.internal,amazonaws.com,inbound,019,Americas,0.769271 -ecasend.com,ecasend.com,inbound,019,Americas,0 -ed.gov,leepfrog.com,inbound,019,Americas,0.106381 -ed10.net,ed10.com,inbound,019,Americas,0 -edirect1.com,ivytech.edu,inbound,019,Americas,0 -edmodo.com,edmodo.com,inbound,019,Americas,1.1e-05 -educationzone.co.in,iaires.com,inbound,019,Americas,0 -effectivesafelist.com,zoothost.com,inbound,019,Americas,0 -eharmony.com,eharmony.com,inbound,019,Americas,1e-06 -eigbox.net,eigbox.net,inbound,019,Americas,0 -elabs3.com,elabs3.com,inbound,019,Americas,0 -elabs3.com,meritline.com,inbound,019,Americas,0 -elabs5.com,elabs5.com,inbound,019,Americas,0 -elabs6.com,elabs6.com,inbound,019,Americas,0 -elanceonline.com,elanceonline.com,inbound,019,Americas,0 -eleadtrack.net,eleadtrack.net,inbound,019,Americas,0 -elitesafelist.com,elitesafelist.com,inbound,019,Americas,0 -email-cooking.com,email-cooking.com,inbound,019,Americas,0 -email-galls.com,email-galls.com,inbound,019,Americas,1 -email-od.com,smtprelayserver.com,inbound,019,Americas,0.999779 -email-ticketdada.com,email-ticketdada.com,inbound,019,Americas,1 -email4-beyond.com,email4-beyond.com,inbound,019,Americas,0 -emailcounts.com,secureserver.net,inbound,019,Americas,0 -emaildir2.com,emaildirect.net,inbound,019,Americas,0 -emaildir2.com,espsnd.com,inbound,019,Americas,0 -emailnotify.net,emailnotify.net,inbound,019,Americas,0.961424 -emailsbancoestado.cl,emailsbancoestado.cl,inbound,019,Americas,0 -emailsripley.cl,etarget.cl,inbound,019,Americas,0 -embluejet.com,embluejet.com,inbound,019,Americas,0 -embluejet.com,emblueuser.com,inbound,019,Americas,0 -emcsend.com,emcsend.com,inbound,019,Americas,0 -emergencyemail.org,emergencyemail.org,inbound,019,Americas,0 -emktsender.net,locaweb.com.br,inbound,019,Americas,0 -emma.cl,emma.cl,inbound,019,Americas,0.996895 -emobile.ad.jp,postini.com,inbound,019,Americas,0 -employboard.com,employboard.com,inbound,019,Americas,1 -entregadeemails.com,locaweb.com.br,inbound,019,Americas,0 -entregadordecampanhas.net,locaweb.com.br,inbound,019,Americas,0 -enviodecampanhas.net,locaweb.com.br,inbound,019,Americas,0 -enviodemkt.com.br,locaweb.com.br,inbound,019,Americas,0 -epriority.com,epriority.com,inbound,019,Americas,0 -equifax.com,equifax.com,inbound,019,Americas,1 -equussafelist.com,equussafelist.com,inbound,019,Americas,0.000265 -esri.com,esri.com,inbound,019,Americas,0.983104 -esteelauder.com,esteelauder.com,inbound,019,Americas,0 -evanguard.com,evanguard.com,inbound,019,Americas,0 -eventbrite.com,eventbrite.com,inbound,019,Americas,0 -eversavelocal.com,eversavelocal.com,inbound,019,Americas,0 -everydayhealthinc.com,waterfrontmedia.net,inbound,019,Americas,0 -everyjobforme.com,everyjobforme.com,inbound,019,Americas,0 -exchangesolutions.com,exchangesolutions.com,inbound,019,Americas,0.000143 -exec-u-net-mail.com,exec-u-net-mail.com,inbound,019,Americas,0 -exprpt.com,exprpt.com,inbound,019,Americas,0 -expvtinboxhub.net,expvtinboxhub.net,inbound,019,Americas,0 -fabletics.com,bronto.com,inbound,019,Americas,0 -facebook.com,facebook.com,inbound,019,Americas,0.719445 -facebook.com,facebook.com,outbound,019,Americas,1 -facebookappmail.com,facebook.com,inbound,019,Americas,1 -facebookmail.com,facebook.com,inbound,019,Americas,1 -facebookmail.com,postini.com,inbound,019,Americas,0.592681 -facebookmail.com,yahoo.{...},inbound,019,Americas,1 -familychristianmail.com,familychristianmail.com,inbound,019,Americas,0 -famousfootwear.com,famousfootwear.com,inbound,019,Americas,0 -fanfiction.com,fictionpress.com,inbound,019,Americas,1 -farmersonly.com,mailgun.us,inbound,019,Americas,1 -fashion2hub.in,mgenie.in,inbound,019,Americas,0 -fastgb.com,fastgb.com,inbound,019,Americas,0 -fastlistmailer.com,zoothost.com,inbound,019,Americas,0 -fastweb.com,fastweb.com,inbound,019,Americas,0 -fbi.gov,fbi.gov,inbound,019,Americas,0 -fbmta.com,fbmta.com,inbound,019,Americas,0 -fc2.com,fc2.com,inbound,019,Americas,0.000601 -fedoraproject.org,fedoraproject.org,inbound,019,Americas,0 -feedblitz.com,feedblitz.com,inbound,019,Americas,0 -fetlifemail.com,fetlifemail.com,inbound,019,Americas,0 -fibertel.com.ar,fibertel.com.ar,inbound,019,Americas,0.003898 -fidelizador.org,fidelizador.org,inbound,019,Americas,0 -findexpvtinbox.com,findexpvtinbox.com,inbound,019,Americas,0 -finishline.com,finishline.com,inbound,019,Americas,0 -firemountaingems.com,firemountaingems.com,inbound,019,Americas,0 -fisher-price.com,fisher-price.com,inbound,019,Americas,0 -fitbit.com,fitbit.com,inbound,019,Americas,1 -fitnessmagazine.com,meredith.com,inbound,019,Americas,0 -fiverr.com,fiverr.com,inbound,019,Americas,0 -flexmls.com,flexmls.com,inbound,019,Americas,0.999992 -flightaware.com,flightaware.com,inbound,019,Americas,0.020698 -flipkart.com,flipkart.com,inbound,019,Americas,1 -flirt.com,ropot.net,inbound,019,Americas,0 -flirthookup.com,flirthookup.com,inbound,019,Americas,1 -flyceb.com,flyceb.com,inbound,019,Americas,0 -flyfrontier.com,flyfrontier.com,inbound,019,Americas,0 -foolsubs.com,foolcs.com,inbound,019,Americas,0 -foolsubs.com,foolsubs.com,inbound,019,Americas,0 -footaction.com,footlocker.com,inbound,019,Americas,0 -footlocker.com,footlocker.com,inbound,019,Americas,0.000605 -forever21.com,forever21.com,inbound,019,Americas,0 -fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,019,Americas,0 -foursquare.com,foursquare.com,inbound,019,Americas,1 -foxnews.com,foxnews.com,inbound,019,Americas,0.0139 -fragrancenet.com,fragrancenet.com,inbound,019,Americas,0.000427 -francescas.com,bronto.com,inbound,019,Americas,0 -freeadsmailer.com,zoothost.com,inbound,019,Americas,0 -freebeesafelist.com,zoothost.com,inbound,019,Americas,0 -freebizmag.com,delivery.net,inbound,019,Americas,0 -freebsd.org,freebsd.org,inbound,019,Americas,0.999845 -freecycle.org,freecycle.org,inbound,019,Americas,1 -freedesktop.org,freedesktop.org,inbound,019,Americas,0 -freeflys.com,freeflys.com,inbound,019,Americas,0 -freelancer.com,freelancer.com,inbound,019,Americas,0 -freelancer.com,freelancernotify.com,inbound,019,Americas,0 -freelancer.com,getafreelancer.com,inbound,019,Americas,0 -freelists.org,iquest.net,inbound,019,Americas,0 -freelotto.com,plasmanetinc.com,inbound,019,Americas,0 -freepeople.com,freepeople.com,inbound,019,Americas,0 -freesafelistking.com,zoothost.com,inbound,019,Americas,0 -freshdesk.com,freshdesk.com,inbound,019,Americas,1 -freshers2015.com,secureserver.net,inbound,019,Americas,0 -freshlatesave.com,freshlatesave.com,inbound,019,Americas,1 -friskone.com,mailurja.com,inbound,019,Americas,0 -frontsight.com,frontsight.com,inbound,019,Americas,0 -frys.com,frys.com,inbound,019,Americas,0.00388 -frysmail.com,frysmail.com,inbound,019,Americas,0 -fspeletters.com,agorapub.co.uk,inbound,019,Americas,0 -fuelrewards.com,britecast.com,inbound,019,Americas,0 -futureshop.com,futureshop.com,inbound,019,Americas,0 -gaiaonline.com,gaiaonline.com,inbound,019,Americas,0 -gamehouse.com,gamehouse.com,inbound,019,Americas,0 -gbyguess.com,guess.com,inbound,019,Americas,0.001787 -gemoney.com,rsys1.com,inbound,019,Americas,0 -generalmills.com,boxtops4education.com,inbound,019,Americas,0 -generalmills.com,pillsbury.com,inbound,019,Americas,0 -gentoo.org,gentoo.org,inbound,019,Americas,1 -get-me-jobs.com,get-me-jobs.com,inbound,019,Americas,0 -gethired.com,gethired.com,inbound,019,Americas,1 -getitfree.us,getitfree.us,inbound,019,Americas,0 -getpaidsolutions.com,getpaidsolutions.com,inbound,019,Americas,1 -getpocket.com,bronto.com,inbound,019,Americas,0 -ghin.com,ghinconnect.com,inbound,019,Americas,0 -ghup.in,mgenie.in,inbound,019,Americas,0 -gillyhicks-email.com,abercrombie-email.com,inbound,019,Americas,0 -github.com,github.com,inbound,019,Americas,1 -github.com,postini.com,inbound,019,Americas,0.837872 -glassdoor.com,glassdoor.com,inbound,019,Americas,1 -glasses.com,glasses.com,inbound,019,Americas,0 -gliq.com,gliq.com,inbound,019,Americas,0.99852 -globalsafelist.com,globalsafelist.com,inbound,019,Americas,0 -globaltestmarket.com,globaltestmarket.com,inbound,019,Americas,0 -gmail.com,amazonaws.com,inbound,019,Americas,0.994381 -gmail.com,anteldata.net.uy,inbound,019,Americas,0.998653 -gmail.com,bell.ca,inbound,019,Americas,0.992481 -gmail.com,bellsouth.net,inbound,019,Americas,0.9996 -gmail.com,blackberry.com,inbound,019,Americas,0.992207 -gmail.com,brasiltelecom.net.br,inbound,019,Americas,0.99995 -gmail.com,centurytel.net,inbound,019,Americas,0.999415 -gmail.com,cgocable.net,inbound,019,Americas,0.99829 -gmail.com,charter.com,inbound,019,Americas,0.999109 -gmail.com,claro.net.br,inbound,019,Americas,1 -gmail.com,comcast.net,inbound,019,Americas,0.999621 -gmail.com,comcastbusiness.net,inbound,019,Americas,0.985462 -gmail.com,cox.net,inbound,019,Americas,0.962619 -gmail.com,embarqhsd.net,inbound,019,Americas,0.999554 -gmail.com,franchiseindia.com,inbound,019,Americas,1 -gmail.com,frontiernet.net,inbound,019,Americas,0.995233 -gmail.com,gvt.net.br,inbound,019,Americas,0.999513 -gmail.com,lorexddns.net,inbound,019,Americas,0 -gmail.com,majesticmoneymailer.com,inbound,019,Americas,1 -gmail.com,mchsi.com,inbound,019,Americas,0.999951 -gmail.com,movistar.cl,inbound,019,Americas,0.999361 -gmail.com,mycingular.net,inbound,019,Americas,0.999918 -gmail.com,myvzw.com,inbound,019,Americas,0.99992 -gmail.com,naukri.com,inbound,019,Americas,0.000998 -gmail.com,optonline.net,inbound,019,Americas,0.999776 -gmail.com,postini.com,inbound,019,Americas,0.650888 -gmail.com,qwest.net,inbound,019,Americas,0.997574 -gmail.com,rcn.com,inbound,019,Americas,0.999275 -gmail.com,rogers.com,inbound,019,Americas,0.999916 -gmail.com,rr.com,inbound,019,Americas,0.986894 -gmail.com,sbcglobal.net,inbound,019,Americas,0.998817 -gmail.com,shawcable.net,inbound,019,Americas,0.999998 -gmail.com,spcsdns.net,inbound,019,Americas,0.999998 -gmail.com,suddenlink.net,inbound,019,Americas,0.961593 -gmail.com,telecom.net.ar,inbound,019,Americas,0.999664 -gmail.com,telesp.net.br,inbound,019,Americas,0.999743 -gmail.com,telus.com,inbound,019,Americas,1 -gmail.com,telus.net,inbound,019,Americas,0.974663 -gmail.com,tmodns.net,inbound,019,Americas,1 -gmail.com,veloxzone.com.br,inbound,019,Americas,0.999969 -gmail.com,verizon.net,inbound,019,Americas,0.990214 -gmail.com,videotron.ca,inbound,019,Americas,0.967196 -gmail.com,vtr.net,inbound,019,Americas,0.999072 -gmail.com,websitewelcome.com,inbound,019,Americas,1 -gmail.com,wideopenwest.com,inbound,019,Americas,0.999729 -gmail.com,windstream.net,inbound,019,Americas,0.951847 -gmail.com,yahoo.{...},inbound,019,Americas,0.999992 -gmail.com,zoothost.com,inbound,019,Americas,0.033858 -gob.ar,gob.ar,inbound,019,Americas,0.160006 -gob.ec,gob.ec,inbound,019,Americas,0.623867 -godaddy.com,secureserver.net,inbound,019,Americas,0 -gogecapital.com,rsys1.com,inbound,019,Americas,0 -goldenopsafelist.com,zoothost.com,inbound,019,Americas,0 -goldstar.com,goldstar.com,inbound,019,Americas,1 -golfmnb.com,golfmnb.com,inbound,019,Americas,0 -google.com,postini.com,inbound,019,Americas,0.584887 -gopusamedia.com,gopusamedia.com,inbound,019,Americas,0 -govdelivery.com,govdelivery.com,inbound,019,Americas,0 -governmentjobs.com,governmentjobs.com,inbound,019,Americas,0 -gpmailer.com.br,parperfeito.com,inbound,019,Americas,0 -grassrootsaction.com,grassfire.net,inbound,019,Americas,0 -greatergood.com,greatergood.com,inbound,019,Americas,0 -groupon.{...},chtah.net,inbound,019,Americas,0 -groupon.{...},groupon.{...},inbound,019,Americas,1.0 -groupon.{...},postini.com,inbound,019,Americas,0.886672 -grouponmail.{...},grouponmail.{...},inbound,019,Americas,0 -grupos.com.br,grupos.com.br,inbound,019,Americas,0 -guess.ca,guess.com,inbound,019,Americas,0.000807 -guess.com,guess.com,inbound,019,Americas,0.003805 -guessfactory.com,guess.com,inbound,019,Americas,0.00164 -gustazos.com,cityoferta.com,inbound,019,Americas,1 -hallmark.com,hallmark.com,inbound,019,Americas,0 -hannaandersson.com,hannaandersson.com,inbound,019,Americas,0.013533 -harristeetermail.com,harristeetermail.com,inbound,019,Americas,0.99673 -harvard.edu,harvard.edu,inbound,019,Americas,0.274906 -hayneedle.com,hayneedle.com,inbound,019,Americas,0 -helpareporter.net,helpareporter.com,inbound,019,Americas,0 -herculist.com,herculist.com,inbound,019,Americas,0 -hilton.com,hiltonemail.com,inbound,019,Americas,0 -hipchat.com,hipchat.com,inbound,019,Americas,1 -hispavista.com,hispavista.com,inbound,019,Americas,0 -hollister-email.com,abercrombie-email.com,inbound,019,Americas,0 -homeaway.com,haspf.com,inbound,019,Americas,0 -homedecorators.com,homedecorators.com,inbound,019,Americas,0 -homedepot.com,homedepot.com,inbound,019,Americas,1 -hootsuite.com,hootsuite.com,inbound,019,Americas,1 -horchowemail.com,horchowemail.com,inbound,019,Americas,0 -hostelworld.com,bronto.com,inbound,019,Americas,0 -hostgator.com,hostgator.com,inbound,019,Americas,0.98061 -hostgator.com,websitewelcome.com,inbound,019,Americas,1 -hotmail.{...},hotmail.{...},inbound,019,Americas,1 -hotmail.{...},hotmail.{...},outbound,019,Americas,1 -hotmail.{...},postini.com,inbound,019,Americas,0.723143 -hotornot.com,monopost.com,inbound,019,Americas,1 -hotschedules.com,hotschedules.com,inbound,019,Americas,0 -house.gov,house.gov,inbound,019,Americas,0.999966 -houseoffraser.co.uk,houseoffraser.co.uk,inbound,019,Americas,0 -houzz.com,houzz.com,inbound,019,Americas,1 -hubspot.com,hubspot.com,inbound,019,Americas,1 -hungry-girl.com,hungry-girl.com,inbound,019,Americas,0 -iamlgnd2.com,iamlgnd2.com,inbound,019,Americas,1 -ibsys.com,ibsys.com,inbound,019,Americas,9e-05 -icbc.com.ar,clickexperts.net,inbound,019,Americas,0 -icbc.com.ar,standardbank.com.ar,inbound,019,Americas,0 -icims.com,icims.com,inbound,019,Americas,0.999939 -icors.org,lsoft.us,inbound,019,Americas,0 -icpbounce.com,icpbounce.com,inbound,019,Americas,0 -idc.email,nmsrv.com,inbound,019,Americas,1 -idgconnect-resources.com,idgconnect-resources.com,inbound,019,Americas,0 -ieee.org,ieee.org,inbound,019,Americas,0.999912 -ig.com.br,ig.com.br,inbound,019,Americas,0 -ig.com.br,ig.com.br,outbound,019,Americas,0 -igot-mails.com,zoothost.com,inbound,019,Americas,0 -iimjobs.com,iimjobs.com,inbound,019,Americas,1 -ikmultimedianews.com,ikmultimedianews.com,inbound,019,Americas,0.993319 -illinois.edu,illinois.edu,inbound,019,Americas,0.866481 -imageshost.ca,imageshost.ca,inbound,019,Americas,0 -imakenews.net,imakenews.com,inbound,019,Americas,0 -imo.im,imo.im,inbound,019,Americas,1 -imodules.com,imodules.com,inbound,019,Americas,0 -imvu.com,imvu.com,inbound,019,Americas,7e-06 -inboxdollars.com,inboxdollars.com,inbound,019,Americas,0 -inboxfirst.com,inboxfirst.com,inbound,019,Americas,0 -inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,019,Americas,0.999877 -inboxpays.com,inboxpays.com,inbound,019,Americas,0 -inboxpounds.co.uk,inboxpounds.co.uk,inbound,019,Americas,0 -indeed.com,indeed.com,inbound,019,Americas,0.000121 -indeedemail.com,indeedemail.com,inbound,019,Americas,0 -independentlivingbullion.com,independentlivingbullion.com,inbound,019,Americas,0 -infobradesco.com.br,infobradesco.com.br,inbound,019,Americas,0 -infojobs.com.br,anuntis.com,inbound,019,Americas,0 -infomoney.com.br,infomoney.com.br,inbound,019,Americas,1 -informz.net,informz.net,inbound,019,Americas,0 -infradead.org,infradead.org,inbound,019,Americas,1 -inman.com,inman.com,inbound,019,Americas,0 -innovyx.net,innovyx.net,inbound,019,Americas,0 -insidehook.com,sailthru.com,inbound,019,Americas,0 -instagram.com,facebook.com,inbound,019,Americas,1 -instantprofitlist.com,screenshotads.com,inbound,019,Americas,0 -interac.ca,certapay.com,inbound,019,Americas,0 -interactivebrokers.com,interactivebrokers.com,inbound,019,Americas,1 -interactiverealtyservices.com,interactiverealtyservices.com,inbound,019,Americas,0 -intercom.io,mailgun.info,inbound,019,Americas,1 -interealty.net,interealty.net,inbound,019,Americas,1 -interweave.com,interweave.com,inbound,019,Americas,0 -intliv2.net,internationalliving.com,inbound,019,Americas,0 -invalidemail.com,taleo.net,inbound,019,Americas,1 -investopedia.com,vclk.net,inbound,019,Americas,0.999999 -investorplace.com,investorplace.com,inbound,019,Americas,0.995093 -irctcshopping.com,chtah.net,inbound,019,Americas,0 -iridium.com,iridium.com,inbound,019,Americas,0.997882 -isendservice.com.br,isendservice.com.br,inbound,019,Americas,0 -itau-unibanco.com.br,itau.com.br,inbound,019,Americas,0 -ittoolbox.com,ittoolbox.com,inbound,019,Americas,0 -ittoolbox.com,toolbox.com,inbound,019,Americas,0 -itwhitepapers.com,itwhitepapers.com,inbound,019,Americas,0 -iwantoneofthose.com,thehut.com,inbound,019,Americas,0 -ixs1.net,ixs1.net,inbound,019,Americas,0.005131 -jcpenney.com,jcpenney.com,inbound,019,Americas,9e-06 -jeevansathi.com,jeevansathi.com,inbound,019,Americas,0 -jetsetter.com,smartertravelmedia.com,inbound,019,Americas,0.041888 -jibjab.com,storybots.com,inbound,019,Americas,0 -jira.com,uc-inf.net,inbound,019,Americas,1 -jobscentral.com.sg,mailgun.net,inbound,019,Americas,1 -jobson.com,jobsonmail.com,inbound,019,Americas,0 -jobsradar.com,jobsradar.com,inbound,019,Americas,0 -jockeycomfort.com,jockeycomfort.com,inbound,019,Americas,0 -johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,019,Americas,0 -jomashop.com,lstrk.net,inbound,019,Americas,1 -josbank.com,josbank.com,inbound,019,Americas,0 -jossandmain.com,jossandmain.com,inbound,019,Americas,0 -jtv.com,jtv.com,inbound,019,Americas,0 -juno.com,untd.com,inbound,019,Americas,0 -juno.com,untd.com,outbound,019,Americas,0 -justdial.com,mailurja.com,inbound,019,Americas,0 -justfab.com,bronto.com,inbound,019,Americas,0 -justfab.fr,bronto.com,inbound,019,Americas,0 -keek.com,keek.com,inbound,019,Americas,1 -kernel.org,kernel.org,inbound,019,Americas,0 -kgstores.com,kgstores.com,inbound,019,Americas,0 -kickstarter.com,kickstarter.com,inbound,019,Americas,1 -kidsfootlocker.com,footlocker.com,inbound,019,Americas,0 -kik.com,kik.com,inbound,019,Americas,1 -kimblegroup.com,kimblegroup.com,inbound,019,Americas,1 -kintera.com,kintera.com,inbound,019,Americas,0 -klaviyomail.com,klaviyomail.com,inbound,019,Americas,1 -klove.com,emfbroadcasting.com,inbound,019,Americas,0 -komando.com,komando.com,inbound,019,Americas,0 -kp.org,kp.org,inbound,019,Americas,0.999975 -krogermail.com,bigfootinteractive.com,inbound,019,Americas,0 -landmarketingmailer.com,zoothost.com,inbound,019,Americas,0 -landofnod.com,landofnod.com,inbound,019,Americas,0.002525 -languagepod101.com,eclient10.com,inbound,019,Americas,0 -languagepod101.com,eddlvr.com,inbound,019,Americas,0 -languagepod101.com,ednwsltr3.com,inbound,019,Americas,0 -languagepod101.com,ednwsltr8.com,inbound,019,Americas,0 -languagepod101.com,emaildirect.net,inbound,019,Americas,0 -lasenza.com,lasenza.com,inbound,019,Americas,0 -lastcallemail.com,lastcallemail.com,inbound,019,Americas,0 -latimes.com,latimes.com,inbound,019,Americas,0 -lauraashley.com,lauraashley.com,inbound,019,Americas,0 -leftlanesports.com,auspient.com,inbound,019,Americas,0.00705 -leftlanesports.com,leftlanesports.com,inbound,019,Americas,0 -legalshieldassociate.com,legalshield.com,inbound,019,Americas,0 -lexico.com,lexico.com,inbound,019,Americas,0 -life360.com,life360.com,inbound,019,Americas,1 -lindenlab.com,lindenlab.com,inbound,019,Americas,0.999444 -linkedin.com,linkedin.com,inbound,019,Americas,0.999873 -linkedin.com,postini.com,inbound,019,Americas,0.713391 -listeneremail.net,listeneremail.net,inbound,019,Americas,0 -listia.com,listia.com,inbound,019,Americas,1 -listnerds.com,listnerds.com,inbound,019,Americas,0 -listreturn.com,zoothost.com,inbound,019,Americas,0 -listserve.com,listserve.com,inbound,019,Americas,0 -listvolta.com,listvolta.com,inbound,019,Americas,0 -listwire.com,listwire.com,inbound,019,Americas,0 -live.{...},hotmail.{...},inbound,019,Americas,1 -live.{...},hotmail.{...},outbound,019,Americas,1 -livefyre.com,andbit.net,inbound,019,Americas,1 -livescribe.com,bronto.com,inbound,019,Americas,0 -livingsocial.com,livingsocial.com,inbound,019,Americas,0 -livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,019,Americas,0 -localhires.com,localhires.com,inbound,019,Americas,1 -logitech.com,dvsops.com,inbound,019,Americas,0 -lojasmarisa.com.br,lojasmarisa.com.br,inbound,019,Americas,0 -lolsolos.com,ultimateadsites.net,inbound,019,Americas,0.999996 -lonelywifehookup.com,iverificationsystems.com,inbound,019,Americas,0 -lordandtaylor.com,lordandtaylor.com,inbound,019,Americas,0 -loveaholics.com,ropot.net,inbound,019,Americas,0 -lovelywholesale.com,lovelywholesale.com,inbound,019,Americas,1 -lrsmail.com,lrsmail.com,inbound,019,Americas,0 -lt02.net,listrak.com,inbound,019,Americas,1 -lt02.net,lstrk.net,inbound,019,Americas,1 -ltdcommodities.com,ltdcomm.net,inbound,019,Americas,0 -luckymag.com,mkt4500.com,inbound,019,Americas,0 -lulu.com,bronto.com,inbound,019,Americas,0 -lulus.com,lstrk.net,inbound,019,Americas,1 -lumosity.com,lumosity.com,inbound,019,Americas,1 -lyst.com,lyst.com,inbound,019,Americas,1 -maccosmetics.com,esteelauder.com,inbound,019,Americas,0 -macupdate.com,mailgun.info,inbound,019,Americas,1 -madmels.info,ultimateadsites.net,inbound,019,Americas,1 -madmimi.com,madmimi.com,inbound,019,Americas,0 -magicjack.com,magicjack.com,inbound,019,Americas,1 -magnetdev.com,magnetmail.net,inbound,019,Americas,0 -mail-route.com,mail-route.com,inbound,019,Americas,0 -mail-thestreet.com,mail-thestreet.com,inbound,019,Americas,0 -mail.mil,mail.mil,inbound,019,Americas,0 -mailaccurate.com,mgenie.in,inbound,019,Americas,0 -mailfacil.com.br,md02.com,inbound,019,Americas,0 -mailfeast.com,mgenie.in,inbound,019,Americas,0 -mailgun.org,mailgun.info,inbound,019,Americas,1 -mailgun.org,mailgun.net,inbound,019,Americas,1 -mailgun.org,mailgun.us,inbound,019,Americas,1 -mailingathome.net,mailingathome.net,inbound,019,Americas,0.999995 -mailjayde.com,mailjayde.com,inbound,019,Americas,0 -mailmachine1050.com,mailmachine1050.com,inbound,019,Americas,0 -mailsend1.com,mailsend6.com,inbound,019,Americas,0 -mailsender.com.br,mailsender.com.br,inbound,019,Americas,0 -manager.com.br,manager.com.br,inbound,019,Americas,0 -mandrillapp.com,mandrillapp.com,inbound,019,Americas,1 -mandrillapp.com,myjobhelperalerts.com,inbound,019,Americas,1 -marcustheatres.com,movio.co,inbound,019,Americas,0 -markandgraham.com,markandgraham.com,inbound,019,Americas,0 -marketer-safelist.com,jsalfianmarketing.com,inbound,019,Americas,1 -marketinghq.net,elabs8.com,inbound,019,Americas,0 -marketingprofs.com,marketingprofs.com,inbound,019,Americas,0.004412 -marlboro.com,marlboro.com,inbound,019,Americas,0 -maropost.com,biotrustnews.com,inbound,019,Americas,0 -maropost.com,mailing-truthaboutabs.com,inbound,019,Americas,0 -maropost.com,maropost.com,inbound,019,Americas,0 -maropost.com,mp2201.com,inbound,019,Americas,0 -masivapp.com,masivapp.com,inbound,019,Americas,1 -massageenvyclinics.com,massageenvyclinics.com,inbound,019,Americas,0 -masterbase.com,masterbase.com,inbound,019,Americas,0 -mastercard-email.com,mastercard-email.com,inbound,019,Americas,0 -mate1.net,mate1.net,inbound,019,Americas,0 -matrixemailer.com,matrixemailer.com,inbound,019,Americas,0 -mbstrm.com,mobilestorm.com,inbound,019,Americas,0 -mcafee.com,mcafee.com,inbound,019,Americas,0.979126 -mcarthurglen.com,mcarthurglen.com,inbound,019,Americas,0 -mcdlv.net,mcdlv.net,inbound,019,Americas,0 -mckinsey.com,bigfootinteractive.com,inbound,019,Americas,0 -mcsv.net,mcsv.net,inbound,019,Americas,0 -mdlinx.com,mdlinx.com,inbound,019,Americas,0 -mec.gov.br,mec.gov.br,inbound,019,Americas,0 -mediabistro.com,iworld.com,inbound,019,Americas,0.006428 -medpagetoday.com,wc09.net,inbound,019,Americas,0 -meetmemail.com,meetmemail.com,inbound,019,Americas,0 -megasenders.com,megasenders.com,inbound,019,Americas,0.07662 -memberdealsusa.com,memberdealsusa.com,inbound,019,Americas,0 -menswearhouse.com,menswearhouse.com,inbound,019,Americas,0 -mercadojobs.com,sendgrid.net,inbound,019,Americas,1 -mercola.com,mercola.com,inbound,019,Americas,0.000369 -messagegears.net,messagegears.net,inbound,019,Americas,0 -met-art.com,hydentra.com,inbound,019,Americas,1 -mgo.com,bronto.com,inbound,019,Americas,0 -michaels.com,chtah.net,inbound,019,Americas,0 -michaels.com,michaels.com,inbound,019,Americas,0 -microcentermedia.com,bfi0.com,inbound,019,Americas,0 -microsoft.com,hotmail.{...},inbound,019,Americas,1 -microsoft.com,msn.com,inbound,019,Americas,1 -midnightsunsafelist.com,zoothost.com,inbound,019,Americas,0 -milfaholic.com,iverificationsystems.com,inbound,019,Americas,0 -miltnews.com,miltnews.com,inbound,019,Americas,0 -mindbodyonline.com,mindbodyonline.com,inbound,019,Americas,1 -mindfieldonline.com,mindfieldonline.com,inbound,019,Americas,0 -mindmoviesmail.com,mindmoviesmail.com,inbound,019,Americas,0.004239 -mindvalleymail3.com,mindvalleymail3.com,inbound,019,Americas,0 -mint.com,mint.com,inbound,019,Americas,0 -minted.com,messagelabs.com,inbound,019,Americas,0.999669 -missselfridge.com,wallis-fashion.com,inbound,019,Americas,0 -mistersafelist.com,zoothost.com,inbound,019,Americas,0 -mit.edu,mit.edu,inbound,019,Americas,0.869568 -mjinn.com,mailurja.com,inbound,019,Americas,0 -mkt015.com,mkt015.com,inbound,019,Americas,0 -mkt022.com,mkt022.com,inbound,019,Americas,0 -mkt063.com,mkt063.com,inbound,019,Americas,0 -mkt1136.com,mkt1136.com,inbound,019,Americas,0 -mkt1985.com,fmlinks.net,inbound,019,Americas,0 -mkt2010.com,mkt2010.com,inbound,019,Americas,0 -mkt2106.com,mkt2106.com,inbound,019,Americas,0 -mkt2170.com,mkt2170.com,inbound,019,Americas,0 -mkt2181.com,mkt2181.com,inbound,019,Americas,0 -mkt2615.com,mkt2615.com,inbound,019,Americas,0 -mkt2813.com,mkt2813.com,inbound,019,Americas,0 -mkt2944.com,mkt2944.com,inbound,019,Americas,0 -mkt3134.com,mkt3134.com,inbound,019,Americas,0 -mkt3142.com,mkt3142.com,inbound,019,Americas,0 -mkt3156.com,mkt3156.com,inbound,019,Americas,0 -mkt3203.com,mkt3203.com,inbound,019,Americas,0 -mkt346.com,mkt346.com,inbound,019,Americas,0 -mkt3544.com,mkt3544.com,inbound,019,Americas,0 -mkt3622.com,mkt3622.com,inbound,019,Americas,0 -mkt3682.com,mkt3682.com,inbound,019,Americas,0 -mkt3690.com,mkt3690.com,inbound,019,Americas,0 -mkt3695.com,mkt3695.com,inbound,019,Americas,0 -mkt3804.com,mkt3804.com,inbound,019,Americas,0 -mkt3815.com,mkt3815.com,inbound,019,Americas,0 -mkt3952.com,xoom.com,inbound,019,Americas,0 -mkt4355.com,mkt4355.com,inbound,019,Americas,0 -mkt4364.com,mkt4364.com,inbound,019,Americas,0 -mkt459.com,mkt459.com,inbound,019,Americas,0 -mkt4701.com,mkt4701.com,inbound,019,Americas,0 -mkt4728.com,mkt4728.com,inbound,019,Americas,0 -mkt4731.com,mkt4731.com,inbound,019,Americas,0 -mkt4738.com,mkt4738.com,inbound,019,Americas,0 -mkt5071.com,mkt5071.com,inbound,019,Americas,0 -mkt5098.com,mkt5098.com,inbound,019,Americas,0 -mkt5131.com,mkt5131.com,inbound,019,Americas,0 -mkt5144.com,mkt5144.com,inbound,019,Americas,0 -mkt5144.com,mkt5980.com,inbound,019,Americas,0 -mkt5144.com,mkt5981.com,inbound,019,Americas,0 -mkt5181.com,mkt5181.com,inbound,019,Americas,0 -mkt5269.com,mkt5269.com,inbound,019,Americas,0 -mkt529.com,mkt529.com,inbound,019,Americas,0 -mkt5297.com,mkt5297.com,inbound,019,Americas,0 -mkt5297.com,mkt5309.com,inbound,019,Americas,0 -mkt5371.com,mkt5371.com,inbound,019,Americas,0 -mkt5806.com,mkt5806.com,inbound,019,Americas,0 -mkt5934.com,mkt5934.com,inbound,019,Americas,0 -mkt5937.com,mkt5937.com,inbound,019,Americas,0 -mkt5970.com,mkt5970.com,inbound,019,Americas,0 -mkt6100.com,mkt6098.com,inbound,019,Americas,0 -mkt6276.com,mkt6276.com,inbound,019,Americas,0 -mkt6323.com,mkt6323.com,inbound,019,Americas,0 -mkt746.com,mkt746.com,inbound,019,Americas,0 -mkt824.com,mkt869.com,inbound,019,Americas,0 -mktdillards.com,mktdillards.com,inbound,019,Americas,0 -mo1send.com,mo1send.com,inbound,019,Americas,0 -mobly.com.br,mobly.com.br,inbound,019,Americas,0 -modellsemail.com,n-email.net,inbound,019,Americas,0 -moneymorning.com,moneymappress.com,inbound,019,Americas,0 -monster.com,monster.com,inbound,019,Americas,0 -monster.com,tmpw.net,inbound,019,Americas,0.000503 -moon-ray.com,moon-ray.com,inbound,019,Americas,0 -mooresclothing.com,mooresclothing.com,inbound,019,Americas,0 -moveon.org,moveon.org,inbound,019,Americas,0.996147 -mrmlsmatrix.com,mrmlsmatrix.com,inbound,019,Americas,0 -ms.com,ms.com,inbound,019,Americas,1 -msn.com,hotmail.{...},inbound,019,Americas,1 -msn.com,hotmail.{...},outbound,019,Americas,1 -mta.info,ealert.com,inbound,019,Americas,0 -mtasv.net,mtasv.net,inbound,019,Americas,0.999999 -musicnotes-alerts.com,mybuys.com,inbound,019,Americas,0 -mustanglist.com,mustanglist.com,inbound,019,Americas,0 -mycheapoair.com,mycheapoair.com,inbound,019,Americas,0.957931 -mydailymoment.biz,mydailymoment.biz,inbound,019,Americas,0 -mydailymoment.info,mydailymoment.info,inbound,019,Americas,0 -mydailymoment.net,mydailymoment.net,inbound,019,Americas,0 -mydailymoment.us,mydailymoment.us,inbound,019,Americas,0 -myfedloan.org,aessuccess.org,inbound,019,Americas,0.328917 -myfxbook.com,myfxbook.com,inbound,019,Americas,0 -mygreatlakes.org,glhec.org,inbound,019,Americas,0.000247 -myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,019,Americas,0 -mymeijer.com,mymeijer.com,inbound,019,Americas,0 -myngp.com,ngpweb.com,inbound,019,Americas,0 -myperfectsale.com,myperfectsale.com,inbound,019,Americas,1 -myprotein.com,thehut.com,inbound,019,Americas,0 -mysafelistmailer.com,mysafelistmailer.com,inbound,019,Americas,0.00033 -mysmartprice.com,itzwow.com,inbound,019,Americas,1 -myzamanamail.com,myzamanamail.com,inbound,019,Americas,0 -n-email.net,n-email.net,inbound,019,Americas,0 -n-email1.net,n-email1.net,inbound,019,Americas,0 -n-email4.net,n-email4.net,inbound,019,Americas,0 -nanomail.com.br,araie.com.br,inbound,019,Americas,1 -napitipp.hu,napitipp.hu,inbound,019,Americas,0 -nasa.gov,nasa.gov,inbound,019,Americas,0.101289 -nastygal.com,bronto.com,inbound,019,Americas,0 -nationbuilder.com,nationbuilder.com,inbound,019,Americas,1 -nature.com,nature.com,inbound,019,Americas,0 -naukri.com,naukri.com,inbound,019,Americas,0.000533 -nauta.cu,etecsa.net,inbound,019,Americas,0 -nba.com,nba.com,inbound,019,Americas,0.005829 -nbaa.org,nbaa.org,inbound,019,Americas,0 -ncl.com,ncl.com,inbound,019,Americas,0 -neimanmarcusemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -net-a-porter.com,net-a-porter.com,inbound,019,Americas,0 -net-empregos.com,net-empregos.com,inbound,019,Americas,0 -netcommunity1.com,blackbaud.com,inbound,019,Americas,0 -netflix.com,netflix.com,inbound,019,Americas,1 -netopia.pt,netopia.pt,inbound,019,Americas,0 -netprosoftmail.com,netprosoftmail.com,inbound,019,Americas,0 -netsuite.com,netsuite.com,inbound,019,Americas,0.60682 -networkworld.com,networkworld.com,inbound,019,Americas,0 -newgrounds.com,newgrounds.com,inbound,019,Americas,0 -newrelic.com,sendlabs.com,inbound,019,Americas,0 -newspaperdirect.com,newspaperdirect.com,inbound,019,Americas,0.000177 -newyorktimesinfo.com,newyorktimesinfo.com,inbound,019,Americas,0 -nexcess.net,nexcess.net,inbound,019,Americas,0.039513 -nextdoor.com,mailgun.info,inbound,019,Americas,1 -nfl.com,bfi0.com,inbound,019,Americas,0 -nih.gov,nih.gov,inbound,019,Americas,8.8e-05 -ninewestmail.com,ninewestmail.com,inbound,019,Americas,0 -ning.com,ning.com,inbound,019,Americas,0 -nixle.com,nixle.com,inbound,019,Americas,0 -nl00.net,netline.com,inbound,019,Americas,5e-06 -nl00.net,nl00.net,inbound,019,Americas,0 -nongnu.org,gnu.org,inbound,019,Americas,1 -nordstrom.com,taleo.net,inbound,019,Americas,1 -nortonfromsymantec.com,rsys1.com,inbound,019,Americas,0 -novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,019,Americas,0 -numbersusa.com,numbersusa.com,inbound,019,Americas,0 -nyandcompany.com,nyandcompany.com,inbound,019,Americas,0 -ocadomail.com,ocadomail.com,inbound,019,Americas,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,019,Americas,0 -ofertasefacil.com.br,ofertasefacil.com.br,inbound,019,Americas,0 -ofertop.pe,icommarketing.com,inbound,019,Americas,0 -officedepot.com,officedepot.com,inbound,019,Americas,2.6e-05 -olympiaedge.net,olympiaedge.net,inbound,019,Americas,0 -oneindia.in,infimail.com,inbound,019,Americas,0 -oneindia.in,mailurja.com,inbound,019,Americas,0 -onepatriotplace.com,britecast.com,inbound,019,Americas,0 -onestopplus.com,neolane.net,inbound,019,Americas,0 -onetravelspecials.com,onetravelspecials.com,inbound,019,Americas,0.365386 -online.com,cnet.com,inbound,019,Americas,0 -onthecitymail.org,onthecitymail.org,inbound,019,Americas,1 -oo155.com,bsftransmit7.com,inbound,019,Americas,0 -oo155.com,oo155.com,inbound,019,Americas,0 -openstack.org,openstack.org,inbound,019,Americas,0.997933 -openstackmail.com,infimail.com,inbound,019,Americas,0 -opentable.com,opentable.com,inbound,019,Americas,2e-06 -opinionoutpost.com,opinionoutpost.com,inbound,019,Americas,0 -opinionsquare.com,opinionsquare.com,inbound,019,Americas,0 -oprah.com,oprah.com,inbound,019,Americas,0 -opticsplanet.com,opticsplanet.com,inbound,019,Americas,0 -optonline.net,cv.net,inbound,019,Americas,0 -optonline.net,optonline.net,outbound,019,Americas,0 -oriental-trading.com,oriental-trading.com,inbound,019,Americas,0 -outlook.com,hotmail.{...},inbound,019,Americas,1 -outlook.com,hotmail.{...},outbound,019,Americas,1 -overnightprints.com,chtah.net,inbound,019,Americas,0 -ovuline.com,ovuline.com,inbound,019,Americas,1 -pagoda.com,zales.com,inbound,019,Americas,0 -pagseguro.com.br,uol.com.br,inbound,019,Americas,0 -pair.com,pair.com,inbound,019,Americas,0.893803 -palmscasinoresort.com,palmscasinoresort.com,inbound,019,Americas,0 -pampers.com,bfi0.com,inbound,019,Americas,0 -pandaresearch.com,pandaresearch.com,inbound,019,Americas,0 -pandora.com,pandora.com,inbound,019,Americas,1 -pandora.net,pandora.net,inbound,019,Americas,1 -parents.com,meredith.com,inbound,019,Americas,0 -parkmobileglobal.com,parkmobile.us,inbound,019,Americas,0 -path.com,path.com,inbound,019,Americas,1 -patriotupdate.com,inboxfirst.com,inbound,019,Americas,0 -payback.in,chtah.net,inbound,019,Americas,0 -paytm.com,paytm.com,inbound,019,Americas,1 -pbteen.com,pbteen.com,inbound,019,Americas,0 -pch.com,ed10.com,inbound,019,Americas,0 -pchfrontpage.com,ed10.com,inbound,019,Americas,0 -pchlotto.com,ed10.com,inbound,019,Americas,0 -pchplayandwin.com,ed10.com,inbound,019,Americas,0 -pchsearch.com,ed10.com,inbound,019,Americas,0 -pcmag.com,ittoolbox.com,inbound,019,Americas,0 -pcworld.com,pcworld.com,inbound,019,Americas,0 -pd25.com,pd25.com,inbound,019,Americas,1 -pearlsofwealth.com,pearlsofwealth.com,inbound,019,Americas,1 -peartreegreetings.com,rexcraft.com,inbound,019,Americas,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,019,Americas,0 -pepboys.com,pepboys.com,inbound,019,Americas,0 -pepperfry.com,epidm.net,inbound,019,Americas,0 -perfectpriceindia.com,infimail.com,inbound,019,Americas,0 -perfora.net,perfora.net,inbound,019,Americas,1 -permissionresearch.com,permissionresearch.com,inbound,019,Americas,0 -personalliberty.com,personalliberty.com,inbound,019,Americas,1 -pga.com,pga.com,inbound,019,Americas,0 -pgeveryday.com,bfi0.com,inbound,019,Americas,0 -phoenix.edu,phoenix.edu,inbound,019,Americas,0 -phsmtpbox.com,phsmtpbox.com,inbound,019,Americas,0 -pinterest.com,pinterest.com,inbound,019,Americas,1 -pivotaltracker.com,pivotaltracker.com,inbound,019,Americas,1 -pixable.com,pixable.com,inbound,019,Americas,1 -pizzahut.com,quikorder.com,inbound,019,Americas,0 -plexapp.com,plex.tv,inbound,019,Americas,1 -pmailus.com,patrontechnology.com,inbound,019,Americas,0 -pnc.com,messagelabs.com,inbound,019,Americas,0.997194 -pobox.com,pobox.com,inbound,019,Americas,0.975372 -pof.com,plentyoffish.co.uk,inbound,019,Americas,0 -pogo.com,pogo.com,inbound,019,Americas,0 -polyvore.com,polyvore.com,inbound,019,Americas,1 -postcardfromhell.com,cyberthugs.com,inbound,019,Americas,1 -potterybarn.com,potterybarn.com,inbound,019,Americas,0 -potterybarnkids.com,potterybarnkids.com,inbound,019,Americas,0 -preferredpetclub.com,preferredpetclub.com,inbound,019,Americas,0 -presslaff.net,dat-e-baseonline.com,inbound,019,Americas,0 -pressmartmail.com,pressmartmail.com,inbound,019,Americas,0 -priorityoneemail.com,priorityoneemail.com,inbound,019,Americas,0 -private-elist.com,private-elist.com,inbound,019,Americas,0 -progressive.com,progressive.com,inbound,019,Americas,0.999893 -promedmail.org,childrenshospital.org,inbound,019,Americas,1 -propertysolutions.com,propertysolutions.com,inbound,019,Americas,1 -prospectgeysercoop.com,prospectgeysercoop.com,inbound,019,Americas,1 -providesupport.com,providesupport.com,inbound,019,Americas,0.000103 -proxyvote.com,adp-ics.com,inbound,019,Americas,0.908635 -psu.edu,psu.edu,inbound,019,Americas,0.073843 -puffinmailer.com,zoothost.com,inbound,019,Americas,0 -purewow.com,purewow.com,inbound,019,Americas,0 -purlsmail.com,purlsmail.com,inbound,019,Americas,0 -pxsmail.com,pxsmail.com,inbound,019,Americas,0 -q.com,synacor.com,inbound,019,Americas,0.999734 -qemailserver.com,qemailserver.com,inbound,019,Americas,0 -qtropnews.com,qtropnews.com,inbound,019,Americas,1.7e-05 -qualicorp.com.br,qualicorp.com.br,inbound,019,Americas,1 -qualitysafelist.com,zoothost.com,inbound,019,Americas,0 -queopinas.com,confirmit.com,inbound,019,Americas,1 -quickrewards.net,quickrewards.net,inbound,019,Americas,0 -quikr.com,quikr.com,inbound,019,Americas,0 -quora.com,quora.com,inbound,019,Americas,1 -qvcemail.com,qvcemail.com,inbound,019,Americas,0 -radarsystems.net,radarsystems.net,inbound,019,Americas,1 -radiantretailapps.com,radiantretailapps.com,inbound,019,Americas,0 -radioshack.com,radioshack.com,inbound,019,Americas,0 -rapattoni.com,rapmls.com,inbound,019,Americas,0 -razerzone.com,chtah.net,inbound,019,Americas,0 -rbc.com,rbc.com,inbound,019,Americas,0.003061 -rdio.com,rdio.com,inbound,019,Americas,1 -realtor.org,realtor.org,inbound,019,Americas,0 -realtytrac.com,realtytrac.com,inbound,019,Americas,0.001681 -recipe.com,meredith.com,inbound,019,Americas,0 -recruit.net,recruit.net,inbound,019,Americas,0 -redbox.com,redbox.com,inbound,019,Americas,0 -redfin.com,redfin.com,inbound,019,Americas,0 -redtri.com,redtri.com,inbound,019,Americas,0 -reebokusnews.com,reebokusnews.com,inbound,019,Americas,0 -reebonz.com,ed10.com,inbound,019,Americas,0 -reebonz.com,reebonz.com,inbound,019,Americas,1 -registro.br,registro.br,inbound,019,Americas,0 -rent.com,rent.com,inbound,019,Americas,0.000397 -rentalcars.com,rentalcars.com,inbound,019,Americas,1 -renweb.com,renweb.com,inbound,019,Americas,0 -researchgate.net,researchgate.net,inbound,019,Americas,0 -restorationhardware.com,restorationhardware.com,inbound,019,Americas,0 -retailmenot.com,retailmenot.com,inbound,019,Americas,0 -reverbnation.com,reverbnation.com,inbound,019,Americas,0 -rewardme.in,bfi0.com,inbound,019,Americas,0 -ricardoeletro.com.br,allin.com.br,inbound,019,Americas,0 -rigzonemail.com,rigzonemail.com,inbound,019,Americas,0 -ripleyperu.com.pe,icommarketing.com,inbound,019,Americas,0 -riseup.net,riseup.net,inbound,019,Americas,1 -rivamail.com,mailurja.com,inbound,019,Americas,0 -rnmk.com,rnmk.com,inbound,019,Americas,0 -roamans.com,neolane.net,inbound,019,Americas,0 -rockwellcollins.com,rockwellcollins.com,inbound,019,Americas,1 -rpinow.org,app-info.net,inbound,019,Americas,0 -rr.com,rr.com,inbound,019,Americas,0.03696 -rr.com,rr.com,outbound,019,Americas,0 -rsgsv.net,rsgsv.net,inbound,019,Americas,0 -rsvpsv.net,rsvpsv.net,inbound,019,Americas,0 -rsvpsv.net,send.esp.br,inbound,019,Americas,0 -rsys2.com,amfam.com,inbound,019,Americas,0 -rsys2.com,cheaptickets.com,inbound,019,Americas,0 -rsys2.com,dishnetworkmail.com,inbound,019,Americas,0 -rsys2.com,e-comms.net,inbound,019,Americas,0 -rsys2.com,eharmony.com,inbound,019,Americas,0 -rsys2.com,fathead.com,inbound,019,Americas,0 -rsys2.com,intuit.com,inbound,019,Americas,0 -rsys2.com,kmart.com,inbound,019,Americas,0 -rsys2.com,kohlernews.com,inbound,019,Americas,0 -rsys2.com,lego.com,inbound,019,Americas,0 -rsys2.com,lenovo.com,inbound,019,Americas,0 -rsys2.com,modcloth.com,inbound,019,Americas,0 -rsys2.com,moxieinteractive.com,inbound,019,Americas,0 -rsys2.com,orbitz.com,inbound,019,Americas,0 -rsys2.com,payless.com,inbound,019,Americas,0 -rsys2.com,petsathome.com,inbound,019,Americas,0 -rsys2.com,quizzle.com,inbound,019,Americas,0 -rsys2.com,robeez.com,inbound,019,Americas,0 -rsys2.com,rsys1.com,inbound,019,Americas,0 -rsys2.com,rsys2.com,inbound,019,Americas,0 -rsys2.com,rsys3.com,inbound,019,Americas,0 -rsys2.com,rsys4.com,inbound,019,Americas,0 -rsys2.com,saucony.com,inbound,019,Americas,0 -rsys2.com,sears.com,inbound,019,Americas,0 -rsys2.com,shopbop.com,inbound,019,Americas,0 -rsys2.com,southwest.com,inbound,019,Americas,0 -rsys2.com,speeddatemail.com,inbound,019,Americas,0 -rsys2.com,thecompanystore.com,inbound,019,Americas,0 -rsys2.com,theknot.com,inbound,019,Americas,0 -rsys5.com,alibris.com,inbound,019,Americas,0 -rsys5.com,allstate-email.com,inbound,019,Americas,0 -rsys5.com,beachmint.com,inbound,019,Americas,0 -rsys5.com,belleandclive.com,inbound,019,Americas,0 -rsys5.com,br.dk,inbound,019,Americas,0 -rsys5.com,charlotterusse.com,inbound,019,Americas,0 -rsys5.com,comixology.com,inbound,019,Americas,0 -rsys5.com,cottonon.com,inbound,019,Americas,0 -rsys5.com,ediblearrangements.com,inbound,019,Americas,0 -rsys5.com,emailworldmarket.com,inbound,019,Americas,0 -rsys5.com,farfetch.com,inbound,019,Americas,0 -rsys5.com,frhiemailcommunications.com,inbound,019,Americas,0 -rsys5.com,harryanddavid.com,inbound,019,Americas,0 -rsys5.com,hollandandbarrett.com,inbound,019,Americas,0 -rsys5.com,icing.com,inbound,019,Americas,0 -rsys5.com,indigo.ca,inbound,019,Americas,0 -rsys5.com,jabong.com,inbound,019,Americas,0 -rsys5.com,jcrew.com,inbound,019,Americas,0 -rsys5.com,jjill.com,inbound,019,Americas,0 -rsys5.com,kanui.com.br,inbound,019,Americas,0 -rsys5.com,kirklands.com,inbound,019,Americas,0 -rsys5.com,lanebryant.com,inbound,019,Americas,0 -rsys5.com,lazada.com,inbound,019,Americas,0 -rsys5.com,leapfrog.com,inbound,019,Americas,0 -rsys5.com,llbean.com,inbound,019,Americas,0 -rsys5.com,lojascolombo.com.br,inbound,019,Americas,0 -rsys5.com,madewell.com,inbound,019,Americas,0 -rsys5.com,magazineluiza.com.br,inbound,019,Americas,0 -rsys5.com,missguided.co.uk,inbound,019,Americas,0 -rsys5.com,moma.org,inbound,019,Americas,0 -rsys5.com,nationalgeographic.com,inbound,019,Americas,0 -rsys5.com,neat.com,inbound,019,Americas,0 -rsys5.com,newbalance.com,inbound,019,Americas,0 -rsys5.com,news-voeazul.com.br,inbound,019,Americas,0 -rsys5.com,nordstrom.com,inbound,019,Americas,0 -rsys5.com,normthompson.com,inbound,019,Americas,0 -rsys5.com,novomundo.com.br,inbound,019,Americas,0 -rsys5.com,ourdeal.com.au,inbound,019,Americas,0 -rsys5.com,pier1.com,inbound,019,Americas,0 -rsys5.com,productmadness.com,inbound,019,Americas,0 -rsys5.com,rainbowshops.com,inbound,019,Americas,0 -rsys5.com,rei.com,inbound,019,Americas,0 -rsys5.com,roadrunnersports.com,inbound,019,Americas,0 -rsys5.com,rosettastone.com,inbound,019,Americas,0 -rsys5.com,seamless.com,inbound,019,Americas,0 -rsys5.com,serenaandlily.com,inbound,019,Americas,0 -rsys5.com,smiles.com.br,inbound,019,Americas,0 -rsys5.com,soubarato.com.br,inbound,019,Americas,0 -rsys5.com,strava.com,inbound,019,Americas,0 -rsys5.com,submarino.com.br,inbound,019,Americas,0 -rsys5.com,thewalkingcompany.com,inbound,019,Americas,0 -rsys5.com,tigerdirect.com,inbound,019,Americas,0 -rsys5.com,udemy.com,inbound,019,Americas,0 -rsys5.com,vitaminshoppe.com,inbound,019,Americas,0 -rsys5.com,vpusa.com,inbound,019,Americas,0 -rsys5.com,walmart.com.br,inbound,019,Americas,0 -rsys5.com,worldofwatches.com,inbound,019,Americas,0 -rsys5.com,xfinity.com,inbound,019,Americas,0 -rue21email.com,rue21email.com,inbound,019,Americas,0 -ruum.com,ruum.com,inbound,019,Americas,0 -saavn.com,saavn.com,inbound,019,Americas,1 -safelistextreme.com,quantumsafelist.com,inbound,019,Americas,0 -safelistpro.com,safelistpro.com,inbound,019,Americas,0.00014 -safeway.com,chtah.com,inbound,019,Americas,0 -sailthru.com,sailthru.com,inbound,019,Americas,0 -saks.com,saks.com,inbound,019,Americas,0 -saksoff5th.com,saksoff5th.com,inbound,019,Americas,0 -salememail.net,salememail.net,inbound,019,Americas,0 -salesforce.com,postini.com,inbound,019,Americas,0.816952 -salesforce.com,salesforce.com,inbound,019,Americas,1 -salesforce.com,salesforce.com,outbound,019,Americas,1 -salliemae.com,salliemae.com,inbound,019,Americas,1 -salsalabs.net,salsalabs.net,inbound,019,Americas,0 -samashmusic.com,wc09.net,inbound,019,Americas,0 -samsclub.com,m0.net,inbound,019,Americas,0 -sans.org,sans.org,inbound,019,Americas,0.497629 -santander.cl,santander.cl,inbound,019,Americas,0.999992 -santander.cl,santandersantiago.cl,inbound,019,Americas,1 -sassieshop.com,sassieshop.com,inbound,019,Americas,0.025887 -savelivefresh.com,livesavemail.com,inbound,019,Americas,1 -savingstar.com,savingstar.com,inbound,019,Americas,1 -sbcglobal.net,yahoo.{...},inbound,019,Americas,1 -sbcglobal.net,yahoodns.net,outbound,019,Americas,1 -sc.com,messagelabs.com,inbound,019,Americas,0.996156 -schwab.com,schwab.com,inbound,019,Americas,0.025055 -seaworld.com,seaworld.com,inbound,019,Americas,0 -securence.com,securence.com,inbound,019,Americas,0.670246 -secureserver.net,secureserver.net,inbound,019,Americas,4.4e-05 -seekingalpha.com,seekingalpha.com,inbound,019,Americas,1 -seekingalpha.com,sendgrid.net,inbound,019,Americas,1 -senate.gov,senate.gov,inbound,019,Americas,0.992994 -sendearnings.com,sendearnings.com,inbound,019,Americas,0 -sendgrid.info,sendgrid.net,inbound,019,Americas,1 -sendgrid.me,sendgrid.net,inbound,019,Americas,1 -sendlane.com,sendlane.com,inbound,019,Americas,0 -serpadres.es,chtah.net,inbound,019,Americas,0 -service-now.com,postini.com,inbound,019,Americas,0.9858 -service-now.com,service-now.com,inbound,019,Americas,0.998562 -sfimg.com,sfimarketing.com,inbound,019,Americas,0 -sfly.com,shutterfly.com,inbound,019,Americas,0 -shaadi.com,shaadi.com,inbound,019,Americas,0.00037 -shadowshopper.com,shadowshopper.com,inbound,019,Americas,5.6e-05 -shaw.ca,shaw.ca,inbound,019,Americas,0 -shaw.ca,shaw.ca,outbound,019,Americas,1 -sheplers.com,sheplers.com,inbound,019,Americas,0 -shiftplanning.com,shiftplanning.com,inbound,019,Americas,0 -shiksha.com,shiksha.com,inbound,019,Americas,0 -shoedazzle.com,shoedazzle.com,inbound,019,Americas,0 -shoes.com,famousfootwear.com,inbound,019,Americas,0 -shopbonton.com,shopbonton.com,inbound,019,Americas,0 -shopcluesemail.com,shopcluesemail.com,inbound,019,Americas,0 -shopcluesmail.com,shopcluesmail.com,inbound,019,Americas,0 -shophq.com,shophq.com,inbound,019,Americas,0 -shopkick.com,shopkick.com,inbound,019,Americas,0 -shopko.com,shopko.com,inbound,019,Americas,0 -shoppersoptimum.ca,thindata.net,inbound,019,Americas,0 -shoptime.com,shoptime.com,inbound,019,Americas,0 -shtyle.fm,shtyle.fm,inbound,019,Americas,0 -sierratradingpost.com,sierratradingpost.com,inbound,019,Americas,0.000885 -sigmabeauty.com,lstrk.net,inbound,019,Americas,1 -sii.cl,sii.cl,inbound,019,Americas,0 -simplyhired.com,simplyhired.com,inbound,019,Americas,0 -siriusxm.com,xmradio.com,inbound,019,Americas,0 -sitecore-mailer.com,sendlabs.com,inbound,019,Americas,0 -sittercity.com,sittercity.com,inbound,019,Americas,0 -sixflags.com,sixflags.com,inbound,019,Americas,0 -skillpages-mailer.com,sendlabs.com,inbound,019,Americas,0 -skillpages-mailer.com,skillpagesmail.com,inbound,019,Americas,0 -sky.com,sky.com,inbound,019,Americas,0 -skype.com,delivery.net,inbound,019,Americas,0 -sld.cu,sld.cu,inbound,019,Americas,1 -slidesharemail.com,newslettergrid.com,inbound,019,Americas,1 -slidesharemail.com,slidesharemail.com,inbound,019,Americas,1 -smartbrief.com,smartbrief.com,inbound,019,Americas,0 -smartdraw.com,smartdraw.com,inbound,019,Americas,0 -smartertravel.com,smartertravelmedia.com,inbound,019,Americas,0.021434 -smartphoneexperts.com,mailgun.net,inbound,019,Americas,1 -snapretail.com,snapretail.com,inbound,019,Americas,1 -socialappsmail.com,socialappsmail.com,inbound,019,Americas,1 -solesociety.com,bronto.com,inbound,019,Americas,0 -solosenders.com,megasenders.com,inbound,019,Americas,8.2e-05 -solosenders.com,traxweb.org,inbound,019,Americas,0 -soma.com,soma.com,inbound,019,Americas,0 -someecards.com,someecards.com,inbound,019,Americas,0 -songkick.com,songkick.com,inbound,019,Americas,1 -sony-latin.com,sony-latin.com,inbound,019,Americas,0.01735 -sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,019,Americas,0 -sourceforge.net,sourceforge.net,inbound,019,Americas,1 -southwest.com,southwest.com,inbound,019,Americas,0 -sp.gov.br,sp.gov.br,inbound,019,Americas,0.496357 -sparkpeople.com,sparkpeople.com,inbound,019,Americas,0 -spectersoft.com,spectersoft.com,inbound,019,Americas,0 -spencersonline.com,spencersonline.com,inbound,019,Americas,0 -spiritairlines.com,ctd004.net,inbound,019,Americas,0 -spiritairlines.com,ctd005.net,inbound,019,Americas,0 -splitwise.com,splitwise.com,inbound,019,Americas,1 -sportlobster.com,sportlobster.com,inbound,019,Americas,1 -sportsdirect.com,sportsdirect.com,inbound,019,Americas,0 -sportsline.com,cbsig.net,inbound,019,Americas,1 -sportsmansguide.com,sportsmansguide.com,inbound,019,Americas,0.736903 -sprint.com,m0.net,inbound,019,Americas,0 -squareup.com,squareup.com,inbound,019,Americas,0.986452 -stanford.edu,highwire.org,inbound,019,Americas,1 -stanford.edu,stanford.edu,inbound,019,Americas,0.93025 -stansberryresearch.com,stansberryresearch.com,inbound,019,Americas,0.001541 -staples.com,staples.com,inbound,019,Americas,0.006379 -starbucks.com,starbucks.com,inbound,019,Americas,0 -stardockcorporation.com,stardockcorporation.com,inbound,019,Americas,0 -stardockentertainment.info,stardockentertainment.info,inbound,019,Americas,0 -startribune.com,startribune.com,inbound,019,Americas,0.001728 -startwire.com,jobsreport.com,inbound,019,Americas,1 -startwire.com,startwire.com,inbound,019,Americas,1 -state.gov,state.gov,inbound,019,Americas,0 -statefarm.com,statefarm.com,inbound,019,Americas,1 -steinmart.com,steinmart.com,inbound,019,Americas,0 -stevemadden.com,stevemadden.com,inbound,019,Americas,0 -streetauthoritydaily.com,streetauthoritydaily.com,inbound,019,Americas,0 -stumblemail.com,stumblemail.com,inbound,019,Americas,1 -suafaturanet.com.br,suafaturanet.com.br,inbound,019,Americas,0.995846 -subscribermail.com,subscribermail.com,inbound,019,Americas,0 -sungard.com,postini.com,inbound,019,Americas,0.498265 -sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,019,Americas,1 -superbalist.com,sailthru.com,inbound,019,Americas,0 -supersafemailer.com,zoothost.com,inbound,019,Americas,0 -supremelist.com,onlinehome-server.com,inbound,019,Americas,1 -surlatable.com,surlatable.com,inbound,019,Americas,0 -surveyjobopportunities.com,surveyjobopportunities.com,inbound,019,Americas,3.7e-05 -surveymonkey.com,surveymonkey.com,inbound,019,Americas,0 -surveysavvy.com,surveysavvy.com,inbound,019,Americas,0 -sylectus.com,sylectus.com,inbound,019,Americas,0.361297 -sympatico.ca,hotmail.{...},outbound,019,Americas,1 -taggedmail.com,taggedmail.com,inbound,019,Americas,0 -tamu.edu,tamu.edu,inbound,019,Americas,0.249876 -tangeroutletsusa.com,bronto.com,inbound,019,Americas,0 -target-safelist.com,safelistpro.com,inbound,019,Americas,0.000215 -targetproblaster.com,targetproblaster.com,inbound,019,Americas,0 -targetx.com,targetx.com,inbound,019,Americas,0 -tarot.com,tarot.com,inbound,019,Americas,0 -tastefullysimpleparty.com,bigfootinteractive.com,inbound,019,Americas,0 -tastingtable.com,tastingtable.com,inbound,019,Americas,0 -taxi4sure.net,infimail.com,inbound,019,Americas,0 -teach12.net,teach12.net,inbound,019,Americas,0 -techtarget.com,techtarget.com,inbound,019,Americas,0.001771 -telus.net,telus.net,inbound,019,Americas,0.006226 -ten24mail.com,ten24mail.com,inbound,019,Americas,0 -terra.com,terra.com,inbound,019,Americas,0.000463 -terra.com.br,terra.com,inbound,019,Americas,0 -terra.com.br,terra.com,outbound,019,Americas,0 -tesco.com,tesco.com,inbound,019,Americas,0 -testfunda.com,testfunda.com,inbound,019,Americas,0 -textnow.me,textnow.me,inbound,019,Americas,0 -tgw.com,tgw.com,inbound,019,Americas,0 -theanimalrescuesite.com,theanimalrescuesite.com,inbound,019,Americas,0 -theatermania.com,wc09.net,inbound,019,Americas,0 -thebay.com,thebay.com,inbound,019,Americas,0 -thegrommet.com,lstrk.net,inbound,019,Americas,1 -thehut.com,thehut.com,inbound,019,Americas,0 -thelimited.com,thelimited.com,inbound,019,Americas,0 -themailbagsafelist.com,thomas-j-brown.com,inbound,019,Americas,0 -theoutnet.com,theoutnet.com,inbound,019,Americas,0 -theskimm.com,theskimm.com,inbound,019,Americas,0 -thesovereigninvestor.com,sovereignsociety.com,inbound,019,Americas,0 -thirtyonegifts.com,thirtyonegifts.com,inbound,019,Americas,0 -thoughtful-mind.com,thoughtful-mind.com,inbound,019,Americas,0 -thumbtack.com,thumbtack.com,inbound,019,Americas,1 -ticketmaster.com,ticketmaster.com,inbound,019,Americas,0.769059 -tmart.com,chtah.net,inbound,019,Americas,0 -tmp.com,tmpw.com,inbound,019,Americas,0 -toluna.com,toluna.com,inbound,019,Americas,0 -tomtommailer.com,tomtommailer.com,inbound,019,Americas,0 -topspin.net,topspin.net,inbound,019,Americas,1 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,019,Americas,0 -touchbase2.com,infimail.com,inbound,019,Americas,0 -touchbase2.com,mailurja.com,inbound,019,Americas,0 -touchbasepro.com,touchbasepro.com,inbound,019,Americas,0 -townhallmail.com,townhallmail.com,inbound,019,Americas,0 -townsquaremedia.info,sailthru.com,inbound,019,Americas,0 -toysrus.com,epsl1.com,inbound,019,Americas,0 -trafficboostermailer.com,trafficboostermailer.com,inbound,019,Americas,1 -trafficleads2incomevm.com,zoothost.com,inbound,019,Americas,0 -trafficprolist.com,thomas-j-brown.com,inbound,019,Americas,0 -trialsmith.com,membercentral.com,inbound,019,Americas,0 -tricaemidia.com.br,tricaemidia.com.br,inbound,019,Americas,0 -tripit.com,tripit.com,inbound,019,Americas,0 -tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,019,Americas,0 -tumblr.com,tumblr.com,inbound,019,Americas,1 -twitch.tv,justin.tv,inbound,019,Americas,0 -uber.com,uber.com,inbound,019,Americas,1 -ubi.com,ubi.com,inbound,019,Americas,0 -ubivox.com,ubivox.com,inbound,019,Americas,0.935106 -ucla.edu,ucla.edu,inbound,019,Americas,0.806204 -uhaul.com,uhaul.com,inbound,019,Americas,0.997947 -uiuc.edu,illinois.edu,inbound,019,Americas,0.999815 -ultimateadsites.net,ultimateadsites.net,inbound,019,Americas,1 -umd.edu,umd.edu,inbound,019,Americas,0.021709 -umn.edu,umn.edu,inbound,019,Americas,0.966964 -umpiredigital.com,inboxmarketer.com,inbound,019,Americas,0 -united.com,coair.com,inbound,019,Americas,1 -united.com,united.com,inbound,019,Americas,0 -universalorlando.com,universalorlando.com,inbound,019,Americas,0 -unrollmail.com,unrollmail.com,inbound,019,Americas,1 -uol.com.br,uol.com.br,inbound,019,Americas,0 -uol.com.br,uol.com.br,outbound,019,Americas,0 -upenn.edu,upenn.edu,inbound,019,Americas,0.16521 -upromise.com,delivery.net,inbound,019,Americas,0 -ups.com,ups.com,inbound,019,Americas,0.997955 -urbanoutfitters.com,freepeople.com,inbound,019,Americas,0 -usaa.com,usaa.com,inbound,019,Americas,0.072606 -usafisnews.org,usafisnews.org,inbound,019,Americas,0 -usajobs.gov,opm.gov,inbound,019,Americas,0.931948 -usc.edu,usc.edu,inbound,019,Americas,0.300314 -uscourts.gov,uscourts.gov,inbound,019,Americas,0 -uslargestsafelist.com,zoothost.com,inbound,019,Americas,0 -usndr.com,unisender.com,inbound,019,Americas,1 -usp.br,usp.br,inbound,019,Americas,0.044595 -usps.com,usps.gov,inbound,019,Americas,0.909591 -usps.gov,usps.gov,inbound,019,Americas,0.940315 -ustream.tv,mailgun.net,inbound,019,Americas,1 -ustream.tv,mailgun.us,inbound,019,Americas,1 -utfsm.cl,utfsm.cl,inbound,019,Americas,0.000352 -vacationstogo.com,vacationstogo.com,inbound,019,Americas,0 -vagas.com.br,vagas.com.br,inbound,019,Americas,0 -valuedopinions.co.uk,researchnow-usa.com,inbound,019,Americas,1 -vanheusenrewards.com,vanheusenrewards.com,inbound,019,Americas,0 -velocityfrequentflyer.com,virginaustralia.com,inbound,019,Americas,0 -vente-exclusive.com,vente-exclusive.com,inbound,019,Americas,0 -venusswimwear.net,venusswimwear.net,inbound,019,Americas,0 -verabradleymail.com,verabradleymail.com,inbound,019,Americas,0 -verizon.net,verizon.net,inbound,019,Americas,0.00713 -verizon.net,verizon.net,outbound,019,Americas,0 -verizonwireless.com,verizonwireless.com,inbound,019,Americas,0.972245 -vhmnetworkemail.com,jobdiagnosis.com,inbound,019,Americas,0 -viagogo.com,chtah.net,inbound,019,Americas,0 -viajanet.com.br,viajanet.com.br,inbound,019,Americas,0 -victoriassecret.com,victoriassecret.com,inbound,019,Americas,0 -videotron.ca,videotron.ca,inbound,019,Americas,0.005886 -vikingrivercruises.com,bfi0.com,inbound,019,Americas,0 -vimeo.com,vimeo.com,inbound,019,Americas,0 -vindale.com,vindale.com,inbound,019,Americas,0 -vipvoice.com,npdor.com,inbound,019,Americas,0 -viraladmagnet.com,viraladmagnet.com,inbound,019,Americas,1 -virginia.edu,virginia.edu,inbound,019,Americas,0.055153 -virtualtarget.com.br,virtualtarget.com.br,inbound,019,Americas,0 -vitacost.com,vitacost.com,inbound,019,Americas,0 -vitaladviews.com,zoothost.com,inbound,019,Americas,0 -vivastreet.com,viwii.net,inbound,019,Americas,0 -votervoice.net,votervoice.net,inbound,019,Americas,0 -vresp.com,verticalresponse.com,inbound,019,Americas,0 -vt.edu,vt.edu,inbound,019,Americas,0.978548 -vtext.com,vtext.com,inbound,019,Americas,0 -vtext.com,vtext.com,outbound,019,Americas,1 -vuezone.com,vuezone.com,inbound,019,Americas,0 -vzwpix.com,vtext.com,inbound,019,Americas,0 -wagjag.com,wagjag.com,inbound,019,Americas,0 -walgreens.com,walgreens.com,inbound,019,Americas,0.006128 -wallst.com,wallst.com,inbound,019,Americas,0 -wallstreetdaily.com,wallstreetdaily.com,inbound,019,Americas,0 -waterstones.com,waterstones.com,inbound,019,Americas,0 -wattpad.com,wattpad.com,inbound,019,Americas,1 -way2movies.net,way2movies.net,inbound,019,Americas,0 -wayfair.com,csnstores.com,inbound,019,Americas,0 -webs.com,epsl1.com,inbound,019,Americas,0 -websaver.ca,websaver.ca,inbound,019,Americas,0 -websitesettings.com,stabletransit.com,inbound,019,Americas,0 -websitewelcome.com,websitewelcome.com,inbound,019,Americas,1 -webstars2k.com,webstars2k.com,inbound,019,Americas,1 -weebly.com,weeblymail.com,inbound,019,Americas,0 -wellsfargo.com,wellsfargo.com,inbound,019,Americas,1.0 -wellsfargoadvisors.com,wellsfargo.com,inbound,019,Americas,1 -westelm.com,westelm.com,inbound,019,Americas,0 -westmarine.com,westmarine.com,inbound,019,Americas,0 -wetransfer.com,wetransfer.com,inbound,019,Americas,1 -wetsealnewsletter.com,wetsealnewsletter.com,inbound,019,Americas,0 -whatcounts.com,wc09.net,inbound,019,Americas,0 -whentowork.com,whentowork.com,inbound,019,Americas,0 -wikia.com,wikia.com,inbound,019,Americas,1 -williams-sonoma.com,williams-sonoma.com,inbound,019,Americas,0 -windstream.net,windstream.net,inbound,019,Americas,0.002172 -wine.com,wine.com,inbound,019,Americas,0 -winkalmail.com,fnbox.com,inbound,019,Americas,0 -wisdomitservices.com,infimail.com,inbound,019,Americas,0 -wldemail-mailer.com,wldemail.com,inbound,019,Americas,0 -womanwithin.com,womanwithin.com,inbound,019,Americas,0 -woodforest.com,woodforest.com,inbound,019,Americas,1 -workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,019,Americas,0 -worldsingles.co,worldsingles.com,inbound,019,Americas,0 -wotif.com,whatcounts.com,inbound,019,Americas,0 -wp.com,wordpress.com,inbound,019,Americas,0 -wuaki.tv,chtah.net,inbound,019,Americas,0 -wustl.edu,app-info.net,inbound,019,Americas,0 -wwe.com,wwe.com,inbound,019,Americas,8e-06 -xen.org,xen.org,inbound,019,Americas,1 -xmeeting.com,xmeeting.com,inbound,019,Americas,1 -xmr3.com,messagereach.com,inbound,019,Americas,0.999933 -xoom.com,xoom.com,inbound,019,Americas,0 -xpnews.com.br,xpnews.com.br,inbound,019,Americas,1 -yahoo.{...},postini.com,inbound,019,Americas,0.664066 -yahoo.{...},yahoo.{...},inbound,019,Americas,0.999 -yahoo.{...},yahoodns.net,outbound,019,Americas,1 -yelp.com,yelpcorp.com,inbound,019,Americas,0 -yipit.com,yipit.com,inbound,019,Americas,1 -ymail.com,yahoo.{...},inbound,019,Americas,1 -ymail.com,yahoodns.net,outbound,019,Americas,1 -yoox.com,yoox.com,inbound,019,Americas,0 -youmail.com,youmail.com,inbound,019,Americas,0 -youreletters3.com,equitymaster.com,inbound,019,Americas,0 -yourezads.com,yourezads.com,inbound,019,Americas,0.002974 -yourezlist.com,simplicityads.com,inbound,019,Americas,9.5e-05 -yourhostingaccount.com,yourhostingaccount.com,inbound,019,Americas,0 -youversion.com,youversion.com,inbound,019,Americas,1 -zacks.com,zacks.com,inbound,019,Americas,0.999687 -zara.com,cheetahmail.com,inbound,019,Americas,0 -zendesk.com,zdsys.com,inbound,019,Americas,1 -zgalleriestyle.com,zgalleriestyle.com,inbound,019,Americas,0 -zibmail.info,zibmail.info,inbound,019,Americas,9e-06 -zinio.com,zinio.com,inbound,019,Americas,0 -zipalerts.com,zipalerts.com,inbound,019,Americas,1 -ziprealty.com,ziprealty.com,inbound,019,Americas,0.994545 -ziprecruiter.com,ziprecruiter.com,inbound,019,Americas,1 -zivamewear.com,infimail.com,inbound,019,Americas,0 -zoothost.com,zoothost.com,inbound,019,Americas,0.107168 -zorpia.com,zorpia.com,inbound,019,Americas,1 -zulily.com,zulily.com,inbound,019,Americas,0 -zzounds.com,zzounds.com,inbound,019,Americas,0 -0101.co.jp,0101.co.jp,inbound,142,Asia,0 -04auto.biz,01auto.biz,inbound,142,Asia,0 -123.com.tw,123.com.tw,inbound,142,Asia,0 -160by2.us,160by2.us,inbound,142,Asia,0 -163.com,163.com,inbound,142,Asia,0.534088 -163.com,netease.com,outbound,142,Asia,1 -17life.com.tw,17life.com.tw,inbound,142,Asia,0 -1lejend.com,asumeru.com,inbound,142,Asia,0 -1lejend.com,asumeru001.com,inbound,142,Asia,0 -33go.com.tw,33go.com.tw,inbound,142,Asia,0 -518.com.tw,518.com.tw,inbound,142,Asia,0 -7net.com.tw,7net.com.tw,inbound,142,Asia,0 -ab0.jp,altovision.co.jp,inbound,142,Asia,0 -activetrail.com,atmailsvr.net,inbound,142,Asia,0 -activetrail.com,mymarketing.co.il,inbound,142,Asia,0 -adchiever.com,kinder-rash-marketing.com,inbound,142,Asia,0 -adityabirla.com,adityabirla.com,inbound,142,Asia,0.00615 -agora.co.il,1host.co.il,inbound,142,Asia,0 -airtel.com,airtel.in,inbound,142,Asia,0.064377 -alertsindia.in,alertsindia.in,inbound,142,Asia,1 -alibaba.com,alibaba.com,inbound,142,Asia,0 -aliexpress.com,alibaba.com,inbound,142,Asia,0 -alipay.com,alipay.com,inbound,142,Asia,1 -alljob.co.il,alljob.co.il,inbound,142,Asia,0 -amazon.{...},amazon.{...},inbound,142,Asia,0 -ana.co.jp,ana.co.jp,inbound,142,Asia,0.534416 -anghami.com,mailgun.net,inbound,142,Asia,1 -apple.com,apple.com,inbound,142,Asia,0.999817 -artscow.com,dyxnet.com,inbound,142,Asia,0.000355 -asus.com,asus.com,inbound,142,Asia,0 -baycrews.co.jp,webcas.net,inbound,142,Asia,0 -beamtele.com,beamtele.com,inbound,142,Asia,0 -beanfun.com,beanfun.com,inbound,142,Asia,1 -belluna.net,belluna.net,inbound,142,Asia,0 -betrend.com,betrend.com,inbound,142,Asia,0 -bharatmatrimony.com,bharatmatrimony.com,inbound,142,Asia,1 -blayn.jp,bserver.jp,inbound,142,Asia,0 -bme.jp,bserver.jp,inbound,142,Asia,0 -bookoffonline.co.jp,bookoffonline.co.jp,inbound,142,Asia,0 -brands4friends.jp,webcas.net,inbound,142,Asia,0 -brandsfever.com,mailgun.net,inbound,142,Asia,1 -buyma.com,buyma.com,inbound,142,Asia,0 -cam2life.com,hinet.net,inbound,142,Asia,0 -camsonline.com,camsonline.com,inbound,142,Asia,0.136261 -careesma.in,careesma.in,inbound,142,Asia,0 -ccavenue.com,avenues.info,inbound,142,Asia,1 -chance.com,data-hotel.net,inbound,142,Asia,0 -chiangcn.com,chiangcn.com,inbound,142,Asia,0 -chinatrust.com.tw,chinatrust.com.tw,inbound,142,Asia,0.001272 -cityheaven.net,cityheaven.net,inbound,142,Asia,0 -clickmailer.jp,clickmailer.jp,inbound,142,Asia,9e-05 -cocacola.co.jp,cocacola.co.jp,inbound,142,Asia,0 -combzmail.jp,combzmail.jp,inbound,142,Asia,0 -communitymatrimony.com,communitymatrimony.com,inbound,142,Asia,1 -conrepmail.com,conrepmail.com,inbound,142,Asia,0 -cookpad.com,cookpad.com,inbound,142,Asia,0 -cpc.gov.in,cpc.gov.in,inbound,142,Asia,0 -crmstyle.com,crmstyle.com,inbound,142,Asia,0 -crocos.jp,crocos.jp,inbound,142,Asia,0 -ctrip.com,ctrip.com,inbound,142,Asia,0.019448 -cuenote.jp,cuenote.jp,inbound,142,Asia,0 -cw.com.tw,cw.com.tw,inbound,142,Asia,0.002535 -cyberlinkmember.com,cyberlinkmember.com,inbound,142,Asia,0 -dena.ne.jp,dena.ne.jp,inbound,142,Asia,0.000249 -dietnavi.com,data-hotel.net,inbound,142,Asia,0 -dip-net.co.jp,dip-net.co.jp,inbound,142,Asia,0 -directresponsemanager.com,wide.ne.jp,inbound,142,Asia,0 -disc.co.jp,disc.co.jp,inbound,142,Asia,0.001051 -dishtv.co.in,dishtv.co.in,inbound,142,Asia,0.047664 -disqus.net,disqus.net,inbound,142,Asia,0 -dks.com.tw,dks.com.tw,inbound,142,Asia,0.018134 -dmm.com,dmm.com,inbound,142,Asia,0 -docomo.ne.jp,docomo.ne.jp,inbound,142,Asia,0 -docomo.ne.jp,docomo.ne.jp,outbound,142,Asia,0 -dreammail.ne.jp,dreammail.jp,inbound,142,Asia,0 -drushim.co.il,drushim.co.il,inbound,142,Asia,0 -dstyleweb.com,dstyleweb.com,inbound,142,Asia,0.001099 -ec21.com,ec21.com,inbound,142,Asia,0.002263 -ecnavi.jp,ecnavi.jp,inbound,142,Asia,0 -en-japan.com,en-japan.com,inbound,142,Asia,0.000204 -eonet.ne.jp,eonet.ne.jp,inbound,142,Asia,0.99955 -epaper.com.tw,epaper.com.tw,inbound,142,Asia,0 -eplus.jp,eplus.jp,inbound,142,Asia,0 -eslitebooks.com,eslitebooks.com,inbound,142,Asia,0 -euromsg.net,euromsg.net,inbound,142,Asia,0 -evaair.com,evaair.com,inbound,142,Asia,0.007363 -ezweb.ne.jp,ezweb.ne.jp,inbound,142,Asia,0.282076 -ezweb.ne.jp,ezweb.ne.jp,outbound,142,Asia,0 -farmersonly.com,mailgun.net,inbound,142,Asia,1 -felissimo.jp,felissimo.jp,inbound,142,Asia,0 -finansbank.com.tr,finansbank.com.tr,inbound,142,Asia,0.927 -fmworld.net,fmworld.net,inbound,142,Asia,0 -fofa.jp,mpme.jp,inbound,142,Asia,0 -freeml.com,gmo-media.jp,inbound,142,Asia,0 -ftchinese.com,ftchinese.com,inbound,142,Asia,0 -fubonshop.com,fubonshop.com,inbound,142,Asia,0 -gamecity.ne.jp,gamecity.ne.jp,inbound,142,Asia,0 -garanti.com.tr,garanti.com.tr,inbound,142,Asia,0.421936 -gilt.jp,gilt.jp,inbound,142,Asia,0 -globalsources.com,globalsources.com,inbound,142,Asia,0.003024 -globetel.com.ph,globetel.com.ph,inbound,142,Asia,1 -gmail.com,asianet.co.th,inbound,142,Asia,0.998511 -gmail.com,au-net.ne.jp,inbound,142,Asia,1 -gmail.com,bbtec.net,inbound,142,Asia,1 -gmail.com,data-hotel.net,inbound,142,Asia,0.000607 -gmail.com,hinet.net,inbound,142,Asia,0.973114 -gmail.com,mtnl.net.in,inbound,142,Asia,0.999527 -gmail.com,netvigator.com,inbound,142,Asia,0.818263 -gmail.com,ocn.ne.jp,inbound,142,Asia,0.99466 -gmail.com,panda-world.ne.jp,inbound,142,Asia,1 -gmail.com,seed.net.tw,inbound,142,Asia,0.992252 -gmail.com,singnet.com.sg,inbound,142,Asia,0.968389 -gmail.com,totbb.net,inbound,142,Asia,0.999876 -gmo.jp,gmo-media.jp,inbound,142,Asia,0 -gmoes.jp,gmoes.jp,inbound,142,Asia,0 -gmt.ne.jp,gmt.ne.jp,inbound,142,Asia,0 -gnavi.co.jp,gnavi.co.jp,inbound,142,Asia,0.002441 -gomaji.com,gomaji.com,inbound,142,Asia,0 -gree.jp,gree.jp,inbound,142,Asia,0 -groupon.{...},groupon.{...},inbound,142,Asia,0 -gunosy.com,gunosy.com,inbound,142,Asia,0.998682 -gurunavi.jp,gurunavi.jp,inbound,142,Asia,0 -hdfcbank.com,powerelay.com,inbound,142,Asia,1 -hdfcbank.net,powerelay.com,inbound,142,Asia,1 -hdfcbank.net,quickvmail.com,inbound,142,Asia,0 -heteml.jp,heteml.jp,inbound,142,Asia,0.950296 -hinet.net,hinet.net,inbound,142,Asia,0.007064 -hinet.net,hinet.net,outbound,142,Asia,0.00565 -home.ne.jp,zaq.ne.jp,inbound,142,Asia,9e-06 -i-part.com.tw,i-part.com.tw,inbound,142,Asia,0.001865 -ibps.in,sify.net,inbound,142,Asia,0 -ibpsorg.org,sify.net,inbound,142,Asia,0 -icicibank.com,icicibank.com,inbound,142,Asia,0.009835 -icicisecurities.com,icicibank.com,inbound,142,Asia,0.000803 -icloud.com,apple.com,inbound,142,Asia,1 -imi.ne.jp,lifemedia.jp,inbound,142,Asia,0 -indiaproperty.com,indiaproperty.com,inbound,142,Asia,0 -intage.co.jp,intage.co.jp,inbound,142,Asia,0.00557 -intuit.com,intuit.com,inbound,142,Asia,0.983698 -isbank.com.tr,isbank.com.tr,inbound,142,Asia,0.009655 -itsmyascent.com,itsmyascent.com,inbound,142,Asia,0 -itunes.com,apple.com,inbound,142,Asia,1 -jcity.com,jcity.com,inbound,142,Asia,0 -jobinthailand.com,jobinthailand.com,inbound,142,Asia,1 -jobmaster.co.il,jobmaster1.co.il,inbound,142,Asia,0 -jobsdbalert.co.id,jobsdbalert.co.id,inbound,142,Asia,0 -jobsdbalert.com,jobsdbalert.com,inbound,142,Asia,0 -jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,142,Asia,0 -jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,142,Asia,0 -jobstreet.com,jobstreet.com,inbound,142,Asia,0 -joshin.co.jp,joshin.co.jp,inbound,142,Asia,8e-06 -kagoya.net,kagoya.net,inbound,142,Asia,0.013052 -karvy.com,karvy.com,inbound,142,Asia,0.045186 -kasikornbank.com,kasikornbank.com,inbound,142,Asia,0 -kotak.com,kotak.com,inbound,142,Asia,0.005566 -krs.bz,tricorn.net,inbound,142,Asia,0 -kvbmail.com,kvbmail.com,inbound,142,Asia,0 -lancers.jp,lancers.jp,inbound,142,Asia,0 -lelong.my,lelong.com.my,inbound,142,Asia,1 -lelong.my,lelong.net.my,inbound,142,Asia,1 -line.me,naver.com,inbound,142,Asia,1 -livedoor.com,livedoor.com,inbound,142,Asia,0 -luxa.jp,luxa.jp,inbound,142,Asia,0 -m3.com,m3.com,inbound,142,Asia,0 -mag2.com,tandem-m.com,inbound,142,Asia,0 -magicbricks.com,tbsl.in,inbound,142,Asia,0 -mail-boss.com,mail-boss.com,inbound,142,Asia,0 -mailgun.org,mailgun.net,inbound,142,Asia,1 -mbga.jp,mbga.jp,inbound,142,Asia,0 -mixi.jp,mixi.jp,inbound,142,Asia,0 -mmagic.jp,mmagic.jp,inbound,142,Asia,0.000531 -mobile01.com,mobile01.com,inbound,142,Asia,0 -monex.co.jp,monex.co.jp,inbound,142,Asia,0.019424 -moneycontrol.com,active18.com,inbound,142,Asia,0 -moneyforward.com,moneyforward.com,inbound,142,Asia,0 -monipla.jp,aainc.co.jp,inbound,142,Asia,0 -monster.co.in,monster.co.in,inbound,142,Asia,0 -morhipo.com,euromsg.net,inbound,142,Asia,0 -mpme.jp,mpme.jp,inbound,142,Asia,0 -mpse.jp,emsaqua.jp,inbound,142,Asia,0 -mpse.jp,emsbeige.jp,inbound,142,Asia,0 -mpse.jp,emsbrown.jp,inbound,142,Asia,0 -mpse.jp,emscyan.jp,inbound,142,Asia,0 -mpse.jp,emsgold.jp,inbound,142,Asia,0 -mpse.jp,emslime.jp,inbound,142,Asia,0 -mpse.jp,emsnavy.jp,inbound,142,Asia,0 -mpse.jp,emspink.jp,inbound,142,Asia,0 -mpse.jp,emssnow.jp,inbound,142,Asia,0 -mpse.jp,mpme.jp,inbound,142,Asia,0 -mpse.jp,yahoo.co.jp,inbound,142,Asia,0 -mynavi.jp,mynavi.jp,inbound,142,Asia,3.5e-05 -naver.com,naver.com,inbound,142,Asia,1 -naver.com,naver.com,outbound,142,Asia,1 -nesinemail.com,euromsg.net,inbound,142,Asia,0 -net-survey.jp,net-survey.jp,inbound,142,Asia,0 -netbk.co.jp,netbk.co.jp,inbound,142,Asia,0.010995 -next-engine.org,next-engine.org,inbound,142,Asia,1 -nextdoor.com,nextdoor.com,inbound,142,Asia,1 -nic.in,relayout.nic.in,inbound,142,Asia,0 -nicovideo.jp,nicovideo.jp,inbound,142,Asia,0 -nifty.com,nifty.com,inbound,142,Asia,0.762068 -nikkei.com,nikkei.co.jp,inbound,142,Asia,0 -nikkeibp.co.jp,nikkeibp.co.jp,inbound,142,Asia,4.9e-05 -nissen.jp,nissen.jp,inbound,142,Asia,0 -ocn.ad.jp,ocn.ad.jp,inbound,142,Asia,0 -ocn.ne.jp,ocn.ad.jp,inbound,142,Asia,0 -p-world.co.jp,p-world.co.jp,inbound,142,Asia,0 -panasonic.jp,panasonic.jp,inbound,142,Asia,0 -pepabo.com,pepabo.com,inbound,142,Asia,0 -pia.jp,pia.jp,inbound,142,Asia,0 -playstation.com,playstation.com,inbound,142,Asia,0 -pointtown.com,gmo-media.jp,inbound,142,Asia,0 -pttplc.com,pttgrp.com,inbound,142,Asia,0 -publicators.com,publicators.com,inbound,142,Asia,0 -qoo10.jp,qoo10.jp,inbound,142,Asia,0 -qq.com,qq.com,inbound,142,Asia,0.999973 -quickbooks.com,intuit.com,inbound,142,Asia,0.99908 -rakuten.co.jp,rakuten.co.jp,inbound,142,Asia,0 -rakuten.co.jp,yahoo.co.jp,inbound,142,Asia,0 -rakuten.ne.jp,rakuten.co.jp,inbound,142,Asia,0 -realus.co.jp,realus.co.jp,inbound,142,Asia,0 -recochoku.jp,recochoku.jp,inbound,142,Asia,0 -rediffmail.com,akadns.net,outbound,142,Asia,0 -rediffmail.com,rediffmail.com,inbound,142,Asia,0 -relianceada.com,relianceada.com,inbound,142,Asia,0.276515 -research-panel.jp,research-panel.jp,inbound,142,Asia,0 -responder.co.il,responder.co.il,inbound,142,Asia,1 -runnet.jp,runnet.jp,inbound,142,Asia,0 -rutenmail.com.tw,rutenmail.com.tw,inbound,142,Asia,0 -sahibinden.com,sahibinden.com,inbound,142,Asia,0 -saisoncard.co.jp,saisoncard.co.jp,inbound,142,Asia,0 -sakura.ne.jp,sakura.ne.jp,inbound,142,Asia,0.673305 -samsung.com,samsung.com,inbound,142,Asia,0.008685 -saramin.co.kr,saramin.co.kr,inbound,142,Asia,0 -sbi.co.in,sbi.co.in,inbound,142,Asia,0 -sbr-inc.co.jp,hdemail.jp,inbound,142,Asia,0 -sc.com,sc.com,inbound,142,Asia,0.99683 -secure.ne.jp,secure.ne.jp,inbound,142,Asia,3.7e-05 -secureserver.net,secureserver.net,inbound,142,Asia,0.000386 -shinseibank.com,shinseibank.com,inbound,142,Asia,0 -shukatsu.jp,shukatsu.jp,inbound,142,Asia,0 -simplymarry.com,tbsl.in,inbound,142,Asia,0 -smartphoneexperts.com,mailgun.net,inbound,142,Asia,1 -smp.ne.jp,smp.ne.jp,inbound,142,Asia,0 -snapdealmail.in,snapdealmail.in,inbound,142,Asia,0 -sofmap.com,sofmap.com,inbound,142,Asia,0.0092 -softbank.jp,softbank.jp,inbound,142,Asia,0 -sony.jp,sony.jp,inbound,142,Asia,0.000654 -sourcenext.info,sourcenext.info,inbound,142,Asia,0 -surfmandelivery.com,surfmandelivery.com,inbound,142,Asia,0 -synergy360.jp,crmstyle.com,inbound,142,Asia,0 -taipeifubon.com.tw,taipeifubon.com.tw,inbound,142,Asia,0 -techgig.com,tbsl.in,inbound,142,Asia,0 -thinkvidya.com,thinkvidya.com,inbound,142,Asia,0 -ticketmonster.co.kr,ticketmonster.co.kr,inbound,142,Asia,0 -timesjobs.com,tbsl.in,inbound,142,Asia,0 -timesjobsmail.com,tbsl.in,inbound,142,Asia,0 -timesofindia.com,indiatimes.com,inbound,142,Asia,0 -tobizaru.jp,tobizaru.jp,inbound,142,Asia,0 -tocoo.jp,aics.ne.jp,inbound,142,Asia,0 -tower.jp,tower.jp,inbound,142,Asia,0 -treemall.com.tw,symphox.com,inbound,142,Asia,0 -tsite.jp,tsite.jp,inbound,142,Asia,0 -tsutaya.co.jp,tsutaya.co.jp,inbound,142,Asia,0 -turkcell.com.tr,turkcell.com.tr,inbound,142,Asia,0.043492 -type.jp,type.jp,inbound,142,Asia,0 -u-shopping.com.tw,u-shopping.com.tw,inbound,142,Asia,0 -udnpaper.com,udnpaper.com,inbound,142,Asia,0.998858 -udnshopping.com,udnshopping.com,inbound,142,Asia,0 -vodafone.com,vodafone.in,inbound,142,Asia,0.617518 -vpass.ne.jp,clickmailer.jp,inbound,142,Asia,0 -vpcontact.com,vpcontact.com,inbound,142,Asia,0 -way2sms.biz,way2sms.biz,inbound,142,Asia,0 -way2sms.in,way2sms.in,inbound,142,Asia,0 -webcas.net,webcas.net,inbound,142,Asia,0 -wechat.com,qq.com,inbound,142,Asia,1 -www.gov.tw,hinet.net,inbound,142,Asia,8e-05 -yahoo.co.jp,yahoo.co.jp,inbound,142,Asia,8e-06 -yahoo.co.jp,yahoo.co.jp,outbound,142,Asia,0 -yahoo.{...},yahoo.co.jp,inbound,142,Asia,0 -yakala.co,euromsg.net,inbound,142,Asia,0 -yesbank.in,yesbank.in,inbound,142,Asia,0.241591 -yodobashi.com,yodobashi.com,inbound,142,Asia,0 -zizigo.com,euromsg.net,inbound,142,Asia,0 -3suisses.be,3suisses.be,inbound,150,Europe,0 -3suisses.fr,3suisses.fr,inbound,150,Europe,0 -aanotifier.nl,aanotifier.nl,inbound,150,Europe,0 -adidas.com,neolane.net,inbound,150,Europe,0 -admail.hu,sanomaonline.hu,inbound,150,Europe,0 -adsender.us,adsender.us,inbound,150,Europe,0 -adverts.ie,adverts.ie,inbound,150,Europe,1 -advfn.com,advfn.com,inbound,150,Europe,0.635382 -agnitas.de,agnitas.de,inbound,150,Europe,0.999265 -airliquide.com,airliquide.com,inbound,150,Europe,1 -alerteimmo.com,alerteimmo.com,inbound,150,Europe,0 -alinea.fr,bp06.net,inbound,150,Europe,0 -allegro.pl,allegro.pl,inbound,150,Europe,0 -allegroup.hu,allegroup.hu,inbound,150,Europe,0 -alza.cz,alza.cz,inbound,150,Europe,0.039449 -alza.sk,alza.cz,inbound,150,Europe,0.027725 -andrewchristian.com,emv8.com,inbound,150,Europe,0 -anpasia.com,anpasia.com,inbound,150,Europe,0 -aprovaconcursos.com.br,eadunicid.com.br,inbound,150,Europe,0 -aruba.it,aruba.it,inbound,150,Europe,0.055666 -ashampoo.com,ashampoo.com,inbound,150,Europe,1 -aswatson.com,emarsys.net,inbound,150,Europe,0 -avira.com,avira.com,inbound,150,Europe,0.042528 -avito.ru,avito.ru,inbound,150,Europe,0.000631 -badoo.com,monopost.com,inbound,150,Europe,1 -balsamik.fr,balsamik.fr,inbound,150,Europe,0 -bancomer.com,postini.com,inbound,150,Europe,1e-05 -bbvacompass.com,postini.com,inbound,150,Europe,0.997006 -be2.com,nmp1.net,inbound,150,Europe,0 -biglion.ru,biglion.ru,inbound,150,Europe,0.999775 -bigmailsender.com,bigmailsender.com,inbound,150,Europe,0 -bioagri.com.br,postini.com,inbound,150,Europe,0.991765 -biomedcentral.com,emv5.com,inbound,150,Europe,0 -blackberry.com,blackberry.com,inbound,150,Europe,0 -blinkboxmusic.com,mediagraft.com,inbound,150,Europe,1 -blogtrottr.com,blogtrottr.com,inbound,150,Europe,0 -blue-compass.com,blue-compass.com,inbound,150,Europe,0 -bmdeda99.com,bmdeda99.com,inbound,150,Europe,0 -bolsfr.fr,colt.net,inbound,150,Europe,0 -bonuszbrigad.hu,bonuszbrigad.hu,inbound,150,Europe,0 -boohooemail.com,smartfocusdigital.net,inbound,150,Europe,0 -booking.com,booking.com,inbound,150,Europe,1 -bouncemanager.it,musvc.com,inbound,150,Europe,0.362026 -br.com,cmailsys.com,inbound,150,Europe,0 -brandalley.com,brandalley.com,inbound,150,Europe,0 -brands4friends.de,emv5.com,inbound,150,Europe,0 -brandsvillage.net,brandsvillage.net,inbound,150,Europe,0 -bt.com,bt.com,inbound,150,Europe,0.409404 -bweeble.com,adlabsinc.com,inbound,150,Europe,0 -cabestan.com,cab07.net,inbound,150,Europe,0 -cadremploi.fr,cadremploi.fr,inbound,150,Europe,0 -cardsys.at,cardsys.at,inbound,150,Europe,1 -carmamail.com,carmamail.com,inbound,150,Europe,0 -casasbahia.com.br,casasbahia.com.br,inbound,150,Europe,0 -catchoftheday.com.au,inxserver.de,inbound,150,Europe,1 -catererglobal.com,madgexjb.com,inbound,150,Europe,0 -caterermail.com,totaljobsmail.co.uk,inbound,150,Europe,0 -cathkidston.com,cathkidston.co.uk,inbound,150,Europe,0 -cccampaigns.com,emv5.com,inbound,150,Europe,0 -cccampaigns.com,emv8.com,inbound,150,Europe,0 -cccampaigns.net,01net.com,inbound,150,Europe,0 -cccampaigns.net,cccampaigns.net,inbound,150,Europe,0 -cccampaigns.net,emv4.net,inbound,150,Europe,0 -ccmbg.com,benchmark.fr,inbound,150,Europe,0 -cdongroup.com,cdongroup.com,inbound,150,Europe,0.000147 -cheapflights.co.uk,cheapflights.co.uk,inbound,150,Europe,0 -cheapflights.com,cheapflights.com,inbound,150,Europe,0 -chelseafc.com,chelseafc.com,inbound,150,Europe,0.003566 -cinesa.es,cccampaigns.com,inbound,150,Europe,0 -cipherzone.com,infimail.com,inbound,150,Europe,0 -citybrands.hu,webinform.hu,inbound,150,Europe,1 -clickon.com.br,clickon.com.br,inbound,150,Europe,0 -clicplan.com,dmdelivery.com,inbound,150,Europe,0 -cobone.com,emarsys.net,inbound,150,Europe,0 -communicatoremail.com,communicatoremail.com,inbound,150,Europe,0 -compute.internal,amazonaws.com,inbound,150,Europe,0.81478 -confirmedoptin.com,confirmedoptin.com,inbound,150,Europe,0 -continente.pt,1-hostingservice.com,inbound,150,Europe,0 -cratusservices.in,ramcorp.in,inbound,150,Europe,0 -cricinfo.com,cricinfo.com,inbound,150,Europe,0 -critsend.com,critsend.com,inbound,150,Europe,0 -crsend.com,crsend.com,inbound,150,Europe,0.008688 -csas.cz,csas.cz,inbound,150,Europe,0.999971 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,150,Europe,1 -cv-library.co.uk,cv-library.co.uk,inbound,150,Europe,0 -cvbankas.lt,efadm.eu,inbound,150,Europe,0 -cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 -d-reizen.nl,dmdelivery.com,inbound,150,Europe,0 -dafiti.com.br,fagms.de,inbound,150,Europe,0 -datingfactory.com,caerussolutions.net,inbound,150,Europe,0 -dbgi.co.uk,emc1.co.uk,inbound,150,Europe,0 -deal.com.sg,emarsys.net,inbound,150,Europe,0 -debian.org,debian.org,inbound,150,Europe,1 -deezer.com,dms30.com,inbound,150,Europe,0 -directcrm.ru,directcrm.ru,inbound,150,Europe,0 -disney.co.uk,emv9.com,inbound,150,Europe,0 -dominosemail.co.uk,dominosemail.co.uk,inbound,150,Europe,0 -doodle.com,doodle.com,inbound,150,Europe,1 -dotmailer-email.com,dotmailer.com,inbound,150,Europe,0 -dotmailer.co.uk,dotmailer.com,inbound,150,Europe,0 -dpapp.nl,prikbordmailer.nl,inbound,150,Europe,0 -dpapp.nl,sslsecuref.nl,inbound,150,Europe,0 -dreivip.com,dreivip.com,inbound,150,Europe,0.000301 -dress-for-less.de,privalia.com,inbound,150,Europe,0 -drweb.com,drweb.com,inbound,150,Europe,0.969315 -e-boks.dk,e-boks.dk,inbound,150,Europe,1 -e-ebuyer.com,e-ebuyer.com,inbound,150,Europe,0 -e-mark.nl,e-mark.nl,inbound,150,Europe,0 -e-ngine.nl,e-ngine.nl,inbound,150,Europe,0 -ebay-kleinanzeigen.de,mobile.de,inbound,150,Europe,1 -ecommzone.com,ecommzone.com,inbound,150,Europe,0 -edarling.fr,fagms.de,inbound,150,Europe,0 -edima.hu,edima.hu,inbound,150,Europe,0 -efox-shop.com,dmdelivery.com,inbound,150,Europe,0 -ejobs.ro,ejobs.ro,inbound,150,Europe,8.5e-05 -elaine-asp.de,artegic.net,inbound,150,Europe,0.999997 -elcorteingles.es,elcorteingles.es,inbound,150,Europe,0 -elektronskaposta.si,eprvak.si,inbound,150,Europe,0 -elettershop.de,servicemail24.de,inbound,150,Europe,1 -emag.ro,emag.ro,inbound,150,Europe,0.005013 -email-comparethemarket.com,smartfocusdigital.net,inbound,150,Europe,0 -email360api.com,email360api.com,inbound,150,Europe,0 -emarsys.net,emarsys.net,inbound,150,Europe,0 -embluejet.com,emblueuser.com,inbound,150,Europe,0 -emsecure.net,emsecure.net,inbound,150,Europe,0 -emsmtp.com,emsmtp.com,inbound,150,Europe,0 -enewsletter.pl,enewsletter.pl,inbound,150,Europe,0.007349 -enewsletter.pl,sare25.com,inbound,150,Europe,0 -espmp-agfr.net,bp06.net,inbound,150,Europe,0 -esprit-friends.com,esprit-friends.com,inbound,150,Europe,0 -evanscycles.com,msgfocus.com,inbound,150,Europe,0 -experteer.com,experteer.com,inbound,150,Europe,0 -extra.com.br,emv8.com,inbound,150,Europe,0 -eyepin.com,eyepin.com,inbound,150,Europe,0 -fabfurnish.com,fagms.de,inbound,150,Europe,0 -facilisimo.com,facilisimo.com,inbound,150,Europe,1 -fagms.net,fagms.de,inbound,150,Europe,0 -finn.no,schibsted-it.no,inbound,150,Europe,0.002893 -fixeads.com,fixeads.com,inbound,150,Europe,0 -flirchi.com,flirchi.com,inbound,150,Europe,1.0 -flymonarchemail.com,flymonarchemail.com,inbound,150,Europe,0 -follow-up.se,follow-up.se,inbound,150,Europe,1 -fotocasa.es,fotocasa.es,inbound,150,Europe,0 -fotostrana.ru,fotocdn.net,inbound,150,Europe,3.4e-05 -free-lance.ru,free-lance.ru,inbound,150,Europe,0 -free.fr,free.fr,inbound,150,Europe,0.984012 -free.fr,free.fr,outbound,150,Europe,6.9e-05 -freecycle.org,freecycle.org,inbound,150,Europe,0.999865 -freemail.hu,freemail.hu,outbound,150,Europe,0 -freshmail.pl,freshmail.pl,inbound,150,Europe,0 -gardeningclubmail.co.uk,msgfocus.com,inbound,150,Europe,0 -giffgaff.com,giffgaff.com,inbound,150,Europe,0 -globasemail.com,globasemail.com,inbound,150,Europe,0.951088 -gmail.com,02.net,inbound,150,Europe,0.999617 -gmail.com,as13285.net,inbound,150,Europe,0.999943 -gmail.com,bbox.fr,inbound,150,Europe,0.919849 -gmail.com,belgacom.be,inbound,150,Europe,0.999444 -gmail.com,blackberry.com,inbound,150,Europe,0.998923 -gmail.com,bluewin.ch,inbound,150,Europe,0.93294 -gmail.com,btcentralplus.com,inbound,150,Europe,0.999969 -gmail.com,chello.nl,inbound,150,Europe,0.999951 -gmail.com,fastwebnet.it,inbound,150,Europe,0.984706 -gmail.com,jazztel.es,inbound,150,Europe,0.999701 -gmail.com,net24.it,inbound,150,Europe,0.999964 -gmail.com,netcabo.pt,inbound,150,Europe,0.998164 -gmail.com,numericable.fr,inbound,150,Europe,0.999723 -gmail.com,ono.com,inbound,150,Europe,0.995991 -gmail.com,orange.es,inbound,150,Europe,0.998742 -gmail.com,orange.fr,inbound,150,Europe,0 -gmail.com,otenet.gr,inbound,150,Europe,0.957917 -gmail.com,postini.com,inbound,150,Europe,0.924066 -gmail.com,proxad.net,inbound,150,Europe,0.998094 -gmail.com,rima-tde.net,inbound,150,Europe,0.99915 -gmail.com,sfr.net,inbound,150,Europe,0.999877 -gmail.com,skybroadband.com,inbound,150,Europe,0.999854 -gmail.com,t-ipconnect.de,inbound,150,Europe,0.999841 -gmail.com,tdc.net,inbound,150,Europe,0.999591 -gmail.com,telecomitalia.it,inbound,150,Europe,0.997 -gmail.com,telekom.hu,inbound,150,Europe,0.999977 -gmail.com,telenet.be,inbound,150,Europe,1 -gmail.com,telepac.pt,inbound,150,Europe,0.999584 -gmail.com,telia.com,inbound,150,Europe,1 -gmail.com,threembb.co.uk,inbound,150,Europe,1 -gmail.com,tpnet.pl,inbound,150,Europe,0.999761 -gmail.com,virginm.net,inbound,150,Europe,0.99661 -gmail.com,vodafone-ip.de,inbound,150,Europe,1 -gmail.com,vodafone.pt,inbound,150,Europe,0.999563 -gmail.com,vodafonedsl.it,inbound,150,Europe,0.999006 -gmail.com,wanadoo.fr,inbound,150,Europe,0.999752 -gmail.com,ziggo.nl,inbound,150,Europe,1 -gmx.de,gmx.net,inbound,150,Europe,1 -gmx.de,gmx.net,outbound,150,Europe,1 -gmx.net,gmx.net,inbound,150,Europe,1 -goalunited.org,ccmdcampaigns.net,inbound,150,Europe,0 -gog.com,gog.com,inbound,150,Europe,0 -gogroopie.com,gogroopie.com,inbound,150,Europe,0.000283 -goldenline.pl,goldenline.pl,inbound,150,Europe,1 -goodgame.com,emsmtp.com,inbound,150,Europe,0 -goodlife.pt,emv8.com,inbound,150,Europe,0 -google.com,postini.com,inbound,150,Europe,0.810567 -googlemail.com,t-ipconnect.de,inbound,150,Europe,0.999957 -gumtree.com,marktplaats.nl,inbound,150,Europe,0 -gumtree.com.au,kijiji.com,inbound,150,Europe,0 -gymglish.com,gymglish.com,inbound,150,Europe,0.000788 -haskell.org,haskell.org,inbound,150,Europe,0.000703 -hazteoir.org,hazteoir.org,inbound,150,Europe,0.31478 -hh.ru,hh.ru,inbound,150,Europe,0.996236 -hotel.de,emp-mail.de,inbound,150,Europe,0 -hotornot.com,monopost.com,inbound,150,Europe,0.983953 -hotukdeals.com,hotukdeals.com,inbound,150,Europe,1 -hpnotifier.nl,hpnotifier.nl,inbound,150,Europe,0 -icelandmail.co.uk,emsg-live.co.uk,inbound,150,Europe,0 -idealista.com,idealista.com,inbound,150,Europe,0.001627 -ideascost.com,ramcorp.in,inbound,150,Europe,0 -inboxair.com,inboxair.com,inbound,150,Europe,0 -infoempleo.com,infoempleo.com,inbound,150,Europe,0 -infojobs.it,infojobs.it,inbound,150,Europe,0 -infojobs.net,infojobs.net,inbound,150,Europe,0 -infopraca.pl,careesma.com,inbound,150,Europe,0 -ingdirect.es,ingdirect.es,inbound,150,Europe,1 -inter-chat.com,inter-chat.com,inbound,150,Europe,0 -interdatesa.com,fagms.net,inbound,150,Europe,0 -internations.org,internations.org,inbound,150,Europe,1 -inx1and1.de,1and1.com,inbound,150,Europe,1 -inxserver.com,inxserver.de,inbound,150,Europe,0.952863 -inxserver.de,inxserver.de,inbound,150,Europe,0.980838 -itms.in.ua,itms.in.ua,inbound,150,Europe,0 -jiscmail.ac.uk,lsoft.se,inbound,150,Europe,0 -jobisjob.com,jobisjob.com,inbound,150,Europe,0 -jobrapidoalert.com,jobrapidoalert.com,inbound,150,Europe,0 -jobs2web.com,ondemand.com,inbound,150,Europe,1 -jobserve.com,jobserve.com,inbound,150,Europe,0 -joobmailer.com,joobmailer.com,inbound,150,Europe,0 -jumia.com.ng,fagms.de,inbound,150,Europe,0 -justclick.ru,justclick.ru,inbound,150,Europe,0 -kalunga.com.br,kalunga.com.br,inbound,150,Europe,0 -kiabi.com,dms-02.net,inbound,150,Europe,0 -kijiji.ca,kijiji.com,inbound,150,Europe,0 -kismia.com,kismia.com,inbound,150,Europe,1 -kiwari.com,kiwari.com,inbound,150,Europe,9e-06 -kundenserver.de,kundenserver.de,inbound,150,Europe,1 -laposte.net,laposte.net,inbound,150,Europe,0.271722 -laposte.net,laposte.net,outbound,150,Europe,0 -laredoute.fr,laredoute.fr,inbound,150,Europe,0 -laterooms.com,laterooms.com,inbound,150,Europe,0.014746 -leboncoin.fr,leboncoin.fr,inbound,150,Europe,0 -leparisien.fr,leparisien.fr,inbound,150,Europe,0 -lexpress.fr,bp06.net,inbound,150,Europe,0 -libero.it,libero.it,inbound,150,Europe,0.00024 -libero.it,libero.it,outbound,150,Europe,0 -lifecooler.com,1-hostingservice.com,inbound,150,Europe,0 -listadventure.com,adlabsinc.com,inbound,150,Europe,0 -listjoe.com,adlabsinc.com,inbound,150,Europe,0 -litres.ru,litres.ru,inbound,150,Europe,1 -loccitane.com,neolane.net,inbound,150,Europe,0 -logentries.com,logentries.com,inbound,150,Europe,0 -loveplanet.ru,pochta.ru,inbound,150,Europe,0.640035 -lowcostholidays.co.uk,communicatoremail.com,inbound,150,Europe,0 -lua.org,pepperfish.net,inbound,150,Europe,1 -ludokados.com,ludokado.com,inbound,150,Europe,0 -mail-cdiscount.com,mail-cdiscount.com,inbound,150,Europe,0 -mail-mbank.pl,mail-mbank.pl,inbound,150,Europe,0 -mail.ru,mail.ru,inbound,150,Europe,0.991269 -mail.ru,mail.ru,outbound,150,Europe,0.006918 -mailer-service.de,mailer-service.de,inbound,150,Europe,4.9e-05 -mailersend.com,mailersend.com,inbound,150,Europe,0 -mailing-list.it,mailing-list.it,inbound,150,Europe,0 -mailjet.com,mailjet.com,inbound,150,Europe,0 -mailplus.nl,brightbase.net,inbound,150,Europe,1 -mailpv.net,pvmailer.net,inbound,150,Europe,1 -maisonsdumonde.com,bp06.net,inbound,150,Europe,0 -makro.nl,srv2.de,inbound,150,Europe,0.888692 -mapfre.com,emv5.com,inbound,150,Europe,0 -marktplaats.nl,marktplaats.nl,inbound,150,Europe,0 -matchwereld.nl,matchwereld.nl,inbound,150,Europe,0 -maxpark.com,gidepark.ru,inbound,150,Europe,1 -mdirector.com,mdrctr.com,inbound,150,Europe,0 -mecumauction.com,mecumauction.com,inbound,150,Europe,0 -meetic.com,meetic.com,inbound,150,Europe,0 -mequedouno.com,mequedouno.com,inbound,150,Europe,0 -metrodeal.com,fagms.de,inbound,150,Europe,0 -mightydeals.co.uk,mightydeals.co.uk,inbound,150,Europe,0 -mirtesen.ru,mtml.ru,inbound,150,Europe,0 -mitula.net,mitula.org,inbound,150,Europe,0 -mitula.org,mitula.org,inbound,150,Europe,0 -mixcloudmail.com,mixcloudmail.com,inbound,150,Europe,0.995482 -mlgns.com,mlgns.com,inbound,150,Europe,0 -mlgnserv.com,mlgnserv.com,inbound,150,Europe,0 -mmks.it,mail-maker.it,inbound,150,Europe,0 -mmsecure.nl,donenad.nl,inbound,150,Europe,0 -modnakasta.ua,emv5.com,inbound,150,Europe,0 -mooply.co,mailendo.com,inbound,150,Europe,0 -moviestarplanet.com,moviestarplanet.com,inbound,150,Europe,0 -mrc.org,msgfocus.com,inbound,150,Europe,0 -msdp1.com,msdp1.com,inbound,150,Europe,0 -msgfocus.com,msgfocus.com,inbound,150,Europe,0.011658 -mymms.com,fagms.de,inbound,150,Europe,0 -namorico.me,namorico.me,inbound,150,Europe,1 -nasza-klasa.pl,nasza-klasa.pl,inbound,150,Europe,0 -nationalexpress.com,nationalexpress.com,inbound,150,Europe,0 -neolane.net,neolane.net,inbound,150,Europe,0 -netopia.pt,netopia.pt,inbound,150,Europe,0.028429 -nhs.jobs,nhscareersjobs.co.uk,inbound,150,Europe,0 -nieuwsblad.be,vummail.be,inbound,150,Europe,0 -nmp1.com,nmp1.net,inbound,150,Europe,0 -nos.pt,netcabo.pt,inbound,150,Europe,6.3e-05 -noticiasaominuto.com,ccmdcampaigns.net,inbound,150,Europe,0 -noticiasaominuto.com,noticiasaominuto.com,inbound,150,Europe,0 -nrholding.net,nrholding.net,inbound,150,Europe,0 -odisseias.com,emv4.net,inbound,150,Europe,0 -odnoklassniki.ru,odnoklassniki.ru,inbound,150,Europe,0 -offerum.com,cccampaigns.com,inbound,150,Europe,0 -offerum.com,ccemails.com,inbound,150,Europe,0 -olx.pt,fixeads.com,inbound,150,Europe,0 -orange.fr,orange.fr,inbound,150,Europe,0 -orange.fr,orange.fr,outbound,150,Europe,0 -oroscopofree.com,adsender.us,inbound,150,Europe,0 -oroscopofree.com,oroscopofree.com,inbound,150,Europe,0 -orsay.com,emp-mail.de,inbound,150,Europe,0 -ovh.net,ovh.net,inbound,150,Europe,0.165188 -oxfam.org.uk,msgfocus.com,inbound,150,Europe,0 -payback.de,artegic.net,inbound,150,Europe,0.999986 -pccomponentes.com,pccomponentes.com,inbound,150,Europe,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,150,Europe,0 -peperoni.de,peperoni.de,inbound,150,Europe,1 -peytz.dk,peytz.dk,inbound,150,Europe,4e-06 -photobox.com,photobox.com,inbound,150,Europe,0 -photoprintit.com,photoprintit.com,inbound,150,Europe,0 -pixum.com,pixum.com,inbound,150,Europe,0 -placedestendances.com,placedestendances.com,inbound,150,Europe,0 -plaisio.gr,fagms.de,inbound,150,Europe,0 -planeo.com,planeo.com,inbound,150,Europe,0 -planeo.pt,planeo.pt,inbound,150,Europe,0 -playtika.com,emv8.com,inbound,150,Europe,0 -poinx.com,poinx.com,inbound,150,Europe,0 -pokerstars.com,pokerstars.eu,inbound,150,Europe,0 -pokerstars.eu,pokerstars.eu,inbound,150,Europe,0 -pontofrio.com.br,emv8.com,inbound,150,Europe,0 -postgresql.org,postgresql.org,inbound,150,Europe,1 -praca.pl,praca.pl,inbound,150,Europe,1 -pracuj.pl,pracuj.pl,inbound,150,Europe,0 -printvenue.com,fagms.de,inbound,150,Europe,0 -privalia.com,privalia.com,inbound,150,Europe,3.94913202027326e-07 -promod-news.fr,promod-news.com,inbound,150,Europe,0 -protopmail.com,protopmail.com,inbound,150,Europe,0 -pur3.net,pur3.net,inbound,150,Europe,0.001123 -python.org,python.org,inbound,150,Europe,1 -quality.net.ua,quality.net.ua,inbound,150,Europe,0 -r-project.org,ethz.ch,inbound,150,Europe,0.99999 -r51.it,musvc.com,inbound,150,Europe,0.092498 -r52.it,musvc.com,inbound,150,Europe,0.000221 -r57.it,musvc.com,inbound,150,Europe,0.139425 -r67.it,musvc.com,inbound,150,Europe,0.199845 -r70.it,musvc.com,inbound,150,Europe,0.311983 -rakuten.co.jp,shareee.jp,inbound,150,Europe,0 -rambler.ru,rambler.ru,inbound,150,Europe,0.08173 -ratedpeople.com,ratedpeople.com,inbound,150,Europe,0.000979 -rax.ru,rax.ru,inbound,150,Europe,0.000258 -regie11.net,odiso.net,inbound,150,Europe,0 -relax7.hu,gruppi.hu,inbound,150,Europe,0 -richersoundsvip.com,ibwmail.com,inbound,150,Europe,0 -rightmove.com,rightmove.com,inbound,150,Europe,0 -roulartamail.be,roulartamail.be,inbound,150,Europe,0 -rueducommerce.com,groupe-rueducommerce.fr,inbound,150,Europe,0 -runtastic.com,runtastic.com,inbound,150,Europe,0 -ryanairmail.com,ryanairmail.com,inbound,150,Europe,0 -rzone.de,rzone.de,inbound,150,Europe,1 -saimails.in,infimail.com,inbound,150,Europe,0 -sainsburys.co.uk,emv5.com,inbound,150,Europe,0 -salesmanago.pl,salesmanago.pl,inbound,150,Europe,0 -samsung.ru,samsung.ru,inbound,150,Europe,0 -sapnetworkmail.com,sap-ag.de,inbound,150,Europe,1 -sapo.pt,sapo.pt,inbound,150,Europe,0.242839 -sapo.pt,sapo.pt,outbound,150,Europe,0 -savingdeals.in,infimail.com,inbound,150,Europe,0 -scmp.com,emarsys.net,inbound,150,Europe,0 -scoop.it,scoop.it,inbound,150,Europe,0 -scoopon.com.au,inxserver.de,inbound,150,Europe,1 -screwfix.info,fwdto.net,inbound,150,Europe,0 -secureserver.net,secureserver.net,inbound,150,Europe,0 -selection-priceminister.com,selection-priceminister.com,inbound,150,Europe,0 -sender.lt,sritis.lt,inbound,150,Europe,0.000339 -sendsmaily.info,sendsmaily.info,inbound,150,Europe,0 -seniorplanet.fr,seniorplanet.fr,inbound,150,Europe,0 -seznam.cz,seznam.cz,inbound,150,Europe,0.001781 -seznam.cz,seznam.cz,outbound,150,Europe,0.001741 -sfr.fr,sfr.fr,inbound,150,Europe,0.236539 -sfr.fr,sfr.fr,outbound,150,Europe,0.004753 -shopto.net,shopto.net,inbound,150,Europe,1 -skype.com,skype.com,inbound,150,Europe,0 -solveerrors.com,infimail.com,inbound,150,Europe,0 -spareroom.co.uk,spareroom.co.uk,inbound,150,Europe,1 -sqlservercentral.com,sqlservercentral.com,inbound,150,Europe,0 -staples-pt.com,1-hostingservice.com,inbound,150,Europe,0 -stepstone.de,stepstone.com,inbound,150,Europe,0.001689 -studentbeans.com,emv8.com,inbound,150,Europe,0 -subito.it,subito.it,inbound,150,Europe,0 -subscribe.ru,subscribe.ru,inbound,150,Europe,0 -superdrug.com,superdrug.com,inbound,150,Europe,0 -superjob.ru,superjob.ru,inbound,150,Europe,0 -support-love.com,support-love.com,inbound,150,Europe,0 -sut1.co.uk,sut1.co.uk,inbound,150,Europe,0.001789 -sut5.co.uk,sut5.co.uk,inbound,150,Europe,0 -swanson-vitamins.com,emv5.com,inbound,150,Europe,0 -t-online.de,t-online.de,inbound,150,Europe,1 -t-online.de,t-online.de,outbound,150,Europe,0.999939 -tatrabanka.sk,tatrabanka.sk,inbound,150,Europe,0 -tchibo.de,srv2.de,inbound,150,Europe,0.980447 -teamo.ru,teamo.ru,inbound,150,Europe,0 -teamviewer.com,teamviewer.com,inbound,150,Europe,0 -teapartyinfo.org,teapartyinfo.org,inbound,150,Europe,0 -telenet.be,telenet-ops.be,inbound,150,Europe,1.5e-05 -teleportmyjob.com,clara.net,inbound,150,Europe,0 -theleadmagnet.com,your-server.de,inbound,150,Europe,1 -timeweb.ru,timeweb.ru,inbound,150,Europe,1 -tiscali.it,tiscali.it,outbound,150,Europe,0 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 -transversal.net,transversal.net,inbound,150,Europe,1 -travisperkins.co.uk,travisperkins.co.uk,inbound,150,Europe,0 -trovit.com,trovit.com,inbound,150,Europe,0 -tucasa.com,grupodtm.com,inbound,150,Europe,0 -twoomail.com,twoomail.com,inbound,150,Europe,0 -ubuntu.com,canonical.com,inbound,150,Europe,0 -ukr.net,fwdcdn.com,inbound,150,Europe,1 -ukr.net,ukr.net,outbound,150,Europe,0.999992 -ulteem.com,ulteem.com,inbound,150,Europe,0 -usndr.com,usndr.com,inbound,150,Europe,1 -venere.com,kiwari.com,inbound,150,Europe,0 -venteprivee.com,venteprivee.com,inbound,150,Europe,0 -virgilio.it,virgilio.net,inbound,150,Europe,0 -visualsoft.co.uk,visualsoft.co.uk,inbound,150,Europe,0 -vouchercloud.com,vouchercloud.com,inbound,150,Europe,0 -voyageprive.com,cccampaigns.net,inbound,150,Europe,0 -voyageprive.es,ccmdcampaigns.net,inbound,150,Europe,0 -voyageprive.it,ccmdcampaigns.net,inbound,150,Europe,0 -voyages-sncf.com,neolane.net,inbound,150,Europe,0 -vueling.com,vueling.com,inbound,150,Europe,0 -wanadoo.fr,orange.fr,inbound,150,Europe,0 -wanadoo.fr,orange.fr,outbound,150,Europe,0 -waves-audio.com,emv8.com,inbound,150,Europe,0 -web.de,web.de,inbound,150,Europe,0.999986 -web.de,web.de,outbound,150,Europe,1 -whereareyounow.com,wayn.net,inbound,150,Europe,0 -wiggle.com,wiggle.com,inbound,150,Europe,0 -william-reed.com,neolane.net,inbound,150,Europe,0 -williamhill.com,williamhill.com,inbound,150,Europe,0 -wldemail.com,emarsys.net,inbound,150,Europe,0 -wmtransfer.com,wmtransfer.com,inbound,150,Europe,0 -wnd.com,emv4.net,inbound,150,Europe,0 -wnd.com,worldnetdaily.com,inbound,150,Europe,0 -work.ua,work.ua,inbound,150,Europe,1 -workcircle.com,workcircle.net,inbound,150,Europe,0 -wp.pl,wp.pl,inbound,150,Europe,0.998027 -wp.pl,wp.pl,outbound,150,Europe,1 -xmailix.com,xmailix.com,inbound,150,Europe,0 -yandex.ru,yandex.net,inbound,150,Europe,0.999658 -yandex.ru,yandex.ru,outbound,150,Europe,1 -ymlpserver.net,ymlpserver.net,inbound,150,Europe,0 -ymlpsrv.net,ymlpsrv.net,inbound,150,Europe,0 -zalando.be,fagms.de,inbound,150,Europe,0 -zalando.dk,fagms.de,inbound,150,Europe,0 -zalando.fi,fagms.de,inbound,150,Europe,0 -zalando.it,fagms.de,inbound,150,Europe,0 -zalando.nl,fagms.de,inbound,150,Europe,0 -zalando.pl,fagms.de,inbound,150,Europe,0 -zumzi.com,neogen.ro,inbound,150,Europe,0 -zumzi.com,zumzi.com,inbound,150,Europe,0 -0bz.biz,hmts.jp,inbound,ZZ,Unknown Region,0 -104.com.tw,104.com.tw,inbound,ZZ,Unknown Region,0.091133 -1105info.com,1105info.com,inbound,ZZ,Unknown Region,0 -1111.com.tw,1111.com.tw,inbound,ZZ,Unknown Region,0 -160by2inbox.com,160by2inbox.com,inbound,ZZ,Unknown Region,0 -160by2invite.com,160by2invite.com,inbound,ZZ,Unknown Region,0 -160by2mail.com,160by2mail.com,inbound,ZZ,Unknown Region,0 -163.com,163.com,inbound,ZZ,Unknown Region,0.996685 -1800flowersinc.com,1800flowersinc.com,inbound,ZZ,Unknown Region,0 -1sale.com,1sale.com,inbound,ZZ,Unknown Region,0 -1v1y.com,euromsg.net,inbound,ZZ,Unknown Region,0 -4shared.com,4shared.com,inbound,ZZ,Unknown Region,1 -6pm.com,6pm.com,inbound,ZZ,Unknown Region,1 -6pm.com,zappos.com,inbound,ZZ,Unknown Region,0.782581 -a8.net,a8.net,inbound,ZZ,Unknown Region,0 -aaa.com,nextjump.com,inbound,ZZ,Unknown Region,0 -aaas-science.org,aaas-science.org,inbound,ZZ,Unknown Region,0 -aarp.org,aarp.org,inbound,ZZ,Unknown Region,0 -about.com,about.com,inbound,ZZ,Unknown Region,8.2e-05 -academy-enews.com,academy-enews.com,inbound,ZZ,Unknown Region,0 -accenture.com,outlook.com,inbound,ZZ,Unknown Region,1 -accountonline.com,accountonline.com,inbound,ZZ,Unknown Region,0.348991 -acehelpfulemails.com,teradatadmc.com,inbound,ZZ,Unknown Region,0 -actorsaccess.com,nonfatmedia.com,inbound,ZZ,Unknown Region,0 -adminforfree.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -adminforfree.net,adminforfree.com,inbound,ZZ,Unknown Region,1 -administrativejobinsider.com,administrativejobinsider.com,inbound,ZZ,Unknown Region,0 -adobe.com,obsmtp.com,inbound,ZZ,Unknown Region,1 -adobesystems.com,adobesystems.com,inbound,ZZ,Unknown Region,0 -adoreme.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -adp.com,adp.com,inbound,ZZ,Unknown Region,1 -adsender.us,adsender.us,inbound,ZZ,Unknown Region,0 -adsolutionline.com,adsolutionline.com,inbound,ZZ,Unknown Region,0 -adultfriendfinder.com,friendfinder.com,inbound,ZZ,Unknown Region,0 -advanceauto.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,ZZ,Unknown Region,0 -af.mil,af.mil,inbound,ZZ,Unknown Region,0.997561 -agoda-emails.com,agoda-emails.com,inbound,ZZ,Unknown Region,0 -agrupemonos.cl,agrupemonos.cl,inbound,ZZ,Unknown Region,1 -albertsonsemail.com,email4-mywebgrocer.com,inbound,ZZ,Unknown Region,0 -alibaba.com,alibaba.com,inbound,ZZ,Unknown Region,0 -alice.it,alice.it,inbound,ZZ,Unknown Region,0 -alice.it,aliceposta.it,outbound,ZZ,Unknown Region,0 -aliexpress.com,alibaba.com,inbound,ZZ,Unknown Region,0 -allegro.pl,allegro.pl,inbound,ZZ,Unknown Region,0 -allegrogroup.ua,allegrogroup.ua,inbound,ZZ,Unknown Region,0 -allsaints.com,allsaints.com,inbound,ZZ,Unknown Region,0 -ama-assn.org,elabs10.com,inbound,ZZ,Unknown Region,0 -amadeus.com,amadeus.net,inbound,ZZ,Unknown Region,0 -amazon.{...},amazon.{...},inbound,ZZ,Unknown Region,0.021685 -amazon.{...},amazonses.com,inbound,ZZ,Unknown Region,0.999971 -amazonses.com,amazonses.com,inbound,ZZ,Unknown Region,0.997919 -amazonses.com,postini.com,inbound,ZZ,Unknown Region,0.924067 -amctheatres.com,amctheatres.com,inbound,ZZ,Unknown Region,0 -americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,ZZ,Unknown Region,0 -ancestry.com,ancestry.com,inbound,ZZ,Unknown Region,0 -angieslist.com,angieslist.com,inbound,ZZ,Unknown Region,0 -anntaylor.com,anntaylor.com,inbound,ZZ,Unknown Region,0 -anpdm.com,anpdm.com,inbound,ZZ,Unknown Region,4e-06 -aol.com,aol.com,outbound,ZZ,Unknown Region,1 -apple.com,apple.com,inbound,ZZ,Unknown Region,0.915839 -argos.co.uk,argos.co.uk,inbound,ZZ,Unknown Region,1 -argos.co.uk,exacttarget.com,inbound,ZZ,Unknown Region,1 -artists-hub.com,artists-hub.com,inbound,ZZ,Unknown Region,0 -asda.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -ask.fm,ask.fm,inbound,ZZ,Unknown Region,1e-05 -askmen.com,askmen.com,inbound,ZZ,Unknown Region,0 -astrocenter.com,center.com,inbound,ZZ,Unknown Region,0 -athleta.com,athleta.com,inbound,ZZ,Unknown Region,0 -atlassian.net,uc-inf.net,inbound,ZZ,Unknown Region,1 -att.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999956 -auctionzip-email.com,email-auctionholdings.com,inbound,ZZ,Unknown Region,0 -auinmeio.com.br,fnac.com.br,inbound,ZZ,Unknown Region,0 -authorize.net,authorize.net,inbound,ZZ,Unknown Region,0 -authorize.net,visa.com,inbound,ZZ,Unknown Region,0.993555 -autoreply.com,autoreply.com,inbound,ZZ,Unknown Region,0 -avomail.com,avomail.com,inbound,ZZ,Unknown Region,0 -avon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 -avon.com,postdirect.com,inbound,ZZ,Unknown Region,0 -aweber.com,aweber.com,inbound,ZZ,Unknown Region,3e-06 -ayi.com,ayi.com,inbound,ZZ,Unknown Region,0 -backcountry.com,backcountry.com,inbound,ZZ,Unknown Region,0 -backlog.jp,backlog.jp,inbound,ZZ,Unknown Region,0 -bagitgetitmailer.in,emce2.in,inbound,ZZ,Unknown Region,0 -banamex.com,citi.com,inbound,ZZ,Unknown Region,0.999958 -bananarepublic.com,bananarepublic.com,inbound,ZZ,Unknown Region,3.48869418874254e-07 -bancochile.cl,bancochile.cl,inbound,ZZ,Unknown Region,0.999504 -bancofalabella.com,bancofalabella.com,inbound,ZZ,Unknown Region,0 -banesco.com,banesco.com,inbound,ZZ,Unknown Region,0 -bankofamerica.com,bankofamerica.com,inbound,ZZ,Unknown Region,0.971142 -banorte.com,gfnorte.com.mx,inbound,ZZ,Unknown Region,0.994999 -barclaycardus.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -basecamp.com,basecamp.com,inbound,ZZ,Unknown Region,1 -basecamphq.com,basecamphq.com,inbound,ZZ,Unknown Region,1 -baskinrobbins.com,baskinrobbins.com,inbound,ZZ,Unknown Region,0 -bazarchic-invitations.com,bazarchic-emstech.com,inbound,ZZ,Unknown Region,0 -bcbg.com,bcbg.com,inbound,ZZ,Unknown Region,0 -beatport-email.com,beatport-email.com,inbound,ZZ,Unknown Region,0 -beautylish.com,beautylish.com,inbound,ZZ,Unknown Region,1 -bebe.com,ed10.com,inbound,ZZ,Unknown Region,0 -befrugal.com,befrugal.com,inbound,ZZ,Unknown Region,0 -belkemail.com,belkemail.com,inbound,ZZ,Unknown Region,0 -bellsouth.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999951 -benihana-news.com,benihana-news.com,inbound,ZZ,Unknown Region,0 -bestbuy.com,bestbuy.com,inbound,ZZ,Unknown Region,0.003289 -beta.lt,mailersend3.com,inbound,ZZ,Unknown Region,0 -beyondtherack.com,beyondtherack.com,inbound,ZZ,Unknown Region,0 -bigfishgames.com,bigfishgames.com,inbound,ZZ,Unknown Region,0 -biglots.com,biglots.com,inbound,ZZ,Unknown Region,0 -bjsrestaurants.com,bjsrestaurants.com,inbound,ZZ,Unknown Region,0 -blackberry.com,blackberry.com,inbound,ZZ,Unknown Region,0 -blackboard.com,notification.com,inbound,ZZ,Unknown Region,0 -blackpeoplemeet.com,blackpeoplemeet.com,inbound,ZZ,Unknown Region,0 -bloglovin.com,bloglovin.com,inbound,ZZ,Unknown Region,0 -blogtrottr.com,blogtrottr.com,inbound,ZZ,Unknown Region,0 -bloomberg.com,bloomberg.com,inbound,ZZ,Unknown Region,0.001463 -bloomberg.net,bloomberg.net,inbound,ZZ,Unknown Region,1 -bluediamondhost3.com,web-hosting.com,inbound,ZZ,Unknown Region,1 -bluehornet.com,bluehornet.com,inbound,ZZ,Unknown Region,0 -bluenile.com,bluenile.com,inbound,ZZ,Unknown Region,0 -bluestatedigital.com,bluestatedigital.com,inbound,ZZ,Unknown Region,0 -bm05.net,bm05.net,inbound,ZZ,Unknown Region,0 -bmnt.jp,bmnt.jp,inbound,ZZ,Unknown Region,0 -bmsend.com,bmsend.com,inbound,ZZ,Unknown Region,0 -bn.com,bn.com,inbound,ZZ,Unknown Region,0 -bncollegemail.com,bncollegemail.com,inbound,ZZ,Unknown Region,0 -booking.com,booking.com,inbound,ZZ,Unknown Region,1 -bookmyshow.com,eccluster.com,inbound,ZZ,Unknown Region,0 -boots.com,boots.com,inbound,ZZ,Unknown Region,0 -box.com,box.com,inbound,ZZ,Unknown Region,0.955607 -brierleycrm.com,brierleycrm.com,inbound,ZZ,Unknown Region,0 -brooksbrothers.com,brooksbrothers.com,inbound,ZZ,Unknown Region,0 -btinternet.com,cpcloud.co.uk,inbound,ZZ,Unknown Region,0 -btinternet.com,cpcloud.co.uk,outbound,ZZ,Unknown Region,0 -btinternet.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -budgettravel.com,email-budgettravel.com,inbound,ZZ,Unknown Region,0 -buyinvite.com.au,buyinvite.com.au,inbound,ZZ,Unknown Region,0 -bv.com.br,bv.com.br,inbound,ZZ,Unknown Region,0 -byway.it,byway.it,inbound,ZZ,Unknown Region,0 -cabelas.com,cabelas.com,inbound,ZZ,Unknown Region,0 -californiajobdepartment.com,californiajobdepartment.com,inbound,ZZ,Unknown Region,0 -callcommand.com,callcommand.com,inbound,ZZ,Unknown Region,0 -calottery.com,calottery.com,inbound,ZZ,Unknown Region,1 -canadiantire.ca,canadiantire.ca,inbound,ZZ,Unknown Region,0 -canalplus.es,canalplus.es,inbound,ZZ,Unknown Region,0 -capitalone.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -capitaloneemail.com,capitaloneemail.com,inbound,ZZ,Unknown Region,0 -career-hub.net,career-hub.net,inbound,ZZ,Unknown Region,1 -careerbuilder-email.com,careerbuilder-email.com,inbound,ZZ,Unknown Region,0 -careerbuilder.com,careerbuilder.com,inbound,ZZ,Unknown Region,0 -careerflash.net,careerflash.net,inbound,ZZ,Unknown Region,1 -carrefour.fr,carrefour.fr,inbound,ZZ,Unknown Region,0 -carters.com,carters.com,inbound,ZZ,Unknown Region,2e-06 -caseyresearch.com,caseyresearch.com,inbound,ZZ,Unknown Region,1 -catchyfreebies.net,mmsend53.com,inbound,ZZ,Unknown Region,0 -cccampaigns.net,emv9.net,inbound,ZZ,Unknown Region,0 -cecentertainment.com,cecentertainment.com,inbound,ZZ,Unknown Region,0 -celebritycruises.com,celebritycruises.com,inbound,ZZ,Unknown Region,0 -centaur.co.uk,centaur.co.uk,inbound,ZZ,Unknown Region,0 -centauro.com.br,centauro.com.br,inbound,ZZ,Unknown Region,0 -centerparcs.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -cfmailer.com,elabs11.com,inbound,ZZ,Unknown Region,0 -change.org,change.org,inbound,ZZ,Unknown Region,1 -channel4.com,channel4.com,inbound,ZZ,Unknown Region,0 -chase.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -cheaperthandirt.com,cheaperthandirt.com,inbound,ZZ,Unknown Region,0 -chefscatalog.com,chefscatalog.com,inbound,ZZ,Unknown Region,0 -chemistdirect.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -chemistry.com,chemistry.com,inbound,ZZ,Unknown Region,0 -chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,ZZ,Unknown Region,0 -chopra.com,chopra.com,inbound,ZZ,Unknown Region,1 -christianmingle.com,postdirect.com,inbound,ZZ,Unknown Region,0 -cir.ca,cir.ca,inbound,ZZ,Unknown Region,1 -citi.com,citi.com,inbound,ZZ,Unknown Region,0.999941 -citibank.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -citibank.com,citi.com,inbound,ZZ,Unknown Region,0.999997 -citicorp.com,citi.com,inbound,ZZ,Unknown Region,0.999999 -citruslane.com,citruslane.com,inbound,ZZ,Unknown Region,5e-06 -clarisonic.com,clarisonic.com,inbound,ZZ,Unknown Region,0 -classmates.com,classmates.com,inbound,ZZ,Unknown Region,0 -clickon.com.ar,clickon.com.ar,inbound,ZZ,Unknown Region,0 -cmail1.com,createsend.com,inbound,ZZ,Unknown Region,0 -cmail2.com,createsend.com,inbound,ZZ,Unknown Region,0 -cmrfalabella.com,cmrfalabella.com,inbound,ZZ,Unknown Region,0 -codebreak.info,codebreak.info,inbound,ZZ,Unknown Region,1 -comcast.net,comcast.net,outbound,ZZ,Unknown Region,1 -confirmsignup.com,mmsend53.com,inbound,ZZ,Unknown Region,0 -constantcontact.com,postini.com,inbound,ZZ,Unknown Region,0.139551 -constantcontact.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999541 -contact-darty.com,mm-send.com,inbound,ZZ,Unknown Region,0 -contactlab.it,contactlab.it,inbound,ZZ,Unknown Region,0 -converse.com,converse.com,inbound,ZZ,Unknown Region,1 -cookingchanneltv.com,cookingchanneltv.com,inbound,ZZ,Unknown Region,0 -copernica.nl,picsrv.net,inbound,ZZ,Unknown Region,0.011507 -copernica.nl,vicinity.nl,inbound,ZZ,Unknown Region,0.011753 -coppel.com,coppel.com,inbound,ZZ,Unknown Region,0 -corporateperks.com,nextjump.com,inbound,ZZ,Unknown Region,0 -countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,ZZ,Unknown Region,0 -coupondunia.in,coupondunia.in,inbound,ZZ,Unknown Region,1 -crabtree-evelyn.com,crabtree-evelyn.com,inbound,ZZ,Unknown Region,0 -crainnewsalerts.com,crainnewsalerts.com,inbound,ZZ,Unknown Region,0 -crashlytics.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -cricut.com,elabs12.com,inbound,ZZ,Unknown Region,0 -criticalimpactinc.com,criticalimpactinc.com,inbound,ZZ,Unknown Region,0 -critsend.com,critsend.com,inbound,ZZ,Unknown Region,0 -crocs-email.com,crocs-email.com,inbound,ZZ,Unknown Region,0 -crunchyroll.com,crunchyroll.com,inbound,ZZ,Unknown Region,0 -cudo.com.au,exacttarget.com,inbound,ZZ,Unknown Region,0 -cuenote.jp,cuenote.jp,inbound,ZZ,Unknown Region,0 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,ZZ,Unknown Region,1 -cuppon.pl,cuppon.pl,inbound,ZZ,Unknown Region,0 -curriculum.com.br,curriculum.com.br,inbound,ZZ,Unknown Region,0 -currys.co.uk,currys.co.uk,inbound,ZZ,Unknown Region,0 -custom-emailing.com,elabs12.com,inbound,ZZ,Unknown Region,0 -cvent-planner.com,cvent-planner.com,inbound,ZZ,Unknown Region,0 -cyberdiet.com.br,allinmedia.com.br,inbound,ZZ,Unknown Region,0 -dabmail.com,iaires.com,inbound,ZZ,Unknown Region,0 -dailyom.com,dailyom.com,inbound,ZZ,Unknown Region,1 -dairyqueen.com,dairyqueen.com,inbound,ZZ,Unknown Region,0 -datadrivenemail.com,datadrivenemail.com,inbound,ZZ,Unknown Region,0 -daveramsey.com,daveramsey.com,inbound,ZZ,Unknown Region,0 -ddc-emails.com,ddc-emails.com,inbound,ZZ,Unknown Region,0 -dealchicken.com,dealchicken.com,inbound,ZZ,Unknown Region,0 -dealchicken.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -dealfind.com,dealfind.com,inbound,ZZ,Unknown Region,0 -dealnews.com,dealnews.com,inbound,ZZ,Unknown Region,0 -dealsdirect.com.au,dealsdirect.com.au,inbound,ZZ,Unknown Region,0 -dealspl.us,dealspl.us,inbound,ZZ,Unknown Region,0 -delta.com,delta.com,inbound,ZZ,Unknown Region,0.040177 -dermstore.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -dhl.com,dhl.com,inbound,ZZ,Unknown Region,0.994107 -dietaesaude.com.br,dietaesaude.com.br,inbound,ZZ,Unknown Region,0 -dinda.com.br,dinda.com.br,inbound,ZZ,Unknown Region,0 -directvla.com,directvla.com,inbound,ZZ,Unknown Region,0 -discover.com,discover.com,inbound,ZZ,Unknown Region,0 -disneydestinations.com,disneyparks.com,inbound,ZZ,Unknown Region,0 -disneydestinations.com,disneyworld.com,inbound,ZZ,Unknown Region,0 -diynetwork.com,diynetwork.com,inbound,ZZ,Unknown Region,0 -doctoroz.com,email-sharecare2.com,inbound,ZZ,Unknown Region,0 -dollartree.com,email-dollartree.com,inbound,ZZ,Unknown Region,0 -dominos.com,dominos.com,inbound,ZZ,Unknown Region,0.011861 -dominos.com.au,dominos.com.au,inbound,ZZ,Unknown Region,0 -donuts.ne.jp,dnuts.jp,inbound,ZZ,Unknown Region,0 -dotz.com.br,dotz.com.br,inbound,ZZ,Unknown Region,0 -doubletakeoffers.com,doubletakeoffers.com,inbound,ZZ,Unknown Region,0 -downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 -dptagent.net,dptagent.net,inbound,ZZ,Unknown Region,0 -draftkings.com,draftkings.com,inbound,ZZ,Unknown Region,0 -dreamhost.com,dreamhost.com,inbound,ZZ,Unknown Region,0 -driftem.com,emce2.in,inbound,ZZ,Unknown Region,0 -drjays-mail.com,drjays-mail.com,inbound,ZZ,Unknown Region,0 -dromadaire-news.com,ecmcluster.com,inbound,ZZ,Unknown Region,0 -dukecareers.com,dukecareers.com,inbound,ZZ,Unknown Region,1 -duluthtradingemail.com,email-duluthtrading.com,inbound,ZZ,Unknown Region,0 -dynect-mailer.net,dynect.net,inbound,ZZ,Unknown Region,0 -dynect-mailer.net,sendlabs.com,inbound,ZZ,Unknown Region,0 -e-bodyc.com,email-bodycentral.com,inbound,ZZ,Unknown Region,0 -e-jobs-ville.com,e-jobs-ville.com,inbound,ZZ,Unknown Region,1 -e-leclerc.com,e-leclerc.com,inbound,ZZ,Unknown Region,0 -easycanvasprints.com,easycanvasprints.com,inbound,ZZ,Unknown Region,0 -easyhits4u.com,easyhits4u.com,inbound,ZZ,Unknown Region,1 -easyhits4u.com,relmax.net,inbound,ZZ,Unknown Region,0 -ebags.com,ebags.com,inbound,ZZ,Unknown Region,0 -ebay-kleinanzeigen.de,mobile.de,inbound,ZZ,Unknown Region,1 -ebay.{...},ebay.{...},inbound,ZZ,Unknown Region,0.999648 -ebay.{...},emarsys.net,inbound,ZZ,Unknown Region,0 -ebay.{...},postdirect.com,inbound,ZZ,Unknown Region,0 -ebizac3.com,ebizac3.com,inbound,ZZ,Unknown Region,0 -ebuildabear.com,ebuildabear.com,inbound,ZZ,Unknown Region,8e-06 -ed10.net,ed10.com,inbound,ZZ,Unknown Region,0 -eddiebauer.com,eddiebauer.com,inbound,ZZ,Unknown Region,0 -eduk.com,eduk.com,inbound,ZZ,Unknown Region,0 -efamilydollar.com,efamilydollar.com,inbound,ZZ,Unknown Region,0 -elabs10.com,elabs10.com,inbound,ZZ,Unknown Region,0 -elabs12.com,elabs12.com,inbound,ZZ,Unknown Region,0 -elistas.net,elistas.net,inbound,ZZ,Unknown Region,0 -elkjop.no,ec-cluster.com,inbound,ZZ,Unknown Region,0 -elkjop.no,eccluster.com,inbound,ZZ,Unknown Region,0 -elo7.com.br,elo7.com.br,inbound,ZZ,Unknown Region,0 -email-1800contacts.com,email-1800contacts.com,inbound,ZZ,Unknown Region,0 -email-aaa.com,email-aaa.com,inbound,ZZ,Unknown Region,0 -email-aeriagames.com,email-aeriagames.com,inbound,ZZ,Unknown Region,0 -email-dressbarn.com,email-dressbarn.com,inbound,ZZ,Unknown Region,0 -email-firestone.com,reminder-firestone.com,inbound,ZZ,Unknown Region,0 -email-honest.com,email-honest.com,inbound,ZZ,Unknown Region,0 -email-od.com,email-od.com,inbound,ZZ,Unknown Region,0.999752 -email-petsmart.com,email-petsmart.com,inbound,ZZ,Unknown Region,0 -email-sportchalet.com,email-sportchalet.com,inbound,ZZ,Unknown Region,0 -email-telekom.de,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -email-totalwine.com,email-totalwine.com,inbound,ZZ,Unknown Region,0 -email-wildstar-online.com,email-carbine.com,inbound,ZZ,Unknown Region,0 -email2-beyond.com,messagebus.com,inbound,ZZ,Unknown Region,0 -email360api.com,email360api.com,inbound,ZZ,Unknown Region,0 -email365inc.com,email365inc.com,inbound,ZZ,Unknown Region,0 -email3m.com,email3m.com,inbound,ZZ,Unknown Region,0 -emailrestaurant.com,emailrestaurant.com,inbound,ZZ,Unknown Region,0 -emailtoryburch.com,emailtoryburch.com,inbound,ZZ,Unknown Region,0 -emarsys.net,emarsys.net,inbound,ZZ,Unknown Region,0.265197 -embarqmail.com,centurylink.net,inbound,ZZ,Unknown Region,0.999918 -emergencyemail.org,emergencyemail.org,inbound,ZZ,Unknown Region,0 -eminentinc.com,eminentinc.com,inbound,ZZ,Unknown Region,0 -emktviajarbarato.com.br,splio.com.br,inbound,ZZ,Unknown Region,0.875902 -employboard.com,employboard.com,inbound,ZZ,Unknown Region,1 -emsecure.net,emsecure.net,inbound,ZZ,Unknown Region,0 -emsmtp.com,emsmtp.com,inbound,ZZ,Unknown Region,0.44284 -en25.com,kqed.org,inbound,ZZ,Unknown Region,0 -enewscartes.net,bp06.net,inbound,ZZ,Unknown Region,0 -enewsletter.pl,mydeal.pl,inbound,ZZ,Unknown Region,0 -enewsletter.pl,sare25.com,inbound,ZZ,Unknown Region,0 -enplenitud.com,enplenitud.com,inbound,ZZ,Unknown Region,0 -entrepreneur.com,entrepreneur.com,inbound,ZZ,Unknown Region,0 -ethingsremembered.com,ethingsremembered.com,inbound,ZZ,Unknown Region,0 -etrade.com,etrade.com,inbound,ZZ,Unknown Region,0 -etransmail.com,etransmail.com,inbound,ZZ,Unknown Region,0 -etransmail.com,ptransmail.com,inbound,ZZ,Unknown Region,0 -etrmailbox.com,etrmailbox.com,inbound,ZZ,Unknown Region,0 -etsy.com,etsy.com,inbound,ZZ,Unknown Region,0.020849 -evernote.com,evernote.com,inbound,ZZ,Unknown Region,1 -everydayfamily.com,everydayfamily.com,inbound,ZZ,Unknown Region,0 -everytown.org,everytown.org,inbound,ZZ,Unknown Region,1 -exacttarget.com,bazaarvoice.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,booksamillion.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,exacttarget.com,inbound,ZZ,Unknown Region,0.000325 -exacttarget.com,msg.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,redboxinstant.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,skylinetechnologies.com,inbound,ZZ,Unknown Region,0 -expediamail.com,airasiago.com,inbound,ZZ,Unknown Region,1 -expediamail.com,exacttarget.com,inbound,ZZ,Unknown Region,1 -expediamail.com,expediamail.com,inbound,ZZ,Unknown Region,0.839662 -expediamail.com,quotitmail.com,inbound,ZZ,Unknown Region,0 -experteer.com,experteer.com,inbound,ZZ,Unknown Region,0 -express.com,expressfashion.com,inbound,ZZ,Unknown Region,0 -facebook.com,facebook.com,inbound,ZZ,Unknown Region,0 -facebook.com,facebook.com,outbound,ZZ,Unknown Region,1 -facebookmail.com,postini.com,inbound,ZZ,Unknown Region,0.918705 -facebookmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999846 -falabella.com,falabella.com,inbound,ZZ,Unknown Region,0 -fanatics.com,fanatics.com,inbound,ZZ,Unknown Region,0 -fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,ZZ,Unknown Region,0 -fanbridge.com,fanbridge.com,inbound,ZZ,Unknown Region,0 -fanofannas.com,fanofannas.com,inbound,ZZ,Unknown Region,0 -fansedge.com,fansedge.com,inbound,ZZ,Unknown Region,0 -farmers.com,farmers.com,inbound,ZZ,Unknown Region,0.999984 -fastcompany.com,fastcompany.com,inbound,ZZ,Unknown Region,0 -fedex.com,fedex.com,inbound,ZZ,Unknown Region,0.919932 -feld-ent.com,postdirect.com,inbound,ZZ,Unknown Region,0 -fellowshiponemail.com,fellowshiponemail.com,inbound,ZZ,Unknown Region,0 -fetlifemail.com,fetlifemail.com,inbound,ZZ,Unknown Region,0 -fidelity.com,fidelity.com,inbound,ZZ,Unknown Region,1 -financialfreedommail.com,financialfreedommail.com,inbound,ZZ,Unknown Region,0 -fingerhut.com,fingerhut.com,inbound,ZZ,Unknown Region,0 -flets.com,flets.com,inbound,ZZ,Unknown Region,0 -flirtlocal.com,flirtlocal.com,inbound,ZZ,Unknown Region,1 -flmsecure.com,fling.com,inbound,ZZ,Unknown Region,0 -flmsecure.com,flmsecure.com,inbound,ZZ,Unknown Region,0 -floridajobdepartment.com,floridajobdepartment.com,inbound,ZZ,Unknown Region,0 -fnac.com,fnac.com,inbound,ZZ,Unknown Region,0.025375 -fnb.co.za,fnb.co.za,inbound,ZZ,Unknown Region,0.325259 -foodnetwork.com,foodnetwork.com,inbound,ZZ,Unknown Region,0 -forcemail.in,iaires.com,inbound,ZZ,Unknown Region,0 -foreseegame.com,iaires.com,inbound,ZZ,Unknown Region,0 -fotffamily.com,fotffamily.com,inbound,ZZ,Unknown Region,0 -fotolivro.com.br,fotolivro.com.br,inbound,ZZ,Unknown Region,0 -fpmailerbr.com,fpmailerbr.com,inbound,ZZ,Unknown Region,0 -freecycle.org,freecycle.org,inbound,ZZ,Unknown Region,1 -freelancer.com,getafreelancer.com,inbound,ZZ,Unknown Region,0 -freesafelistmailer.com,waters-advertising.com,inbound,ZZ,Unknown Region,0 -freshmail.pl,freshmail.pl,inbound,ZZ,Unknown Region,0 -fridays.com,fridays.com,inbound,ZZ,Unknown Region,0 -frk.com,frk.com,inbound,ZZ,Unknown Region,1 -frontdoor.com,frontdoor.com,inbound,ZZ,Unknown Region,0 -frontgate-email.com,frontgate-email.com,inbound,ZZ,Unknown Region,0 -fuckbooknet.net,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -fundplaza.co.in,arrowsignindia.com,inbound,ZZ,Unknown Region,0 -fundplaza.in,fundplaza.in,inbound,ZZ,Unknown Region,0 -funonthenet.in,funonthenet.in,inbound,ZZ,Unknown Region,1 -futurmailer.pt,futurmailer.pt,inbound,ZZ,Unknown Region,0 -gabbar.info,gabbar.info,inbound,ZZ,Unknown Region,1 -gamefly.com,gamefly.com,inbound,ZZ,Unknown Region,0.014317 -gamingmails.com,gamingmails.com,inbound,ZZ,Unknown Region,0 -gap.com,gap.com,inbound,ZZ,Unknown Region,0 -gap.eu,gap.eu,inbound,ZZ,Unknown Region,0 -gapcanada.ca,gapcanada.ca,inbound,ZZ,Unknown Region,0 -garanti.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 -garnethill-email.com,garnethill-email.com,inbound,ZZ,Unknown Region,0 -gaylordalert.com,gaylordalert.com,inbound,ZZ,Unknown Region,0 -gcast.com.au,systemsserver.net,inbound,ZZ,Unknown Region,0 -gdtsuccess.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 -geico.com,geico.com,inbound,ZZ,Unknown Region,0.096795 -gene.com,roche.com,inbound,ZZ,Unknown Region,1 -geocaching.com,groundspeak.com,inbound,ZZ,Unknown Region,1 -getinbox.net,getinbox.net,inbound,ZZ,Unknown Region,0 -getkeepsafe.com,getkeepsafe.com,inbound,ZZ,Unknown Region,1 -getmein.com,getmein.com,inbound,ZZ,Unknown Region,0 -getresponse.com,getresponse.com,inbound,ZZ,Unknown Region,0 -gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,ZZ,Unknown Region,0 -gilt.com,gilt.com,inbound,ZZ,Unknown Region,1e-06 -github.com,github.net,inbound,ZZ,Unknown Region,1 -glassdoor.com,glassdoor.com,inbound,ZZ,Unknown Region,0 -globalmembersupport.com,globalmembersupport.com,inbound,ZZ,Unknown Region,0 -globalspec.com,globalspec.com,inbound,ZZ,Unknown Region,0 -gmail.com,blackberry.com,inbound,ZZ,Unknown Region,0.998326 -gmail.com,postini.com,inbound,ZZ,Unknown Region,0.891175 -gmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.998661 -go.com,starwave.com,inbound,ZZ,Unknown Region,0.006276 -godtubemail.com,godtubemail.com,inbound,ZZ,Unknown Region,0 -godvinemail.com,godvinemail.com,inbound,ZZ,Unknown Region,0 -gohappy.com.tw,gohappy.com.tw,inbound,ZZ,Unknown Region,0 -goldenbrands.gr,goldenbrands.gr,inbound,ZZ,Unknown Region,1 -golfnow.com,email-golfnow.com,inbound,ZZ,Unknown Region,0 -google.com,postini.com,inbound,ZZ,Unknown Region,0.81059 -gop.com,gop.com,inbound,ZZ,Unknown Region,0 -grabone-mail-ie.com,grabone-mail-ie.com,inbound,ZZ,Unknown Region,0 -grabone-mail.com,grabone-mail.com,inbound,ZZ,Unknown Region,0 -gratka.pl,gratka.pl,inbound,ZZ,Unknown Region,0 -grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,ZZ,Unknown Region,0 -groopdealz.com,groopdealz.com,inbound,ZZ,Unknown Region,1 -groupalia.es,groupalia.es,inbound,ZZ,Unknown Region,0 -groupalia.it,groupalia.it,inbound,ZZ,Unknown Region,0 -groupon.jp,data-hotel.net,inbound,ZZ,Unknown Region,0 -groupon.{...},groupon.{...},inbound,ZZ,Unknown Region,0.990176 -grouponmail.{...},grouponmail.{...},inbound,ZZ,Unknown Region,0 -grubhubmail.com,grubhubmail.com,inbound,ZZ,Unknown Region,0 -grupanya.com,euromsg.net,inbound,ZZ,Unknown Region,0 -guruin.info,guru.net.in,inbound,ZZ,Unknown Region,1 -habitaclia.com,splio.es,inbound,ZZ,Unknown Region,0.951406 -harborfreightemail.com,harborfreightemail.com,inbound,ZZ,Unknown Region,0 -hautelook.com,hautelook.com,inbound,ZZ,Unknown Region,0 -hepsiburada.com,euromsg.net,inbound,ZZ,Unknown Region,0 -herbalifemail.com,herbalifemail.com,inbound,ZZ,Unknown Region,0 -herculist.com,herculist.com,inbound,ZZ,Unknown Region,0 -hgtv.com,hgtv.com,inbound,ZZ,Unknown Region,0 -hhgreggemail.com,hhgreggemail.com,inbound,ZZ,Unknown Region,0 -hipmunk.com,hipmunk.com,inbound,ZZ,Unknown Region,0 -hln.be,persgroep-ops.net,inbound,ZZ,Unknown Region,0 -hm-f.jp,hm-f.jp,inbound,ZZ,Unknown Region,0 -hobsonsmail.com,hobsonsmail.com,inbound,ZZ,Unknown Region,0 -homebaselife.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -homechoice.co.za,homechoice.co.za,inbound,ZZ,Unknown Region,0 -homedepotemail.com,homedepotemail.com,inbound,ZZ,Unknown Region,0 -honto.jp,honto.jp,inbound,ZZ,Unknown Region,0 -horoscope.com,center.com,inbound,ZZ,Unknown Region,0 -hotels.com,hotels.com,inbound,ZZ,Unknown Region,0 -hotelurbano.com.br,allin.com.br,inbound,ZZ,Unknown Region,0 -hotmail.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999944 -hotmail.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 -hotspotmailer.com,hotspotmailer.com,inbound,ZZ,Unknown Region,1 -hp.com,hp.com,inbound,ZZ,Unknown Region,0.198696 -hsn.com,hsn.com,inbound,ZZ,Unknown Region,0 -htcampusmailer.com,eccluster.com,inbound,ZZ,Unknown Region,0 -hubspot.com,hubspot.com,inbound,ZZ,Unknown Region,1 -huinforma.com.br,huinforma.com.br,inbound,ZZ,Unknown Region,0 -hulumail.com,hulumail.com,inbound,ZZ,Unknown Region,0 -hungryhouse.co.uk,mxmfb.com,inbound,ZZ,Unknown Region,0 -huntington.com,huntington.com,inbound,ZZ,Unknown Region,0.986937 -i-say.com,ipsos-interactive.com,inbound,ZZ,Unknown Region,1 -icloud.com,apple.com,inbound,ZZ,Unknown Region,1 -icloud.com,icloud.com,outbound,ZZ,Unknown Region,1 -icloud.com,mac.com,inbound,ZZ,Unknown Region,1 -icloud.com,me.com,inbound,ZZ,Unknown Region,0.999995 -ifttt.com,ifttt.com,inbound,ZZ,Unknown Region,1 -ign.com,ign.com,inbound,ZZ,Unknown Region,0 -ignitionsender.com,ignitionsender.com,inbound,ZZ,Unknown Region,0 -iheart.com,iheart.com,inbound,ZZ,Unknown Region,0 -immobilienscout24.de,immobilienscout24.de,inbound,ZZ,Unknown Region,1 -in-boxpays.com,in-boxpays.com,inbound,ZZ,Unknown Region,0 -indiamart.com,indiamart.com,inbound,ZZ,Unknown Region,1 -indiatimes.com,speakingtree.in,inbound,ZZ,Unknown Region,0 -indiatimeshop.com,sendpal.in,inbound,ZZ,Unknown Region,0 -indieroyale.com,desura.com,inbound,ZZ,Unknown Region,1 -infibeam.com,eccluster.com,inbound,ZZ,Unknown Region,0 -infopanel.jp,mailds.jp,inbound,ZZ,Unknown Region,0 -infos-micromania.com,infos-micromania.com,inbound,ZZ,Unknown Region,0 -infosephora.com,splio.com,inbound,ZZ,Unknown Region,0.962167 -infoworld.com,infoworld.com,inbound,ZZ,Unknown Region,0 -infusionmail.com,infusionmail.com,inbound,ZZ,Unknown Region,0 -inmotionhosting.com,inmotionhosting.com,inbound,ZZ,Unknown Region,1 -ino.com,ino.com,inbound,ZZ,Unknown Region,0.999981 -interwell.gr,interwell.gr,inbound,ZZ,Unknown Region,0 -ipcmedia.co.uk,ipcmedia.co.uk,inbound,ZZ,Unknown Region,0 -itunes.com,apple.com,inbound,ZZ,Unknown Region,0.043012 -jackwills.com,jackwills.com,inbound,ZZ,Unknown Region,0 -jalag.de,jalag.de,inbound,ZZ,Unknown Region,1 -jane.com,jane.com,inbound,ZZ,Unknown Region,1 -jango.com,jango.com,inbound,ZZ,Unknown Region,0 -jared.com,jared.com,inbound,ZZ,Unknown Region,0 -jdate.com,postdirect.com,inbound,ZZ,Unknown Region,0 -jetprivilege.com,jetprivilege.com,inbound,ZZ,Unknown Region,0 -jeuxvideo.com,jeuxvideo.com,inbound,ZZ,Unknown Region,0.148921 -jeweloscoemail.com,email-mywebgrocer2.com,inbound,ZZ,Unknown Region,0 -joann-mail.com,joann-mail.com,inbound,ZZ,Unknown Region,0 -jobinsider.com,jobinsider.com,inbound,ZZ,Unknown Region,0 -jobisjob.com,jobisjob.com,inbound,ZZ,Unknown Region,0 -jobomas.com,jobomas.com,inbound,ZZ,Unknown Region,1 -jobstreet.com,jobstreet.com,inbound,ZZ,Unknown Region,0 -jpcycles.com,jpcycles.com,inbound,ZZ,Unknown Region,0.001716 -jungleerummy.com,jungleerummy.com,inbound,ZZ,Unknown Region,1 -jusbrasil.com.br,jusbrasil.com.br,inbound,ZZ,Unknown Region,0 -just-eat.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -justclick.ru,justclick.ru,inbound,ZZ,Unknown Region,0 -justdial.com,iaires.com,inbound,ZZ,Unknown Region,0 -k1speed.com,k1speed.com,inbound,ZZ,Unknown Region,1 -kaskusnetworks.com,kaskus.com,inbound,ZZ,Unknown Region,0 -kay.com,kay.com,inbound,ZZ,Unknown Region,0 -keek.com,keek.com,inbound,ZZ,Unknown Region,1 -kgbdeals.co.uk,email1-kgbdeals.com,inbound,ZZ,Unknown Region,0 -kliksa.net,euromsg.net,inbound,ZZ,Unknown Region,0 -kliktoday.com,kliktoday.com,inbound,ZZ,Unknown Region,0 -klm-mail.com,klm-mail.com,inbound,ZZ,Unknown Region,0 -kohls.com,kohls.com,inbound,ZZ,Unknown Region,0 -kongregate.com,kongregate.com,inbound,ZZ,Unknown Region,0 -krs.bz,tricorn.net,inbound,ZZ,Unknown Region,0 -la-meteo-mail.fr,splio.com,inbound,ZZ,Unknown Region,1 -laaptuemail.com,laaptuemail.com,inbound,ZZ,Unknown Region,0 -lakewoodchurch.com,lakewoodchurch.com,inbound,ZZ,Unknown Region,0 -landsend.com,email-landsend.com,inbound,ZZ,Unknown Region,0 -landsend.com,postdirect.com,inbound,ZZ,Unknown Region,0 -laptuinvite.com,laptuinvite.com,inbound,ZZ,Unknown Region,0 -lastminute.com,lastminute.com,inbound,ZZ,Unknown Region,0 -lazerhits.com,lazerhits.com,inbound,ZZ,Unknown Region,1 -leadercontato.com.br,leadercontato.com.br,inbound,ZZ,Unknown Region,0 -lefigaro.fr,splio.com,inbound,ZZ,Unknown Region,1 -lemonde.fr,lemonde.fr,inbound,ZZ,Unknown Region,0 -life360.com,life360.com,inbound,ZZ,Unknown Region,1 -lifecare-news.com,email-lifecare.com,inbound,ZZ,Unknown Region,0 -lifemiles.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -lifescript.com,ilinkmd.com,inbound,ZZ,Unknown Region,0 -line6.com,line6.com,inbound,ZZ,Unknown Region,0 -linkedin.com,linkedin.com,inbound,ZZ,Unknown Region,0.995201 -linkedin.com,postini.com,inbound,ZZ,Unknown Region,0.940251 -liquidation.com,liquidation.com,inbound,ZZ,Unknown Region,0 -listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,ZZ,Unknown Region,0.00023 -live.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999921 -live.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 -livejournal.com,livejournal.com,inbound,ZZ,Unknown Region,0 -livemailservice.com,livemailservice.com,inbound,ZZ,Unknown Region,0 -livenation.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -lmlmgv.com.br,gvarev.com.br,inbound,ZZ,Unknown Region,0 -loccitane.com,neolane.net,inbound,ZZ,Unknown Region,0 -loft.com,anntaylor.com,inbound,ZZ,Unknown Region,0 -lombardipublishing.com,lombardipublishing.com,inbound,ZZ,Unknown Region,0 -lookout.com,lookout.com,inbound,ZZ,Unknown Region,1 -lsi.com,postini.com,inbound,ZZ,Unknown Region,0.981115 -lynxmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -lyris.net,lyris.net,inbound,ZZ,Unknown Region,0 -m1e.net,m1e.net,inbound,ZZ,Unknown Region,0.000342 -mac.com,icloud.com,outbound,ZZ,Unknown Region,1 -mac.com,mac.com,inbound,ZZ,Unknown Region,1 -macromill.com,macromill.com,inbound,ZZ,Unknown Region,0 -macys.com,macys.com,inbound,ZZ,Unknown Region,0 -magix.net,magix.net,inbound,ZZ,Unknown Region,0.673176 -mail-backcountry.com,email-bcmarketing.com,inbound,ZZ,Unknown Region,0 -mail.ru,mail.ru,inbound,ZZ,Unknown Region,0.986526 -mail.ru,mail.ru,outbound,ZZ,Unknown Region,0.006561 -maileclipse.com,emce2.in,inbound,ZZ,Unknown Region,0 -mailengine1.com,mailengine1.com,inbound,ZZ,Unknown Region,0 -mailer4u.in,elabs10.com,inbound,ZZ,Unknown Region,0 -mailersend.com,mailersend.com,inbound,ZZ,Unknown Region,0 -mailjet.com,mailjet.com,inbound,ZZ,Unknown Region,0.803083 -mailmailmail.net,mailmailmail.net,inbound,ZZ,Unknown Region,0 -mailoct.in,tcmailer14.in,inbound,ZZ,Unknown Region,0 -mailoct1.in,mailoct1.in,inbound,ZZ,Unknown Region,0 -mailoct1.in,myntramail2.in,inbound,ZZ,Unknown Region,0 -mailorama.fr,mailorama.fr,inbound,ZZ,Unknown Region,0 -mailpost.in,iaires.com,inbound,ZZ,Unknown Region,0 -mailquant.com,iaires.com,inbound,ZZ,Unknown Region,0 -mandrillapp.com,backpage.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,mcsignup.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,myjobhelperalerts.com,inbound,ZZ,Unknown Region,1 -mango.com,emstechnology2.net,inbound,ZZ,Unknown Region,0 -manipal.edu,iaires.com,inbound,ZZ,Unknown Region,0 -manta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -mar0.net,mar0.net,inbound,ZZ,Unknown Region,0.976409 -markavip.com,markavip.com,inbound,ZZ,Unknown Region,0 -marketingstudio.com,marketingstudio.com,inbound,ZZ,Unknown Region,0 -marksandspencer.com,marksandspencer.com,inbound,ZZ,Unknown Region,0 -maropost.com,mp2200.com,inbound,ZZ,Unknown Region,0 -maropost.com,survivallife.com,inbound,ZZ,Unknown Region,0 -marykay.com,marykay.com,inbound,ZZ,Unknown Region,0 -masivapp.com,masivapp.com,inbound,ZZ,Unknown Region,1 -match.com,match.com,inbound,ZZ,Unknown Region,0 -mbga.jp,mbga.jp,inbound,ZZ,Unknown Region,0 -mbna.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -mcdlv.net,mcdlv.net,inbound,ZZ,Unknown Region,0 -mcsv.net,mcsv.net,inbound,ZZ,Unknown Region,0 -me.com,icloud.com,outbound,ZZ,Unknown Region,1 -me.com,mac.com,inbound,ZZ,Unknown Region,1 -medallia.com,medallia.com,inbound,ZZ,Unknown Region,1 -mediapost.com,mediapost.com,inbound,ZZ,Unknown Region,0 -medium.com,messagebus.com,inbound,ZZ,Unknown Region,0 -medscape.com,medscape.com,inbound,ZZ,Unknown Region,0 -meetup.com,meetup.com,inbound,ZZ,Unknown Region,0 -melaleuca.com,melaleuca.com,inbound,ZZ,Unknown Region,0 -mercadojobs.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -mercadolibre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 -mercadolivre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 -merceworld.com,merceworld.com,inbound,ZZ,Unknown Region,0.996737 -merodea.me,sendgrid.net,inbound,ZZ,Unknown Region,1 -metro.co.in,srv2.de,inbound,ZZ,Unknown Region,0.966947 -mgmresorts.com,mgmresorts.com,inbound,ZZ,Unknown Region,0 -microsoft.com,msn.com,inbound,ZZ,Unknown Region,1 -microsoftemail.com,microsoftemail.com,inbound,ZZ,Unknown Region,0 -microsoftemail.com,microsoftstoreemail.com,inbound,ZZ,Unknown Region,0 -mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,ZZ,Unknown Region,0 -minhavida.com.br,minhavida.com.br,inbound,ZZ,Unknown Region,0 -mixi.jp,mixi.jp,inbound,ZZ,Unknown Region,0 -mjinn.com,mailurja.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktdns.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktomail.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktroute.com,inbound,ZZ,Unknown Region,0 -mlsend.com,mlsend.com,inbound,ZZ,Unknown Region,0 -mlsend2.com,mlsend2.com,inbound,ZZ,Unknown Region,0 -mlssoccer.com,mlssoccer.com,inbound,ZZ,Unknown Region,0 -mmaco.net,mmaco.net,inbound,ZZ,Unknown Region,0.999998 -mmorpg.com,mmorpg.com,inbound,ZZ,Unknown Region,0.002979 -mocospace.com,mocospace.com,inbound,ZZ,Unknown Region,0 -moneyforward.com,moneyforward.com,inbound,ZZ,Unknown Region,0 -moneysupermarketmail.com,moneysupermarketmail.com,inbound,ZZ,Unknown Region,0 -monografias.com,elistas.net,inbound,ZZ,Unknown Region,0 -monster.com,monster.com,inbound,ZZ,Unknown Region,0.000503 -monsterindia.com,monster.co.in,inbound,ZZ,Unknown Region,0 -mooply.co,mailendo.com,inbound,ZZ,Unknown Region,0 -morningstar.net,morningstar.net,inbound,ZZ,Unknown Region,0 -mothercaregroup.com,neolane.net,inbound,ZZ,Unknown Region,0 -mozilla.org,mozilla.com,inbound,ZZ,Unknown Region,0.633241 -mpse.jp,mpme.jp,inbound,ZZ,Unknown Region,0 -ms00.net,ms00.net,inbound,ZZ,Unknown Region,0 -msdp1.com,msdp1.com,inbound,ZZ,Unknown Region,0 -msn.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999946 -msn.com,hotmail.{...},outbound,ZZ,Unknown Region,1 -musiciansfriend.com,musiciansfriend.com,inbound,ZZ,Unknown Region,0 -mxmfb.com,mxmfb.com,inbound,ZZ,Unknown Region,0 -mycolorscreen.com,mta4.net,inbound,ZZ,Unknown Region,0 -myfitnesspal.com,messagebus.com,inbound,ZZ,Unknown Region,0 -mygroupon.co.th,grouponmail.{...},inbound,ZZ,Unknown Region,0 -myheritage.com,myheritage.com,inbound,ZZ,Unknown Region,0 -myideeli.com,myideeli.com,inbound,ZZ,Unknown Region,0 -myntramail.com,iaires.com,inbound,ZZ,Unknown Region,0 -myntramail.com,myntramail.com,inbound,ZZ,Unknown Region,0 -myntramails.in,icubes.in,inbound,ZZ,Unknown Region,0 -myoutlets.in,trustmailer.com,inbound,ZZ,Unknown Region,0 -mypoints.com,mypoints.com,inbound,ZZ,Unknown Region,0 -mysale.my,mysale.my,inbound,ZZ,Unknown Region,0 -mysale.ph,mysale.ph,inbound,ZZ,Unknown Region,0 -mysupermarket.co.uk,mysupermarket.co.uk,inbound,ZZ,Unknown Region,0 -mysurvey.com,mysurvey.com,inbound,ZZ,Unknown Region,0 -mysurvey.eu,mysurvey.com,inbound,ZZ,Unknown Region,0 -myvegas.com,myvegas.com,inbound,ZZ,Unknown Region,1 -naaptoldeals.com,eccluster.com,inbound,ZZ,Unknown Region,0 -nanomail.com.br,araie.com.br,inbound,ZZ,Unknown Region,1 -nascar.com,nascar.com,inbound,ZZ,Unknown Region,0 -nationbuilder.com,nationbuilder.com,inbound,ZZ,Unknown Region,1 -nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,ZZ,Unknown Region,0 -naukri.com,naukri.com,inbound,ZZ,Unknown Region,0 -navy.mil,navy.mil,inbound,ZZ,Unknown Region,0.000243 -nend.net,postini.com,inbound,ZZ,Unknown Region,0 -netatlantic.com,netatlantic.com,inbound,ZZ,Unknown Region,0.001085 -netflix.com,amazonses.com,inbound,ZZ,Unknown Region,0.999999 -netflix.com,netflix.com,inbound,ZZ,Unknown Region,1 -netlogmail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 -netprosoftmail.com,netprosoftmail.com,inbound,ZZ,Unknown Region,0 -netshoes.com.br,netshoes.com.br,inbound,ZZ,Unknown Region,0 -newegg.com,newegg.com,inbound,ZZ,Unknown Region,2e-06 -newmarkethealth.com,newmarkethealth.com,inbound,ZZ,Unknown Region,0 -news-h5g.com,news-h5g.com,inbound,ZZ,Unknown Region,0 -newsletter-verychic.com,splio.es,inbound,ZZ,Unknown Region,0.9804 -newsmax.com,newsmax.com,inbound,ZZ,Unknown Region,0 -nflshop.com,nflshop.com,inbound,ZZ,Unknown Region,0 -nieuwsblad.be,vummail.be,inbound,ZZ,Unknown Region,0 -nike.com,nike.com,inbound,ZZ,Unknown Region,0 -ninewestmail.com,ninewestmail.com,inbound,ZZ,Unknown Region,0 -nokia.com,nokia.com,inbound,ZZ,Unknown Region,0.001256 -npr.org,npr.org,inbound,ZZ,Unknown Region,0 -ns.nl,tripolis.com,inbound,ZZ,Unknown Region,0 -nsandi.com,mxmfb.com,inbound,ZZ,Unknown Region,0 -nytimes.com,nytimes.com,inbound,ZZ,Unknown Region,0 -nzsale.co.nz,nzsale.co.nz,inbound,ZZ,Unknown Region,0 -oakley.com,oakley.com,inbound,ZZ,Unknown Region,0 -ocmail1.in,tcmail.in,inbound,ZZ,Unknown Region,0 -ocmail14.in,tcmailer5.in,inbound,ZZ,Unknown Region,0 -ocmail22.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 -ocmail22.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 -ocmail40.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 -ocmail40.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 -ocnmail.in,ocmail6.in,inbound,ZZ,Unknown Region,0 -ocnmail.in,tcmail3.in,inbound,ZZ,Unknown Region,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,ZZ,Unknown Region,0 -ofertix.com,ofertix.com,inbound,ZZ,Unknown Region,0 -offers.com,offers.com,inbound,ZZ,Unknown Region,1 -officedepot.com,officedepot.com,inbound,ZZ,Unknown Region,0.019269 -officemax.com,officemax.com,inbound,ZZ,Unknown Region,0 -officemax.com,officemaxworkplace.com,inbound,ZZ,Unknown Region,0 -ofsys.com,bulletin-metro.ca,inbound,ZZ,Unknown Region,0 -oknotify2.com,oknotify2.com,inbound,ZZ,Unknown Region,0 -oldnavy.ca,oldnavy.ca,inbound,ZZ,Unknown Region,0 -oldnavy.com,oldnavy.com,inbound,ZZ,Unknown Region,0 -omahasteaks.com,omahasteaks.com,inbound,ZZ,Unknown Region,0 -oneindia.in,mailurja.com,inbound,ZZ,Unknown Region,0 -onekingslane.com,onekingslane.com,inbound,ZZ,Unknown Region,0 -onlive.com,ipost.com,inbound,ZZ,Unknown Region,0 -onmicrosoft.com,outlook.com,inbound,ZZ,Unknown Region,1 -optimusmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -orderscatalog.com,orderscatalog.com,inbound,ZZ,Unknown Region,0 -oroscopofree.com,adsender.us,inbound,ZZ,Unknown Region,0 -os-email.com,os-email.com,inbound,ZZ,Unknown Region,0 -oshkoshbgosh.com,oshkoshbgosh.com,inbound,ZZ,Unknown Region,6e-06 -osu.edu,outlook.com,inbound,ZZ,Unknown Region,1 -otto.de,eccluster.com,inbound,ZZ,Unknown Region,1 -ouffer.com,ouffer.com,inbound,ZZ,Unknown Region,0 -ourtime.com,seniorpeoplemeet.com,inbound,ZZ,Unknown Region,0 -outback.com,outback.com,inbound,ZZ,Unknown Region,0 -outlook.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999843 -outlook.com,hotmail.{...},outbound,ZZ,Unknown Region,1 -outspot.be,teneo.be,inbound,ZZ,Unknown Region,0 -outspot.nl,teneo.be,inbound,ZZ,Unknown Region,0 -ovenmail.com,iaires.com,inbound,ZZ,Unknown Region,0 -overstock.com,overstock.com,inbound,ZZ,Unknown Region,0 -ovh.net,ovh.net,inbound,ZZ,Unknown Region,0.352531 -ozsale.com.au,ozsale.com.au,inbound,ZZ,Unknown Region,0.000192 -panelplace.com,smtp.com,inbound,ZZ,Unknown Region,0 -panerabreadnews.com,panerabreadnews.com,inbound,ZZ,Unknown Region,0 -pantaloondirect.net,iaires.com,inbound,ZZ,Unknown Region,0 -papajohns-specials.com,papajohns-specials.com,inbound,ZZ,Unknown Region,0 -paradisepublishers.com,paradisepublishers.com,inbound,ZZ,Unknown Region,0.99957 -path.com,path.com,inbound,ZZ,Unknown Region,1 -payback.in,eccluster.com,inbound,ZZ,Unknown Region,0 -payback.in,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -payback.in,ecmcluster.com,inbound,ZZ,Unknown Region,0 -paypal.co.uk,paypal.com,inbound,ZZ,Unknown Region,1 -paypal.com,paypal.com,inbound,ZZ,Unknown Region,0.608834 -paypal.com.au,paypal.com,inbound,ZZ,Unknown Region,1 -paypal.de,paypal.com,inbound,ZZ,Unknown Region,1 -pd25.com,pd25.com,inbound,ZZ,Unknown Region,1 -peanuthome.info,adopterc.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,aguitytr.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bevest.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bluester.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,burror.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bursion.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cantexi.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,caserhi.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,celect.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,chintone.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,citery.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cleathal.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,colicom.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,complec.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,deciarc.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,declaws.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,dewest.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,epconce.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,fnotec.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,folkswor.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,forepert.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,gurgaro.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,heallyps.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,holeph.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,homewor.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,hydroni.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,ingenbu.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,unglibshudder.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,adcrent.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,addmiel.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,agilhe.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,allegap.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,andvore.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,angogl.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,animass.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,arettery.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,aribank.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,arkefoc.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,avenog.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,barrave.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bindowmo.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bitravit.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,borsand.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,branti.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,breaserp.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bredogly.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,briantra.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bridea.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,carial.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,castac.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,chedoner.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,chiquent.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cinchoi.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cliate.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cognn.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,abjibbin.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,audiette.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,blancer.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,bluester.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,burror.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,bursion.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,caserhi.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,celect.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,cleathal.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,complec.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,condost.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,deciarc.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,declaws.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,epconce.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ferrayer.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,fnotec.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,folkswor.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,forepert.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,gurgaro.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,holeph.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,homewor.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,hydroni.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ingenbu.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,ZZ,Unknown Region,0 -pennwell.com,pennwell.com,inbound,ZZ,Unknown Region,0 -perfectworld.com,perfectworld.com,inbound,ZZ,Unknown Region,0 -personare.com.br,personare.com.br,inbound,ZZ,Unknown Region,0 -pge.com,pge.com,inbound,ZZ,Unknown Region,0.149792 -philosophy.com,philosophy.com,inbound,ZZ,Unknown Region,0 -phpclasses.org,phpclasses.org,inbound,ZZ,Unknown Region,0 -pinger.com,pinger.com,inbound,ZZ,Unknown Region,0 -pinterest.com,pinterest.com,inbound,ZZ,Unknown Region,1 -pizzahutoffers.com,pizzahutoffers.com,inbound,ZZ,Unknown Region,0 -playstationmail.net,playstationmail.net,inbound,ZZ,Unknown Region,0 -plumdistrict.com,plumdistrict.com,inbound,ZZ,Unknown Region,1 -politicoemail.com,politicoemail.com,inbound,ZZ,Unknown Region,0 -polyvore.com,polyvore.com,inbound,ZZ,Unknown Region,1 -popsugar.com,popsugar.com,inbound,ZZ,Unknown Region,0 -priceline.com,priceline.com,inbound,ZZ,Unknown Region,1 -princess.com,princess.com,inbound,ZZ,Unknown Region,0 -privoscite.si,privoscite.si,inbound,ZZ,Unknown Region,0 -profitcenteronline.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 -progressive.com,progressive.com,inbound,ZZ,Unknown Region,0.814003 -publix.com,publix.com,inbound,ZZ,Unknown Region,0 -puritan.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 -qoinpro.com,qoinpro.com,inbound,ZZ,Unknown Region,1 -qoo10.jp,qoo10.jp,inbound,ZZ,Unknown Region,4e-06 -qoo10.sg,qoo10.co.id,inbound,ZZ,Unknown Region,1.9e-05 -qoo10.sg,qoo10.com,inbound,ZZ,Unknown Region,0 -qoo10.sg,qoo10.my,inbound,ZZ,Unknown Region,2.2e-05 -qoo10.sg,qoo10.sg,inbound,ZZ,Unknown Region,8e-06 -quinstreet.com,neoquin.com,inbound,ZZ,Unknown Region,0 -quora.com,quora.com,inbound,ZZ,Unknown Region,1 -rabota.ua,rabota.ua,inbound,ZZ,Unknown Region,0 -rackroom-email.com,rackroom-email.com,inbound,ZZ,Unknown Region,0 -railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,ZZ,Unknown Region,0 -rakuten.co.jp,rakuten.co.jp,inbound,ZZ,Unknown Region,0 -rakuten.com,rakuten.com,inbound,ZZ,Unknown Region,0 -reactiveadz.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 -realage-mail.com,postdirect.com,inbound,ZZ,Unknown Region,0 -redbox.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -redcross.org.uk,redcross.org.uk,inbound,ZZ,Unknown Region,0 -rediffmail.com,rediffmail.com,inbound,ZZ,Unknown Region,0 -reebonz.com,reebonz.com,inbound,ZZ,Unknown Region,0.759458 -reed.co.uk,reed.co.uk,inbound,ZZ,Unknown Region,0 -regie11.net,odiso.net,inbound,ZZ,Unknown Region,0 -regionalhelpwanted.com,regionalhelpwanted.com,inbound,ZZ,Unknown Region,1 -registrar-servers.com,registrar-servers.com,inbound,ZZ,Unknown Region,0.05163 -repica.jp,kakaku.com,inbound,ZZ,Unknown Region,0 -repica.jp,repica.jp,inbound,ZZ,Unknown Region,0 -republicwireless.com,republicwireless.com,inbound,ZZ,Unknown Region,0.033564 -retailjobinsider.com,retailjobinsider.com,inbound,ZZ,Unknown Region,0 -retailmenot.com,retailmenot.com,inbound,ZZ,Unknown Region,4.68294583517529e-07 -reverbnation.com,reverbnation.com,inbound,ZZ,Unknown Region,0 -revolutiongolf.com,revolutiongolf.com,inbound,ZZ,Unknown Region,0 -reyrey.net,reyrey.net,inbound,ZZ,Unknown Region,0.999904 -richyrichmailer.com,maddog-productions.info,inbound,ZZ,Unknown Region,1 -rikunabi.com,rikunabi.com,inbound,ZZ,Unknown Region,0 -ringcentral.com,ringcentral.com,inbound,ZZ,Unknown Region,1 -rmtr.de,rapidmail.de,inbound,ZZ,Unknown Region,0 -rnmk.com,rnmk.com,inbound,ZZ,Unknown Region,0 -rockpath.info,rockpath.info,inbound,ZZ,Unknown Region,1 -rogers.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -rogers.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -rookiestewsemails.com,rookiestewsemails.com,inbound,ZZ,Unknown Region,0 -royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,ZZ,Unknown Region,0 -rpo9usa.email,rpo9usa.email,inbound,ZZ,Unknown Region,0 -rr.com,rr.com,inbound,ZZ,Unknown Region,2e-06 -rr.com,rr.com,outbound,ZZ,Unknown Region,0 -rsgsv.net,rsgsv.net,inbound,ZZ,Unknown Region,0 -rummycirclemails.com,eccluster.com,inbound,ZZ,Unknown Region,0 -runkeeper.com,runkeeper.com,inbound,ZZ,Unknown Region,0 -s3s-br1.net,splio.com.br,inbound,ZZ,Unknown Region,0.908187 -s3s-main.net,splio.com,inbound,ZZ,Unknown Region,0.970428 -s4s-pl1.pl,splio.com,inbound,ZZ,Unknown Region,0.939032 -safe-sender.net,safe-sender.net,inbound,ZZ,Unknown Region,0 -safeway.com,safeway.com,inbound,ZZ,Unknown Region,0.000382 -salesforce.com,postini.com,inbound,ZZ,Unknown Region,0.915635 -salesforce.com,salesforce.com,inbound,ZZ,Unknown Region,0.952721 -samsungusa.com,samsungusa.com,inbound,ZZ,Unknown Region,0 -sanmina-sci.com,postini.com,inbound,ZZ,Unknown Region,0.999991 -sanmina.com,postini.com,inbound,ZZ,Unknown Region,0.999828 -saturday.com,saturday.com,inbound,ZZ,Unknown Region,0 -sbcglobal.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999985 -sears.ca,sears.ca,inbound,ZZ,Unknown Region,0.005447 -searscard.com,searscard.com,inbound,ZZ,Unknown Region,1 -secretescapes.com,secretescapes.com,inbound,ZZ,Unknown Region,0 -secure.ne.jp,secure.ne.jp,inbound,ZZ,Unknown Region,0.000618 -secureserver.net,secureserver.net,inbound,ZZ,Unknown Region,0 -seek.com.au,seek.com.au,inbound,ZZ,Unknown Region,0 -selectacast.net,selectacast.net,inbound,ZZ,Unknown Region,0.000561 -semana.com,semana.com,inbound,ZZ,Unknown Region,0 -sendgrid.info,sendgrid.net,inbound,ZZ,Unknown Region,0.999896 -sendgrid.me,sendgrid.net,inbound,ZZ,Unknown Region,1 -sendpal.in,sendpal.in,inbound,ZZ,Unknown Region,0 -serviciobancomer.com,serviciobancomer.com,inbound,ZZ,Unknown Region,0 -sfid01.com,sfid01.com,inbound,ZZ,Unknown Region,0 -shaadi.com,shaadi.com,inbound,ZZ,Unknown Region,0 -shop2gether.com.br,shop2gether.com.br,inbound,ZZ,Unknown Region,0 -shopjustice.com,shopjustice.com,inbound,ZZ,Unknown Region,0 -shopnineteenmails.in,iaires.com,inbound,ZZ,Unknown Region,0 -shoppersstop.com,shoppersstop.com,inbound,ZZ,Unknown Region,0 -shoprite-email.com,email-mywebgrocer.com,inbound,ZZ,Unknown Region,0 -showingtime.com,showingtime.com,inbound,ZZ,Unknown Region,0 -showroomprive.com,showroomprive.be,inbound,ZZ,Unknown Region,0 -showroomprive.com,showroomprive.nl,inbound,ZZ,Unknown Region,0 -showroomprive.es,showroomprive.es,inbound,ZZ,Unknown Region,0 -showroomprive.it,showroomprive.pt,inbound,ZZ,Unknown Region,0 -showroomprive.pt,showroomprive.co.uk,inbound,ZZ,Unknown Region,0 -shtyle.fm,shtyle.fm,inbound,ZZ,Unknown Region,0 -simplesafelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -simpletextadz.com,web-hosting.com,inbound,ZZ,Unknown Region,1 -singsale.com.sg,singsale.com.sg,inbound,ZZ,Unknown Region,0 -skillpages-mailer.com,dynect.net,inbound,ZZ,Unknown Region,0 -skymall.com,skymall.com,inbound,ZZ,Unknown Region,0 -skynet.be,belgacom.be,inbound,ZZ,Unknown Region,0 -skype.com,skype.com,inbound,ZZ,Unknown Region,0 -skyscanner.net,skyscanner.net,inbound,ZZ,Unknown Region,1 -slickdeals.net,slickdeals.net,inbound,ZZ,Unknown Region,0 -slidesharemail.com,slideshare.net,inbound,ZZ,Unknown Region,1 -smartresponder.ru,smartresponder.ru,inbound,ZZ,Unknown Region,1 -snagajob-email.com,snagajob-email.com,inbound,ZZ,Unknown Region,0 -snapdeal.com,snapdeal.com,inbound,ZZ,Unknown Region,0 -socialsex.biz,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -softbank.jp,softbank.jp,outbound,ZZ,Unknown Region,0 -softbank.ne.jp,softbank.ne.jp,inbound,ZZ,Unknown Region,0 -softbank.ne.jp,softbank.ne.jp,outbound,ZZ,Unknown Region,0 -sony.com,sony.com,inbound,ZZ,Unknown Region,0 -sonyrewards.com,sonyrewards.com,inbound,ZZ,Unknown Region,0 -soundcloudmail.com,soundcloudmail.com,inbound,ZZ,Unknown Region,0.999996 -spanishdict.com,spanishdict.com,inbound,ZZ,Unknown Region,0 -sparklist.com,sparklist.com,inbound,ZZ,Unknown Region,0 -spartoo.com,spartoo.com,inbound,ZZ,Unknown Region,0 -speedyrewards-email.com,speedyrewards-email.com,inbound,ZZ,Unknown Region,0 -spotifymail.com,spotifymail.com,inbound,ZZ,Unknown Region,1 -ssgadm.com,ssg.com,inbound,ZZ,Unknown Region,0 -staffeazymailers.com,iaires.com,inbound,ZZ,Unknown Region,0 -stakemail.com,iaires.com,inbound,ZZ,Unknown Region,0 -stampmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -standaard.be,vummail.be,inbound,ZZ,Unknown Region,0 -stansberryresearch.com,stansberry-re.net,inbound,ZZ,Unknown Region,0 -stansberryresearch.com,stansberryresearch.com,inbound,ZZ,Unknown Region,0 -staples.co.uk,ncrwebhost.de,inbound,ZZ,Unknown Region,0 -starsports.com,eccluster.com,inbound,ZZ,Unknown Region,0 -startwire.com,jobsreport.com,inbound,ZZ,Unknown Region,1 -starwoodhotels.com,outlook.com,inbound,ZZ,Unknown Region,1 -state-of-the-art-mailer.com,futurebanners.net,inbound,ZZ,Unknown Region,0 -stayfriends.de,stayfriends.de,inbound,ZZ,Unknown Region,0 -steampowered.com,steampowered.com,inbound,ZZ,Unknown Region,1 -stelladot.com,stelladot.com,inbound,ZZ,Unknown Region,0 -stjobs.sg,st701.com,inbound,ZZ,Unknown Region,1 -stjude.org,stjude.org,inbound,ZZ,Unknown Region,0.006723 -strava.com,strava.com,inbound,ZZ,Unknown Region,1 -streeteasy.com,streeteasy.com,inbound,ZZ,Unknown Region,0 -stylecareers.com,stylecareers.com,inbound,ZZ,Unknown Region,0 -subtend.info,subtend.info,inbound,ZZ,Unknown Region,1 -subway.com,subway.com,inbound,ZZ,Unknown Region,0 -surveyspot.com,ssisurveys.com,inbound,ZZ,Unknown Region,0 -sweepstakesalerts.com,sweepstakesalerts.com,inbound,ZZ,Unknown Region,0 -swimoutlet.com,isport.com,inbound,ZZ,Unknown Region,0 -sympatico.ca,hotmail.{...},inbound,ZZ,Unknown Region,1 -synchronyfinancial.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tadtopmails.com,tadtopmails.com,inbound,ZZ,Unknown Region,0 -taishinbank.com.tw,taishinbank.com.tw,inbound,ZZ,Unknown Region,0 -take2games.com,take2games.com,inbound,ZZ,Unknown Region,0 -talkmatch.com,talkmatch.com,inbound,ZZ,Unknown Region,0 -tanga.com,tanga.com,inbound,ZZ,Unknown Region,0 -tanningmail.com,tanningmail.com,inbound,ZZ,Unknown Region,0 -tappingsolutionemail.com,tappingsolutionemail.com,inbound,ZZ,Unknown Region,0 -target.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tasteofhome.com,tasteofhome.com,inbound,ZZ,Unknown Region,0 -tchibo.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 -teambuymail.com,teambuymail.com,inbound,ZZ,Unknown Region,0 -teamsnap.com,teamsnap.com,inbound,ZZ,Unknown Region,1 -technolutions.net,technolutions.net,inbound,ZZ,Unknown Region,1 -telegraph.co.uk,telegraph.co.uk,inbound,ZZ,Unknown Region,0 -telus.com,telus.com,inbound,ZZ,Unknown Region,0 -templeandwebster.com.au,templeandwebster.com.au,inbound,ZZ,Unknown Region,0 -texasjobdepartment.com,texasjobdepartment.com,inbound,ZZ,Unknown Region,0 -thebodyshop-usa.com,email-bodyshop.com,inbound,ZZ,Unknown Region,0 -thebodyshop-usa.com,postdirect.com,inbound,ZZ,Unknown Region,0 -thecarousell.com,thecarousell.com,inbound,ZZ,Unknown Region,1 -theguardian.com,theguardian.com,inbound,ZZ,Unknown Region,0 -thepamperedchef.com,thepamperedchef.com,inbound,ZZ,Unknown Region,0 -thephonehouse.es,splio.com,inbound,ZZ,Unknown Region,0.983578 -thephonehouse.es,splio.es,inbound,ZZ,Unknown Region,0.983266 -therealreal.com,email-realreal.com,inbound,ZZ,Unknown Region,0 -thesource.ca,thesource.ca,inbound,ZZ,Unknown Region,0 -thewarehouse.co.nz,thewarehouse.co.nz,inbound,ZZ,Unknown Region,0 -thinkgeek.com,thinkgeek.com,inbound,ZZ,Unknown Region,1e-06 -thirtyonegifts.com,thirtyonegifts.com,inbound,ZZ,Unknown Region,0 -thomascook.com,eccluster.com,inbound,ZZ,Unknown Region,0 -ticketmaster.com,ticketmaster.com,inbound,ZZ,Unknown Region,0.001001 -ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,ZZ,Unknown Region,0 -timehop.com,timehop.com,inbound,ZZ,Unknown Region,1 -timeout.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -timewarnercable.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tinyletterapp.com,tinyletterapp.com,inbound,ZZ,Unknown Region,0 -tobi.com,messagebus.com,inbound,ZZ,Unknown Region,0 -topface.com,topface.com,inbound,ZZ,Unknown Region,1e-06 -topica.com,topica-silver-y.com,inbound,ZZ,Unknown Region,0 -topqpon.si,topqpon.si,inbound,ZZ,Unknown Region,0 -touchbase2.com,mailurja.com,inbound,ZZ,Unknown Region,0 -townnews-mail.com,townnews-mail.com,inbound,ZZ,Unknown Region,0 -trabajar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999997 -trabalhar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999994 -tradeloop.com,tradeloop.com,inbound,ZZ,Unknown Region,1 -trafficwave.net,trafficwave.net,inbound,ZZ,Unknown Region,0 -transittraveljobinsider.com,transittraveljobinsider.com,inbound,ZZ,Unknown Region,0 -transportexchangegroup.com,transportexchangegroup.com,inbound,ZZ,Unknown Region,0.998825 -travelchannel.com,travelchannel.com,inbound,ZZ,Unknown Region,0 -travelocity.com,travelocity.com,inbound,ZZ,Unknown Region,0.000195 -travelzoo.com,travelzoo.com,inbound,ZZ,Unknown Region,0 -trclient.com,trclient.com,inbound,ZZ,Unknown Region,0 -trello.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 -triongames.com,triongames.com,inbound,ZZ,Unknown Region,0.085718 -tripadvisor.com,tripadvisor.com,inbound,ZZ,Unknown Region,0.000588 -tripolis.com,tripolis.com,inbound,ZZ,Unknown Region,0 -trulia.com,trulia.com,inbound,ZZ,Unknown Region,0 -tsmmail.com,tsmmail.com,inbound,ZZ,Unknown Region,0 -tumblr.com,tumblr.com,inbound,ZZ,Unknown Region,1 -turbine.com,turbine.com,inbound,ZZ,Unknown Region,0 -turner.com,cnn.com,inbound,ZZ,Unknown Region,0 -twe-safelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -twitter.com,twitter.com,inbound,ZZ,Unknown Region,0.999969 -twoomail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 -ubivox.com,ubivox.com,inbound,ZZ,Unknown Region,0.964085 -uga.edu,outlook.com,inbound,ZZ,Unknown Region,1 -uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,ZZ,Unknown Region,0 -ulta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -ulta.com,ulta.com,inbound,ZZ,Unknown Region,0 -uniqlo-usa.com,uniqlo-usa.com,inbound,ZZ,Unknown Region,0 -unitedrepublic.org,unitedrepublic.org,inbound,ZZ,Unknown Region,1 -unosinsidersclub.com,unosinsidersclub.com,inbound,ZZ,Unknown Region,0 -urx.com.br,urx.com.br,inbound,ZZ,Unknown Region,0 -usaa.com,usaa.com,inbound,ZZ,Unknown Region,0.999997 -usahockey-email.com,usahockey-email.com,inbound,ZZ,Unknown Region,0 -usndr.com,usndr.com,inbound,ZZ,Unknown Region,1 -usx.com.br,uqx.com.br,inbound,ZZ,Unknown Region,0 -usx.com.br,usx.com.br,inbound,ZZ,Unknown Region,0 -usx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 -utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,ZZ,Unknown Region,0 -utx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 -uvarosa.com.br,uvarosa.com.br,inbound,ZZ,Unknown Region,0 -vakifbank.com.tr,vakifbank.com.tr,inbound,ZZ,Unknown Region,1 -vd.nl,emsecure.net,inbound,ZZ,Unknown Region,0 -venca.es,eccluster.com,inbound,ZZ,Unknown Region,0 -verizon.com,verizon.com,inbound,ZZ,Unknown Region,0.999816 -vfoutletvip.com,vfoutletvip.com,inbound,ZZ,Unknown Region,0 -vicinity.nl,picsrv.net,inbound,ZZ,Unknown Region,0.022107 -vietcombank.com.vn,vietcombank.com.vn,inbound,ZZ,Unknown Region,1 -viralsender.com,viralsender.com,inbound,ZZ,Unknown Region,0 -vistaprint.com,vistaprint.com,inbound,ZZ,Unknown Region,0 -vistaprint.com.au,vistaprint.com.au,inbound,ZZ,Unknown Region,0 -vitaminworld.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 -vk.com,vkontakte.ru,inbound,ZZ,Unknown Region,0 -vocus.com,vocus.com,inbound,ZZ,Unknown Region,0 -vovici.com,vovici.com,inbound,ZZ,Unknown Region,0 -vresp.com,verticalresponse.com,inbound,ZZ,Unknown Region,0 -vudu.com,vudu.com,inbound,ZZ,Unknown Region,0 -walmart.ca,walmart.ca,inbound,ZZ,Unknown Region,0 -walmart.com,walmart.com,inbound,ZZ,Unknown Region,0.315059 -warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,ZZ,Unknown Region,0 -way2sms.biz,way2sms.biz,inbound,ZZ,Unknown Region,0 -way2sms.in,way2sms.in,inbound,ZZ,Unknown Region,0 -way2smsemail.com,way2smsemail.com,inbound,ZZ,Unknown Region,0 -way2smsemails.com,way2smsemails.com,inbound,ZZ,Unknown Region,0 -way2smsmail.in,way2smsmail.in,inbound,ZZ,Unknown Region,0 -way2smsmails.com,way2smsmails.com,inbound,ZZ,Unknown Region,0 -wealthyaffiliate.com,wealthyaffiliate.com,inbound,ZZ,Unknown Region,1 -webmd.com,webmd.com,inbound,ZZ,Unknown Region,0 -wegottickets.com,wegottickets.com,inbound,ZZ,Unknown Region,0 -weheartit.com,weheartit.com,inbound,ZZ,Unknown Region,1 -wehkamp.nl,wehkamp.nl,inbound,ZZ,Unknown Region,0 -wellsfargo.com,wellsfargo.com,inbound,ZZ,Unknown Region,1 -wemakeprice.com,wemakeprice.com,inbound,ZZ,Unknown Region,0 -westwing.com.br,cust-cluster.com,inbound,ZZ,Unknown Region,0 -westwing.es,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -westwing.ru,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -wgbh.org,wgbh.org,inbound,ZZ,Unknown Region,0 -whaakky.com,whaakky.com,inbound,ZZ,Unknown Region,0 -whereareyounow.com,wayn.net,inbound,ZZ,Unknown Region,0 -whitehouse.gov,whitehouse.gov,inbound,ZZ,Unknown Region,0 -whitelabelpros.com,whitelabelpros.com,inbound,ZZ,Unknown Region,0 -wikia.com,wikia.com,inbound,ZZ,Unknown Region,0.289222 -wisdomitservices.com,infimail.com,inbound,ZZ,Unknown Region,0 -wolfmedia.us,wolfmedia.us,inbound,ZZ,Unknown Region,0 -wordfly.com,wordfly.com,inbound,ZZ,Unknown Region,0 -workhunter.net,workhunter.net,inbound,ZZ,Unknown Region,1 -worldwinner.com,worldwinner.com,inbound,ZZ,Unknown Region,0 -wowcher.co.uk,wowcher.co.uk,inbound,ZZ,Unknown Region,0 -wp.com,wordpress.com,inbound,ZZ,Unknown Region,0 -wpengine.com,wpengine.com,inbound,ZZ,Unknown Region,1 -writers-community.com,writers-community.com,inbound,ZZ,Unknown Region,0 -writersstore.com,writersstore.com,inbound,ZZ,Unknown Region,0 -wsjemail.com,wsjemail.com,inbound,ZZ,Unknown Region,0 -wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,ZZ,Unknown Region,0 -xbox.com,xbox.com,inbound,ZZ,Unknown Region,0 -xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,ZZ,Unknown Region,0 -xing.com,xing.com,inbound,ZZ,Unknown Region,0 -xxxconnect.com,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -yahoo-inc.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -yahoo.{...},yahoo.{...},inbound,ZZ,Unknown Region,0.999989 -yahoo.{...},yahoodns.net,outbound,ZZ,Unknown Region,1 -yahoogroups.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -yammer.com,yammer.com,inbound,ZZ,Unknown Region,1 -yapikredi.com.tr,yapikredi.com.tr,inbound,ZZ,Unknown Region,1 -yapstone.com,yapstone.com,inbound,ZZ,Unknown Region,0 -yesbank.in,yesbank.in,inbound,ZZ,Unknown Region,0 -yipit.com,yipit.com,inbound,ZZ,Unknown Region,1 -ymail.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -ymail.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -youravon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 -yournewsletters.net,everydayhealth.com,inbound,ZZ,Unknown Region,0 -youversion.com,youversion.com,inbound,ZZ,Unknown Region,1 -zappos.com,zappos.com,inbound,ZZ,Unknown Region,0.625377 -zattoo.com,sendnode.com,inbound,ZZ,Unknown Region,0 -zelonews.com.br,zelonews.com.br,inbound,ZZ,Unknown Region,0 -zendesk.com,zdsys.com,inbound,ZZ,Unknown Region,1 -zibmail.info,zibmail.info,inbound,ZZ,Unknown Region,0 -zillow.com,zillow.com,inbound,ZZ,Unknown Region,2.82543102656668e-07 -zinio.net,zinio.com,inbound,ZZ,Unknown Region,1 -zipalerts.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -zipalerts.com,zipalerts.com,inbound,ZZ,Unknown Region,1 -zlavadna.sk,zlavadna.sk,inbound,ZZ,Unknown Region,0 -zoom.com.br,zoom.com.br,inbound,ZZ,Unknown Region,0 -zoominternet.net,synacor.com,inbound,ZZ,Unknown Region,0 -zoosk.com,zoosk.com,inbound,ZZ,Unknown Region,0 -zorpia.com,zorpia.com,inbound,ZZ,Unknown Region,0.520147 -zovifashion.com,eccluster.com,inbound,ZZ,Unknown Region,0 -zyngamail.com,zyngamail.com,inbound,ZZ,Unknown Region,0 \ No newline at end of file diff --git a/tools/CheckSTARTTLS.py b/tools/CheckSTARTTLS.py deleted file mode 100755 index ef0bf2e5c..000000000 --- a/tools/CheckSTARTTLS.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import errno -import smtplib -import socket -import subprocess -import re -import json -import collections - -import dns.resolver -from M2Crypto import X509 -from publicsuffix import PublicSuffixList - -public_suffix_list = PublicSuffixList() -CERTS_OBSERVED = 'certs-observed' - -def mkdirp(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: raise - -def extract_names(pem): - """Return a set of DNS subject names from PEM-encoded leaf cert.""" - leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) - - subj = leaf.get_subject() - # Certs have a "subject" identified by a Distingushed Name (DN). - # Host certs should also have a Common Name (CN) with a DNS name. - common_names = subj.get_entries_by_nid(subj.nid['CN']) - common_names = [name.get_data().as_text() for name in common_names] - try: - # The SAN extension allows one cert to cover multiple domains - # and permits DNS wildcards. - # http://www.digicert.com/subject-alternative-name.htm - # The field is a comma delimited list, e.g.: - # >>> twitter_cert.get_ext('subjectAltName').get_value() - # 'DNS:www.twitter.com, DNS:twitter.com' - alt_names = leaf.get_ext('subjectAltName').get_value() - alt_names = alt_names.split(', ') - alt_names = [name.partition(':') for name in alt_names] - alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] - except: - alt_names = [] - return set(common_names + alt_names) - -def tls_connect(mx_host, mail_domain): - """Attempt a STARTTLS connection with openssl and save the output.""" - if supports_starttls(mx_host): - # smtplib doesn't let us access certificate information, - # so shell out to openssl. - try: - output = subprocess.check_output( - """openssl s_client \ - -starttls smtp -connect %s:25 -showcerts /dev/null - """ % mx_host, shell = True) - except subprocess.CalledProcessError: - print "Failed s_client" - return - - # Save a copy of the certificate for later analysis - with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f: - f.write(output) - -def valid_cert(filename): - """Return true if the certificate is valid. - - Note: CApath must have hashed symlinks to the trust roots. - TODO: Include the -attime flag based on file modification time.""" - - if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: - return False - try: - # The file contains both the leaf cert and any intermediates, so we pass it - # as both the cert to validate and as the "untrusted" chain. - output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ - -untrusted "%s" \ - "%s" - """ % (filename, filename), shell = True) - return True - except subprocess.CalledProcessError: - return False - -def check_certs(mail_domain): - """ - Return "" if any certs for any mx domains pointed to by mail_domain - were invalid, and a public suffix for one if they were all valid - """ - dir = os.path.join(CERTS_OBSERVED, mail_domain) - if not os.path.exists(dir): - collect(mail_domain) - names = set() - for mx_hostname in os.listdir(dir): - filename = os.path.join(dir, mx_hostname) - if not valid_cert(filename): - return "" - else: - new_names = extract_names_from_openssl_output(filename) - new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names) - names.update(new_names) - if len(names) >= 1: - # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. - return names.pop() - else: - return "" - -def common_suffix(hosts): - num_components = min(len(h.split(".")) for h in hosts) - longest_suffix = "" - for i in range(1, num_components + 1): - suffixes = set(".".join(h.split(".")[-i:]) for h in hosts) - if len(suffixes) == 1: - longest_suffix = suffixes.pop() - else: - return longest_suffix - return longest_suffix - -def extract_names_from_openssl_output(certificates_file): - openssl_output = open(certificates_file, "r").read() - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", openssl_output, flags = re.DOTALL) - return extract_names(cert[0]) - -def supports_starttls(mx_host): - try: - smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) - smtpserver.ehlo() - smtpserver.starttls() - return True - print "Success: %s" % mx_host - except socket.error as e: - print "Connection to %s failed: %s" % (mx_host, e.strerror) - return False - except smtplib.SMTPException, e: - # In order to talk to some hosts, you need to run this from a host that has a - # reverse DNS entry. AWS instances all have reverse DNS, as an example. - if e[0] == 554: - print e[1] - else: - print "No STARTTLS support on %s" % mx_host, e[0] - return False - -def min_tls_version(mail_domain): - protocols = [] - for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)): - filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname) - contents = open(filename).read() - protocol = re.findall("Protocol : (.*)", contents)[0] - protocols.append(protocol) - return min(protocols) - -def collect(mail_domain): - """ - Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS. - Store the output in a directory with the same name as mail_domain to make - subsequent analysis faster. - """ - print "Checking domain %s" % mail_domain - mkdirp(os.path.join(CERTS_OBSERVED, mail_domain)) - answers = dns.resolver.query(mail_domain, 'MX') - for rdata in answers: - mx_host = str(rdata.exchange).rstrip(".") - tls_connect(mx_host, mail_domain) - -if __name__ == '__main__': - """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) < 2: - print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") - - config = collections.defaultdict(dict) - - for input in sys.argv[1:]: - for domain in open(input).readlines(): - domain = domain.strip() - suffix = check_certs(domain) - if suffix != "": - min_version = min_tls_version(domain) - suffix_match = "." + suffix - config["acceptable-mxs"][domain] = { - "accept-mx-domains": [suffix_match] - } - config["tls-policies"][suffix_match] = { - "require-tls": True, - "min-tls-version": min_version - } - - print json.dumps(config, indent=2, sort_keys=True) diff --git a/tools/ProcessGoogleSTARTTLSDomains.py b/tools/ProcessGoogleSTARTTLSDomains.py deleted file mode 100755 index 815ec5d4e..000000000 --- a/tools/ProcessGoogleSTARTTLSDomains.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Process Google's TLS delivery data from -https://www.google.com/transparencyreport/saferemail/data/?hl=en -to look for outbound domains that can negotiate an encrypted -connection >99% of the time. - -Usage: - ./ProcessGoogleSTARTTLSDomains.py google-starttls-domains.csv -""" -import csv -import codecs -import sys -from collections import defaultdict - -csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') -d = defaultdict(set) -# Google's report doesn't include gmail.com because it's local delivery, but we -# know they support STARTTLS, so manually include them. -d["gmail.com"] = set([1]) -for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader: - if direction == "outbound": - # Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}. - # We're tryingto get a solid list of the relevant TLDs, but in the meantime - # just use .com. - address_suffix = address_suffix.replace("{...}", "com") - try: - d[address_suffix].add(float(fraction_encrypted)) - except ValueError: - pass - -for address_suffix, fraction_encrypted in d.iteritems(): - if min(fraction_encrypted) >= 0.99: - print address_suffix diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh deleted file mode 100755 index 82622edea..000000000 --- a/vagrant-bootstrap.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -export DEBIAN_FRONTEND=noninteractive - -apt-get update -q -apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - postfix dnsmasq mutt vim - -# Provide hostnames so the boxes can talk to each other. DNSMasq will also serve -# results to each box based on these contents. -cat >> /etc/hosts < /etc/dnsmasq.conf -# Not sure why restart is necessary, but otherwise dnsmasq doesn't use -# /etc/hosts to answer queries. -/etc/init.d/dnsmasq restart - -if [ "`hostname`" = "sender" ]; then - (while sleep 10; do - echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com - done) & - #ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy -fi - -#ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf -#ln -sf "/vagrant/certificates" /etc/certificates -postfix reload diff --git a/vagrant-shared/certificates/ca.crt b/vagrant-shared/certificates/ca.crt deleted file mode 100644 index dce966d32..000000000 --- a/vagrant-shared/certificates/ca.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDmzCCAoOgAwIBAgIJAIheS+k2UDobMA0GCSqGSIb3DQEBBQUAMGQxCzAJBgNV -BAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxDDAKBgNVBAoMA0VGRjEt -MCsGA1UECwwkU1RBUlRUTFMgRXZlcnl3aGVyZSB0ZXN0IGNlcnRpZmljYXRlMB4X -DTE0MDYxMDE4MDg0OVoXDTE5MDYxMDE4MDg1MFowZDELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRT -VEFSVFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDDYmUQUjaX6L3n9fNks2yQUFhKUBR1NYenx3w2 -DnaZiwpLzI3igJPMQfBdyJGLM1jXZZcpvpgt6yN4OMOLNS2QKBY20gDoQIh0Jmaj -KCoXUbX30H1FfTn+pyU02UpuFsFN3TAk5bQ/BQUYOlMouCowyZ25mnEzzHLeRHKH -Gi2uCH59T53rcgDwjq88pKMVUlndixkOKpeXZkTL++Edg0b5SUpRuzMs6kFmwLoQ -x4xG5lgaAHyu2/9KXhcqielE95s5FNGfi9U2q3nkmpa4dM266DWfkibfvCBP3hiO -Ks+IN+Hyohi2SY7NoDN6hMKcuUNlAK/xD+foA4Ck3R45HII3AgMBAAGjUDBOMB0G -A1UdDgQWBBRPlrBeFMPb27oWESmXykOW1XWLLDAfBgNVHSMEGDAWgBRPlrBeFMPb -27oWESmXykOW1XWLLDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBY -lBQEuRKo927jJFzgvdTKJGIAzxBnC5vg+qKeZkduwxeVEwB13HPP1syvdpAK5dkl -JGuHCF/Yc39HX1OZv7huVRMnyrSKpMt25uqUHirH6Db17HRCS4ZA6rARJfxS6RWu -J31lfXvGRr0hI4yw3XpKGM/c8Gkzji6PsYn4TCPnXjJbRj1GjHFaAuIeyoO0zQjo -OuHnzxs4bIDaV32NHNetuDKSO1GNbenxRiiN1HvQ1vfhzpqerRRaPCHrW4eUcynQ -AAeuC+Ek925t9mH7Ni/kZ0eN7XUwvg3c2lm+LeV+ICP+NWv9r92kfXDZNMhlhNsf -Y8+m1Y9WL3CLj99voNQS ------END CERTIFICATE----- diff --git a/vagrant-shared/certificates/ca.key b/vagrant-shared/certificates/ca.key deleted file mode 100644 index 3ebf4159e..000000000 --- a/vagrant-shared/certificates/ca.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAw2JlEFI2l+i95/XzZLNskFBYSlAUdTWHp8d8Ng52mYsKS8yN -4oCTzEHwXciRizNY12WXKb6YLesjeDjDizUtkCgWNtIA6ECIdCZmoygqF1G199B9 -RX05/qclNNlKbhbBTd0wJOW0PwUFGDpTKLgqMMmduZpxM8xy3kRyhxotrgh+fU+d -63IA8I6vPKSjFVJZ3YsZDiqXl2ZEy/vhHYNG+UlKUbszLOpBZsC6EMeMRuZYGgB8 -rtv/Sl4XKonpRPebORTRn4vVNqt55JqWuHTNuug1n5Im37wgT94YjirPiDfh8qIY -tkmOzaAzeoTCnLlDZQCv8Q/n6AOApN0eORyCNwIDAQABAoIBAQCwNXASDSNBU1zZ -8v3kZtDVQjCuLJSWtIU4cnd6RQb/KN9LRxr7GJyyzREbc4SXduJ7uBphQov6daMS -jJcGWBpUdWK7ZB//Vhv6LJvKL7HuP/oNmhEwd2SzXkj25bTznkANmhsOW794Sm2y -0P8orRcX0u0Vc8z+Ozepby+e2qQx29FXznberRv2rXmMeeQqF+sc2MBu34SlcLnK -KVSe7SmDc+DhWJ3XiPoEpiOTbv4EynndXC85owdse/eN/EahFtrfzqm6jDWVga8A -xA/7RD/Urc2L2IsZOB92xOk/tGs5Df8ZrHavzbSo+i0pzYAKvIxr+G2Krl1Tl/Lh -IXwjWPLZAoGBAOsbdxeZUHH6pk497ICIcmW/NDhAY/mSA6vBWZWLcYF7OU/wMzZ6 -Wlx9v6oz8tBTzQj/Xkpv0ZLIeQqGTuSQzQ7EcrOiTz4PIpM5y/nf/S0oSXwJ7mJw -Sx8w3XHqHEPCT9G83o7EV7xHuRy8/zQzJ9dkFxznA9sRcO7RhE/tsrMNAoGBANS/ -Ql140oyDqyABIlxswyvfJ8Ll4kmb52yNGaCJPSfrAVmSoN15AkbDkpVKFb6hsL6u -xqVPyeVxard4twddrV3GSvTibkGw4fZ8x0FDgDGvCgJ36e4NHb7jQVwwGfNXKMAP -qvYtnE28eAM+zrDHryEhgp4k59zApphr+IU7JQlTAoGAc1h5ODm+rvzTBMX6tyC6 -R1Lkcsicg//wDx8ALY9JM8ZZ2u80oQCsPn5vPzjXYwAKMuTexNRRVJtITzKPmDG2 -eQ1GXP0/tWnFg8eyXDhZRQNj8hgJPYBsSrQ1oMLD9TZq5LKt2gtYJAZoOkI7Tsfe -Px1a/ZIVYTAQYQqnyHMM3i0CgYAQNirWeJiCwJ3PqIZ3yInu0+hxv5bIySqPaQkk -5JBWdF/79WJwvgHgZpLK8YRKrIONZEAa5MObylK5fGdmFktZs/yOQJrqQpJVeBiu -7nfcUVxP59dZnoI/w419euTfWCrwx8DdVYhtnAkBJk4VxoGf4q/TYTiR59RKFSAw -9trRpQKBgDjFQ9mXJUUKoc61P3PIrSYOlnx89c0sznrwm2eeLLAwr/VQ+vNDGRkC -6kXbVLuTxRaFnLSZTlUd0nSXPbJEM5GII48zbGZ7t/ysPtA9390xk1efTO/ryf/j -pn4FBvpgXaneRs4ExyknLgFfJRUEu17fRbyRPZr0bUlTkKg/ZS5M ------END RSA PRIVATE KEY----- diff --git a/vagrant-shared/certificates/certificates b/vagrant-shared/certificates/certificates deleted file mode 120000 index d162b42af..000000000 --- a/vagrant-shared/certificates/certificates +++ /dev/null @@ -1 +0,0 @@ -/vagrant/certificates \ No newline at end of file diff --git a/vagrant-shared/certificates/valid.crt b/vagrant-shared/certificates/valid.crt deleted file mode 100644 index f5fe4c213..000000000 --- a/vagrant-shared/certificates/valid.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDbTCCAlUCAQEwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCVVMxCzAJBgNV -BAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRTVEFS -VFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwHhcNMTQwNjEwMTgxMTQ3 -WhcNMTkwNjEwMTgxMTQ3WjCBlDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQsw -CQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMTQwMgYDVQQLDCtTVEFSVFRMUyBFdmVy -eXdoZXJlIHRlc3QgY2VydGlmaWNhdGUgKGxlYWYpMScwJQYDVQQDDB5teC52YWxp -ZC1leGFtcGxlLXJlY2lwaWVudC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQDZ2NdAwI0LUmXrV7bItKhP8NRzypkYHgApXaoZ6iB6TJnYb1FZyv2Y -wyBdsQQ8zcIgKRyl0rKvNPgHt3riM7nSoB16II6fIQDuR2UnXkhtDOfUd7Ye0wKX -l3A+qoEge42/TfRvzPC6IMa1KH7+dwayprIjLxFUzfMl6GqA/5auLZZtSsV6Pix3 -jXZYFPUHPoBJEyNo/bizcdvSZS/7Kwzfc64l7JLA/OGtRbQpcMAOpRRXCx32nt3N -2L1OXz8Q4It+J20oN8Vfin1a7GbAGdktRbz2bV8bmb0ux1NOnddigPUHRRYMIaKN -W7Nrxp2YQpqmsqBmVEVPA903yRc0ZvuRAgMBAAEwDQYJKoZIhvcNAQEFBQADggEB -AMJ3neFM+to/tFhTiAfUnIrQOKyfk+zYN8gC99HD2SUVbu+Cu1qCNJWxbE3bdxxX -y960yX+Or8E4nG9sWbFbzlGEzz0F8DAPEh4UCtb/5MRkhW158Y9LtbheQAQpZolQ -Sma6lnngCmSDr/OpDW34oM8S7+3VOawYv1e1ruCMqizBwilgcsGq2hY1hWca6LMP -X2FHQ+m4mYBV9wJ9WuKMs9uADz9cfMYuA/wMzNeMZyM82w+nSQGZ+mX7YPu+WBJM -k9OcFrUWxrC/uOQfsyqdjLvFjg4/b49jnusn9mZ/KbtLVmkz5P+5kwSn4kcd5Rlw -LLmEGiXiyEjo8UmEVyumrIQ= ------END CERTIFICATE----- diff --git a/vagrant-shared/certificates/valid.csr b/vagrant-shared/certificates/valid.csr deleted file mode 100644 index a58aab677..000000000 --- a/vagrant-shared/certificates/valid.csr +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIC2jCCAcICAQAwgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UE -BwwCU0YxDDAKBgNVBAoMA0VGRjE0MDIGA1UECwwrU1RBUlRUTFMgRXZlcnl3aGVy -ZSB0ZXN0IGNlcnRpZmljYXRlIChsZWFmKTEnMCUGA1UEAwwebXgudmFsaWQtZXhh -bXBsZS1yZWNpcGllbnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9RWcr9mMMgXbEE -PM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2HtMCl5dwPqqB -IHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrFej4sd412WBT1 -Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd9p7dzdi9Tl8/ -EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UWDCGijVuza8ad -mEKaprKgZlRFTwPdN8kXNGb7kQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAAhe -pMLNDAaA9fXZ/yqW4ov1pBKYZL1R1RX+YgZqUPoAlrTCiJ0UGIuPcTDGJDMdpZ7x -9gSqwsXrvzXafuHI4GuFjnbY5SIv8zvc+nMXib7IMlyMUcuSBZP8W0sl3ZGWnnpk -legC10c3+I9TfQ7Tl0mpdyf/6yLhM1plxLcIy5bguLJjbBK9JhKNfc84rivqrxUI -+cUqWU13WjMzWdKS6rK5m/Bfleg+jyZ11xYY0QfwNGwuPjfiEBjCs2iqJvdLFRem -FoDq3XBsrH5XohSwWZ6UrZY7wARkmHsYJeYTHulb3MkDPCQKlbU+2SMIpbNk43+/ -fN/ctxWXg3vd/q6eJiE= ------END CERTIFICATE REQUEST----- diff --git a/vagrant-shared/certificates/valid.key b/vagrant-shared/certificates/valid.key deleted file mode 100644 index b5e8371cd..000000000 --- a/vagrant-shared/certificates/valid.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9R -Wcr9mMMgXbEEPM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2 -HtMCl5dwPqqBIHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrF -ej4sd412WBT1Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd -9p7dzdi9Tl8/EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UW -DCGijVuza8admEKaprKgZlRFTwPdN8kXNGb7kQIDAQABAoIBAHAZgUq0yt+UmxWr -oUdOj33zc5/SFU2vwm2G4U1MiUHlwRT602Xdavn9Dt6nhIK1bruV7EP4VDKMk0WF -SRq1e13DPuflcP65wPzciFTl02cqSPGwWGssMh1HtF7K5n+MlLhoqOwPDaD51MbL -++192lh8Jxar1cNJ52EOZB/VZfhiUZ88JAITWam6clId8KS4cy7SvQ9haptcPlfl -wIeCti2eI4nvC0IR0SCEqWAax6XqypA5k2fjQJklcnBK1R+H1muc00J5lMIz4U8q -qFb5RgxIDMcEeQZWmzfBjU5hO0q/51QyWFFFuBL00MkT4djCHjO/IPYc6DQoAoZR -TbMCWwECgYEA+i3W44tSY96Fdn1gDyYhDor9GPN0XwmRBHpl44TxBLG48I6cGLYI -l0InEh7pu7/0fw+cWylx2OLrl2HeGlWilEOpP2ucJcswcNibATtgSdnu7PjC4eAW -46a5+2hbFk/NONU5VTGClJHCvKzYPWMrAIkRT//WdkIVs23i7bRcOjkCgYEA3upr -cnMhnwVmZrGAIm+DtiDJCc4uVXeqLD2j9csCVNCw/CQAjJ3/5VlQDSjzerjijw/V -uoA+Su8xMoMCg6mAihqdxBMrbwdlh7vje23RaxbeQkIY5JIVAsdDr+UtwofaX3Zx -j3RW5jNeouVlvtExyKZZhyMwuh8d59m4V65/rBkCgYBH+f40EvZOQ0v0jhef5CFo -lLZCgnB9kzwEpM5BihLpfdQuaWkhduW71s102i321UAbejtKwv69HnQXZpHG09Jl -g53i4CvZd77lCHx3+0Q1mxyxUtSGtbkAIAyr9xcVsTni2v2WtBrUcacsLzI7XxeV -HNo9QObLuTGTIM9EAjryiQKBgQCIZEBX56/jn6c3IFX5O+gH8OlxEXFyI+TAavq+ -Mnd7s7EGpXSclTP0fYAofSz0otkklZi9Iyh6Kv4cHOLV8klOtthfFyeVKJ5rvX+D -jv76miRlwBGBEQzABXIZ1oz4IK1xiYQUNSfSdA3sd5WYemEOlxHiSJrQ1qcyrBlJ -tOAzSQKBgQDjLQIZ0rmSAMUCGi7WlDle/pnf6sUEFUPWUqzHOJbJk/CZIIf1IT6o -Evep9eeP6QjiupusY/uSlgjjjj7yxZU5q3VV/0cQl6QhwxG9wqpa8x32wwaM1WBZ -rJgsuSjcEoluAqfX/bqRtumNtQ213DSADzSCpOetlJb+ARANk7YULg== ------END RSA PRIVATE KEY----- diff --git a/vagrant-shared/postfix-config-sender-tls_policy b/vagrant-shared/postfix-config-sender-tls_policy deleted file mode 100644 index f4f1e80c5..000000000 --- a/vagrant-shared/postfix-config-sender-tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com secure match=valid-example-recipient.com:.valid-example-recipient.com diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf deleted file mode 100644 index b9f265058..000000000 --- a/vagrant-shared/postfix-config-sender.cf +++ /dev/null @@ -1,39 +0,0 @@ -# See /usr/share/postfix/main.cf.dist for a commented, more complete version - - -# Debian specific: Specifying a file name will cause the first -# line of that file to be used as the name. The Debian default -# is /etc/mailname. -#myorigin = /etc/mailname - -smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) -biff = no - -# appending .domain is the MUA's job. -append_dot_mydomain = no - -# Uncomment the next line to generate "delayed mail" warnings -#delay_warning_time = 4h - -readme_directory = no - -# TLS parameters -smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key -smtpd_use_tls=yes -smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache - -# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for -# information on enabling SSL in the smtp client. - -myhostname = sender.example.com -alias_maps = hash:/etc/aliases -alias_database = hash:/etc/aliases -myorigin = /etc/mailname -mydestination = sender.example.com, localhost.example.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 -mailbox_size_limit = 0 -recipient_delimiter = + -inet_interfaces = all diff --git a/vagrant-shared/postfix-config-valid-example-recipient.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf deleted file mode 100644 index 1d08a20cd..000000000 --- a/vagrant-shared/postfix-config-valid-example-recipient.cf +++ /dev/null @@ -1,46 +0,0 @@ -# See /usr/share/postfix/main.cf.dist for a commented, more complete version - - -# Debian specific: Specifying a file name will cause the first -# line of that file to be used as the name. The Debian default -# is /etc/mailname. -#myorigin = /etc/mailname - -smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) -biff = no - -# appending .domain is the MUA's job. -append_dot_mydomain = no - -# Uncomment the next line to generate "delayed mail" warnings -#delay_warning_time = 4h - -readme_directory = no - -# TLS parameters -smtpd_use_tls=yes -smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache - -# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for -# information on enabling SSL in the smtp client. - -myhostname = valid-example-recipient.com -alias_maps = hash:/etc/aliases -alias_database = hash:/etc/aliases -myorigin = /etc/mailname -mydestination = valid-example-recipient.com, localhost.example.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 -mailbox_size_limit = 0 -recipient_delimiter = + -inet_interfaces = all - -# STARTLS Everywhere recommended best-practice settings -smtpd_tls_session_cache_timeout = 3600s -smtpd_tls_received_header = yes - -#STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy -smtpd_tls_cert_file=/etc/certificates/valid.crt -smtpd_tls_key_file=/etc/certificates/valid.key From 19a4e6079ea70b37cbb2e6451ff74d69a1f6bae7 Mon Sep 17 00:00:00 2001 From: mvi Date: Thu, 26 Oct 2017 19:09:06 +0200 Subject: [PATCH 204/631] [#5155] - replaces instances of isinstance(x, str) with isinstance(x, six.string_types) --- certbot/cli.py | 10 +++++----- certbot/plugins/null_test.py | 3 ++- certbot/plugins/webroot_test.py | 2 +- certbot/renewal.py | 4 ++-- certbot/tests/util_test.py | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 880ffd543..3a12f86c7 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -142,14 +142,14 @@ def report_config_interaction(modified, modifiers): between config options. :param modified: config options that can be modified by modifiers - :type modified: iterable or str + :type modified: iterable or str (string_types) :param modifiers: config options that modify modified - :type modifiers: iterable or str + :type modifiers: iterable or str (string_types) """ - if isinstance(modified, str): + if isinstance(modified, six.string_types): modified = (modified,) - if isinstance(modifiers, str): + if isinstance(modifiers, six.string_types): modifiers = (modifiers,) for var in modified: @@ -477,7 +477,7 @@ class HelpfulArgumentParser(object): if isinstance(help1, bool) and isinstance(help2, bool): self.help_arg = help1 or help2 else: - self.help_arg = help1 if isinstance(help1, str) else help2 + self.help_arg = help1 if isinstance(help1, six.string_types) else help2 short_usage = self._usage_string(plugins, self.help_arg) diff --git a/certbot/plugins/null_test.py b/certbot/plugins/null_test.py index 0d04a2bc5..d5de33fb3 100644 --- a/certbot/plugins/null_test.py +++ b/certbot/plugins/null_test.py @@ -1,5 +1,6 @@ """Tests for certbot.plugins.null.""" import unittest +import six import mock @@ -12,7 +13,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), str)) + self.assertTrue(isinstance(self.installer.more_info(), six.string_types)) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 5a311716e..92160bdfa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -50,7 +50,7 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, str)) + self.assertTrue(isinstance(more_info, six.string_types)) self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): diff --git a/certbot/renewal.py b/certbot/renewal.py index 2c41d2f9e..7d0240f73 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -108,7 +108,7 @@ def _restore_webroot_config(config, renewalparams): elif "webroot_path" in renewalparams: logger.debug("Ancient renewal conf file without webroot-map, restoring webroot-path") wp = renewalparams["webroot_path"] - if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string + if isinstance(wp, six.string_types): # prior to 0.1.0, webroot_path was a string wp = [wp] config.webroot_path = wp @@ -194,7 +194,7 @@ def _restore_pref_challs(unused_name, value): # If pref_challs has only one element, configobj saves the value # with a trailing comma so it's parsed as a list. If this comma is # removed by the user, the value is parsed as a str. - value = [value] if isinstance(value, str) else value + value = [value] if isinstance(value, six.string_types) else value return cli.parse_preferred_challenges(value) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 7e320012a..50d323ffd 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -258,7 +258,7 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for _ in six.moves.range(10): f, name = self._call("wow") self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, str)) + self.assertTrue(isinstance(name, six.string_types)) self.assertTrue("wow-0009.conf" in name) @mock.patch("certbot.util.os.fdopen") From f962b5c83da13030ff781ba515e0e9bd46c7e5ac Mon Sep 17 00:00:00 2001 From: yomna Date: Tue, 31 Oct 2017 12:52:40 -0700 Subject: [PATCH 205/631] Forcing pip to use https on older docker images (#5214) --- letsencrypt-auto-source/Dockerfile.precise | 2 ++ letsencrypt-auto-source/Dockerfile.wheezy | 2 ++ 2 files changed, 4 insertions(+) diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.precise index c8b593774..5ee32c7cc 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.precise @@ -10,6 +10,8 @@ RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo - RUN apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean + +ENV PIP_INDEX_URL https://pypi.python.org/simple RUN pip install nose # Let that user sudo: diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.wheezy index f86795e08..acdb791a4 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.wheezy @@ -10,6 +10,8 @@ RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo - RUN apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean + +ENV PIP_INDEX_URL https://pypi.python.org/simple RUN pip install nose # Let that user sudo: From 68e37b03c821560e7f316d260f8da97ef3e2087c Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Nov 2017 02:41:32 +0200 Subject: [PATCH 206/631] Nginx IPv6 support (#5178) * Nginx IPv6 support * Test and lint fixes * IPv6 tests to Nginx plugin * Make ipv6_info() port aware * Named tuple values for readability * Lint fix * Requested changes --- certbot-nginx/certbot_nginx/configurator.py | 56 ++++++++++++++++++- certbot-nginx/certbot_nginx/obj.py | 53 ++++++++++++++---- .../certbot_nginx/tests/configurator_test.py | 24 ++++++-- .../certbot_nginx/tests/parser_test.py | 34 +++++++---- .../testdata/etc_nginx/sites-enabled/ipv6.com | 5 ++ .../etc_nginx/sites-enabled/ipv6ssl.com | 5 ++ .../certbot_nginx/tests/tls_sni_01_test.py | 8 +-- certbot-nginx/certbot_nginx/tls_sni_01.py | 23 +++++++- certbot/plugins/common.py | 2 +- 9 files changed, 171 insertions(+), 39 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 76a7e3eef..98990664f 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -257,6 +257,31 @@ class NginxConfigurator(common.Installer): return vhost + + def ipv6_info(self, port): + """Returns tuple of booleans (ipv6_active, ipv6only_present) + ipv6_active is true if any server block listens ipv6 address in any port + + ipv6only_present is true if ipv6only=on option exists in any server + block ipv6 listen directive for the specified port. + + :param str port: Port to check ipv6only=on directive for + + :returns: Tuple containing information if IPv6 is enabled in the global + configuration, and existence of ipv6only directive for specified port + :rtype: tuple of type (bool, bool) + """ + vhosts = self.parser.get_vhosts() + ipv6_active = False + ipv6only_present = False + for vh in vhosts: + for addr in vh.addrs: + if addr.ipv6: + ipv6_active = True + if addr.ipv6only and addr.get_port() == port: + ipv6only_present = True + return (ipv6_active, ipv6only_present) + def _vhost_from_duplicated_default(self, domain): if self.new_vhost is None: default_vhost = self._get_default_vhost() @@ -449,9 +474,12 @@ class NginxConfigurator(common.Installer): all_names.add(host) elif not common.private_ips_regex.match(host): # If it isn't a private IP, do a reverse DNS lookup - # TODO: IPv6 support try: - socket.inet_aton(host) + if addr.ipv6: + host = addr.get_ipv6_exploded() + socket.inet_pton(socket.AF_INET6, host) + else: + socket.inet_pton(socket.AF_INET, host) all_names.add(socket.gethostbyaddr(host)[0]) except (socket.error, socket.herror, socket.timeout): continue @@ -487,16 +515,38 @@ class NginxConfigurator(common.Installer): :type vhost: :class:`~certbot_nginx.obj.VirtualHost` """ + ipv6info = self.ipv6_info(self.config.tls_sni_01_port) + ipv6_block = [''] + ipv4_block = [''] + # If the vhost was implicitly listening on the default Nginx port, # have it continue to do so. if len(vhost.addrs) == 0: listen_block = [['\n ', 'listen', ' ', self.DEFAULT_LISTEN_PORT]] self.parser.add_server_directives(vhost, listen_block, replace=False) + if vhost.ipv6_enabled(): + ipv6_block = ['\n ', + 'listen', + ' ', + '[::]:{0} ssl'.format(self.config.tls_sni_01_port)] + if not ipv6info[1]: + # ipv6only=on is absent in global config + ipv6_block.append(' ') + ipv6_block.append('ipv6only=on') + + if vhost.ipv4_enabled(): + ipv4_block = ['\n ', + 'listen', + ' ', + '{0} ssl'.format(self.config.tls_sni_01_port)] + + snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = ([ - ['\n ', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)], + ipv6_block, + ipv4_block, ['\n ', 'ssl_certificate', ' ', snakeoil_cert], ['\n ', 'ssl_certificate_key', ' ', snakeoil_key], ['\n ', 'include', ' ', self.mod_ssl_conf], diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index d7604bdf9..5816c5571 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -34,10 +34,13 @@ class Addr(common.Addr): UNSPECIFIED_IPV4_ADDRESSES = ('', '*', '0.0.0.0') CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] - def __init__(self, host, port, ssl, default): + def __init__(self, host, port, ssl, default, ipv6, ipv6only): + # pylint: disable=too-many-arguments super(Addr, self).__init__((host, port)) self.ssl = ssl self.default = default + self.ipv6 = ipv6 + self.ipv6only = ipv6only self.unspecified_address = host in self.UNSPECIFIED_IPV4_ADDRESSES @classmethod @@ -46,6 +49,8 @@ class Addr(common.Addr): parts = str_addr.split(' ') ssl = False default = False + ipv6 = False + ipv6only = False host = '' port = '' @@ -56,15 +61,25 @@ class Addr(common.Addr): if addr.startswith('unix:'): return None - tup = addr.partition(':') - if re.match(r'^\d+$', tup[0]): - # This is a bare port, not a hostname. E.g. listen 80 - host = '' - port = tup[0] + # IPv6 check + ipv6_match = re.match(r'\[.*\]', addr) + if ipv6_match: + ipv6 = True + # IPv6 handling + host = ipv6_match.group() + # The rest of the addr string will be the port, if any + port = addr[ipv6_match.end()+1:] else: - # This is a host-port tuple. E.g. listen 127.0.0.1:* - host = tup[0] - port = tup[2] + # IPv4 handling + tup = addr.partition(':') + if re.match(r'^\d+$', tup[0]): + # This is a bare port, not a hostname. E.g. listen 80 + host = '' + port = tup[0] + else: + # This is a host-port tuple. E.g. listen 127.0.0.1:* + host = tup[0] + port = tup[2] # The rest of the parts are options; we only care about ssl and default while len(parts) > 0: @@ -73,8 +88,10 @@ class Addr(common.Addr): ssl = True elif nextpart == 'default_server': default = True + elif nextpart == "ipv6only=on": + ipv6only = True - return cls(host, port, ssl, default) + return cls(host, port, ssl, default, ipv6, ipv6only) def to_string(self, include_default=True): """Return string representation of Addr""" @@ -114,8 +131,6 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) - # Nginx plugin currently doesn't support IPv6 but this will - # future-proof it return super(Addr, self).__eq__(other) def __eq__(self, other): @@ -195,6 +210,20 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return True return False + def ipv6_enabled(self): + """Return true if one or more of the listen directives in vhost supports + IPv6""" + for a in self.addrs: + if a.ipv6: + return True + + def ipv4_enabled(self): + """Return true if one or more of the listen directives in vhost are IPv4 + only""" + for a in self.addrs: + if not a.ipv6: + return True + def _find_directive(directives, directive_name): """Find a directive of type directive_name in directives """ diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 0028986fa..aa94abecb 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -46,7 +46,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(8, len(self.config.parser.parsed)) + self.assertEqual(10, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -90,7 +90,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com"])) + "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'staple-ocsp'], @@ -132,6 +132,7 @@ class NginxConfiguratorTest(util.NginxTest): server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) foo_conf = set(['*.www.foo.com', '*.www.example.com']) + ipv6_conf = set(['ipv6.com']) results = {'localhost': localhost_conf, 'alias': server_conf, @@ -140,7 +141,8 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': example_conf, 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, - 'www.bar.co.uk': localhost_conf} + 'www.bar.co.uk': localhost_conf, + 'ipv6.com': ipv6_conf} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -149,7 +151,8 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': "etc_nginx/sites-enabled/example.com", 'test.www.example.com': "etc_nginx/foo.conf", 'abc.www.foo.com': "etc_nginx/foo.conf", - 'www.bar.co.uk': "etc_nginx/nginx.conf"} + 'www.bar.co.uk': "etc_nginx/nginx.conf", + 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] @@ -160,11 +163,24 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(results[name], vhost.names) self.assertEqual(conf_path[name], path) + # IPv6 specific checks + if name == "ipv6.com": + self.assertTrue(vhost.ipv6_enabled()) + # Make sure that we have SSL enabled also for IPv6 addr + self.assertTrue( + any([True for x in vhost.addrs if x.ssl and x.ipv6])) for name in bad_results: self.assertRaises(errors.MisconfigurationError, self.config.choose_vhost, name) + def test_ipv6only(self): + # ipv6_info: (ipv6_active, ipv6only_present) + self.assertEquals((True, False), self.config.ipv6_info("80")) + # Port 443 has ipv6only=on because of ipv6ssl.com vhost + self.assertEquals((True, True), self.config.ipv6_info("443")) + + def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index ec0cfd288..ca5de7ff6 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -50,7 +50,9 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods 'sites-enabled/example.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', - 'sites-enabled/globalssl.com']]), + 'sites-enabled/globalssl.com', + 'sites-enabled/ipv6.com', + 'sites-enabled/ipv6ssl.com']]), set(nparser.parsed.keys())) self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], nparser.parsed[nparser.abs_path('server.conf')]) @@ -74,7 +76,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(5, len( + self.assertEqual(7, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -110,7 +112,8 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), - [obj.Addr('4.8.2.6', '57', True, False)], + [obj.Addr('4.8.2.6', '57', True, False, + False, False)], True, True, set(['globalssl.com']), [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] @@ -121,35 +124,42 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost1 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('', '8080', False, False)], + [obj.Addr('', '8080', False, False, + False, False)], False, True, set(['localhost', r'~^(www\.)?(example|bar)\.']), [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('somename', '8080', False, False), - obj.Addr('', '8000', False, False)], + [obj.Addr('somename', '8080', False, False, + False, False), + obj.Addr('', '8000', False, False, + False, False)], False, True, set(['somename', 'another.alias', 'alias']), [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', - False, False), - obj.Addr('127.0.0.1', '', False, False)], + False, False, False, False), + obj.Addr('127.0.0.1', '', False, False, + False, False)], False, True, set(['.example.com', 'example.*']), [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), - [obj.Addr('myhost', '', False, True), - obj.Addr('otherhost', '', False, True)], + [obj.Addr('myhost', '', False, True, + False, False), + obj.Addr('otherhost', '', False, True, + False, False)], False, True, set(['www.example.org']), [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), - [obj.Addr('*', '80', True, True)], + [obj.Addr('*', '80', True, True, + False, False)], True, True, set(['*.www.foo.com', '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(10, len(vhosts)) + self.assertEqual(12, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com new file mode 100644 index 000000000..7a7744b92 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com @@ -0,0 +1,5 @@ +server { + listen 80; + listen [::]:80; + server_name ipv6.com; +} diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com new file mode 100644 index 000000000..d8f7eff12 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -0,0 +1,5 @@ +server { + listen 443 ssl; + listen [::]:443 ssl ipv6only=on; + server_name ipv6ssl.com; +} diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index af477c460..32a5ed7d2 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -125,10 +125,10 @@ class TlsSniPerformTest(util.NginxTest): self.sni.add_chall(self.achalls[0]) self.sni.add_chall(self.achalls[2]) - v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False), - obj.Addr("127.0.0.1", "", False, False)] - v_addr2 = [obj.Addr("myhost", "", False, True)] - v_addr2_print = [obj.Addr("myhost", "", False, False)] + v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False, False, False), + obj.Addr("127.0.0.1", "", False, False, False, False)] + v_addr2 = [obj.Addr("myhost", "", False, True, False, False)] + v_addr2_print = [obj.Addr("myhost", "", False, False, False, False)] ll_addr = [v_addr1, v_addr2] self.sni._mod_config(ll_addr) # pylint: disable=protected-access diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index bbfecc282..7f597ac4a 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,14 +51,32 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) + ipv6, ipv6only = self.configurator.ipv6_info( + self.configurator.config.tls_sni_01_port) + for achall in self.achalls: vhost = self.configurator.choose_vhost(achall.domain, raise_if_no_match=False) if vhost is not None and vhost.addrs: addresses.append(list(vhost.addrs)) else: - addresses.append([obj.Addr.fromstring(default_addr)]) - logger.info("Using default address %s for TLSSNI01 authentication.", default_addr) + if ipv6: + # If IPv6 is active in Nginx configuration + ipv6_addr = "[::]:{0} ssl".format( + self.configurator.config.tls_sni_01_port) + if not ipv6only: + # If ipv6only=on is not already present in the config + ipv6_addr = ipv6_addr + " ipv6only=on" + addresses.append([obj.Addr.fromstring(default_addr), + obj.Addr.fromstring(ipv6_addr)]) + logger.info(("Using default addresses %s and %s for " + + "TLSSNI01 authentication."), + default_addr, + ipv6_addr) + else: + addresses.append([obj.Addr.fromstring(default_addr)]) + logger.info("Using default address %s for TLSSNI01 authentication.", + default_addr) # Create challenge certs responses = [self._setup_challenge_cert(x) for x in self.achalls] @@ -112,7 +130,6 @@ class NginxTlsSni01(common.TLSSNI01): raise errors.MisconfigurationError( 'Certbot could not find an HTTP block to include ' 'TLS-SNI-01 challenges in %s.' % root) - config = [self._make_server_block(pair[0], pair[1]) for pair in six.moves.zip(self.achalls, ll_addrs)] config = nginxparser.UnspacedList(config) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index f605eb751..420d15679 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -251,7 +251,7 @@ class Addr(object): """Normalized representation of addr/port tuple """ if self.ipv6: - return (self._normalize_ipv6(self.tup[0]), self.tup[1]) + return (self.get_ipv6_exploded(), self.tup[1]) return self.tup def __eq__(self, other): From 884fc56a3e6dba4f183f54aba17e5c9e0c694f14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Nov 2017 10:59:56 -0700 Subject: [PATCH 207/631] Use pipstrap to ensure pip works on older systems (#5216) * Use pipstrap in tools/_venv_common.sh * Use _venv_common.sh in test_sdists --- tests/letstest/scripts/test_sdists.sh | 6 ++---- tools/_venv_common.sh | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index ced21e338..f18a64065 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -7,13 +7,11 @@ PLUGINS="certbot-apache certbot-nginx" PYTHON=$(command -v python2.7 || command -v python27 || command -v python2 || command -v python) TEMP_DIR=$(mktemp -d) VERSION=$(letsencrypt-auto-source/version.py) +export VENV_ARGS="-p $PYTHON" # setup venv -virtualenv --no-site-packages -p $PYTHON --setuptools venv +tools/_venv_common.sh --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate -pip install -U pip -pip install -U setuptools -pip install --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt # build sdists for pkg_dir in acme . $PLUGINS; do diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh index 20ed4c034..0f0ff7e28 100755 --- a/tools/_venv_common.sh +++ b/tools/_venv_common.sh @@ -15,10 +15,10 @@ mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS . ./$VENV_NAME/bin/activate -# Separately install setuptools and pip to make sure following -# invocations use latest -pip install -U pip -pip install -U setuptools +# Use pipstrap to update Python packaging tools to only update to a well tested +# version and to work around https://github.com/pypa/pip/issues/4817 on older +# systems. +python letsencrypt-auto-source/pieces/pipstrap.py ./tools/pip_install.sh "$@" set +x From 0137055c240f4bd4fd6475d7df93eb2ac2288b04 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Sun, 5 Nov 2017 21:59:55 +0100 Subject: [PATCH 208/631] First shot at updates at documentation, plenty of questions left at issue #4736 --- certbot/main.py | 262 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 237 insertions(+), 25 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 9e2850891..77d474d5f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -43,7 +43,14 @@ logger = logging.getLogger(__name__) def _suggest_donation_if_appropriate(config): - """Potentially suggest a donation to support Certbot.""" + """Potentially suggest a donation to support Certbot. + + :param interfaces.IConfig config: Configuration object + + :returns: `None` + :rtype: None + + """ assert config.verb != "renew" if config.staging: # --dry-run implies --staging @@ -55,6 +62,14 @@ def _suggest_donation_if_appropriate(config): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) def _report_successful_dry_run(config): + """Reports on successful dry run + + :param interfaces.IConfig config: Configuration object + + :returns: `None` + :rtype: None + + """ reporter_util = zope.component.getUtility(interfaces.IReporter) assert config.verb != "renew" reporter_util.add_message("The dry run was successful.", @@ -68,8 +83,16 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N then performs that action. Includes calls to hooks, various reports, checks, and requests for user input. + :param interfaces.IConfig config: Configuration object + :param list domains: domains to get a certificate. This argument is optional, if not supplied it will default to `None` + :param str certname: Name of new cert. This argument is optional, if not supplied it will default to `None` + :param storage.RenewableCert lineage: + :returns: the issued certificate or `None` if doing a dry run - :rtype: `storage.RenewableCert` or `None` + :rtype: storage.RenewableCert or None + + :raises errors.Error: if certificate could not be obtained + """ hooks.pre_hook(config) try: @@ -96,6 +119,8 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested + :param interfaces.IConfig config: Configuration object + :param list domains: Domain names. :param storage.RenewableCert cert: :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname @@ -137,6 +162,7 @@ def _handle_subset_cert_request(config, domains, cert): def _handle_identical_cert_request(config, lineage): """Figure out what to do if a lineage has the same names as a previously obtained one + :param interfaces.IConfig config: Configuration object :param storage.RenewableCert lineage: :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname @@ -186,11 +212,14 @@ def _find_lineage_for_domains(config, domains): the client run if the user chooses to cancel the operation when prompted). + :param interfaces.IConfig config: Configuration object + :param list domains: Domain names. + :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either - a RenewableCert instance or None if renewal shouldn't occur. + a RenewableCert instance or `None` if renewal shouldn't occur. - :raises .Error: If the user would like to rerun the client again. + :raises errors.Error: If the user would like to rerun the client again. """ # Considering the possibility that the requested certificate is @@ -214,6 +243,10 @@ def _find_lineage_for_domains(config, domains): def _find_cert(config, domains, certname): """Finds an existing certificate object given domains and/or a certificate name. + :param interfaces.IConfig config: Configuration object + :param list domains: Domain names. + :param str certname: Name of cert + :returns: Two-element tuple of a boolean that indicates if this function should be followed by a call to fetch a certificate from the server, and either a RenewableCert instance or None. @@ -226,11 +259,15 @@ def _find_cert(config, domains, certname): def _find_lineage_for_domains_and_certname(config, domains, certname): """Find appropriate lineage based on given domains and/or certname. + :param interfaces.IConfig config: Configuration object + :param list domains: Domain names. + :param str certname: Name of cert + :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either - a RenewableCert instance or None if renewal shouldn't occur. + a RenewableCert instance or None if renewal should not occur. - :raises .Error: If the user would like to rerun the client again. + :raises errors.Error: If the user would like to rerun the client again. """ if not certname: @@ -255,6 +292,17 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): """Ask user to confirm update cert certname to contain new_domains. + + :param interfaces.IConfig config: Configuration object + :param list new_domains: Domain names. + :param str certname: Name of cert + :param list old_domains: Domain names. + + :returns: None + :rtype: None + + :raises errors.ConfigurationError: if cert name and domains mismatch + """ if config.renew_with_new_domains: return @@ -272,6 +320,15 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): def _find_domains_or_certname(config, installer): """Retrieve domains and certname from config or user input. + + :param interfaces.IConfig config: Configuration object + :param: TODO installer? + + :returns: Two-part tuple of domains and certname + :rtype: tuple + + :raises errors.Error: Usage message, if parameters are not used correctly + """ domains = None certname = config.certname @@ -303,6 +360,9 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None): :param str fullchain_path: path to full chain :param str key_path: path to private key, if available + :returns: 'None' + :rtype: None + """ if config.dry_run: _report_successful_dry_run(config) @@ -337,14 +397,13 @@ def _determine_account(config): if ``config.account`` is ``None``, it will be updated based on the user input. Same for ``config.email``. - :param argparse.Namespace config: CLI arguments - :param certbot.interface.IConfig config: Configuration object - :param .AccountStorage account_storage: Account storage. + :param interfaces.IConfig config: Configuration object :returns: Account and optionally ACME client API (biproduct of new registration). - :rtype: `tuple` of `certbot.account.Account` and - `acme.client.Client` + :rtype: tuple of certbot.account.Account and acme.client.Client + + :raises errors.Error: If unable to register an account with ACME server """ account_storage = account.AccountFileStorage(config) @@ -394,7 +453,7 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b :param `configuration.NamespaceConfig` config: parsed command line arguments - :raises `error.Errors`: If anything goes wrong, including bad user input, if an overlapping + :raises errors.Error: If anything goes wrong, including bad user input, if an overlapping archive dir is found for the specified lineage, etc ... """ display = zope.component.getUtility(interfaces.IDisplay) @@ -474,6 +533,15 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b def _init_le_client(config, authenticator, installer): + """Initialize Let's Encrypt Client + + :param interfaces.IConfig config: Configuration object + :param: TODO authenticator + :param: TODO installer + + :returns: client: Client object + + """ if authenticator is not None: # if authenticator was given, then we will need account... acc, acme = _determine_account(config) @@ -487,7 +555,15 @@ def _init_le_client(config, authenticator, installer): def unregister(config, unused_plugins): - """Deactivate account on server""" + """Deactivate account on server + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + + """ account_storage = account.AccountFileStorage(config) accounts = account_storage.find_all() reporter_util = zope.component.getUtility(interfaces.IReporter) @@ -516,8 +592,15 @@ def unregister(config, unused_plugins): def register(config, unused_plugins): - """Create or modify accounts on the server.""" + """Create or modify accounts on the server. + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` or a string indicating and error + :rtype: None or str + + """ # Portion of _determine_account logic to see whether accounts already # exist or not. account_storage = account.AccountFileStorage(config) @@ -566,7 +649,15 @@ def _install_cert(config, le_client, domains, lineage=None): le_client.enhance_config(domains, path_provider.chain_path) def install(config, plugins): - """Install a previously obtained cert in a server.""" + """Install a previously obtained cert in a server. + + :param interfaces.IConfig config: Configuration object + :param plugins: list of plugins + + :returns: `None` + :rtype: None + + """ # XXX: Update for renewer/RenewableCert # FIXME: be consistent about whether errors are raised or returned from # this function ... @@ -582,7 +673,15 @@ def install(config, plugins): def plugins_cmd(config, plugins): - """List server software plugins.""" + """List server software plugins. + + :param interfaces.IConfig config: Configuration object + :param plugins: list of plugins + + :returns: `None` + :rtype: None + + """ logger.debug("Expected interfaces: %s", config.ifaces) ifaces = [] if config.ifaces is None else config.ifaces @@ -610,7 +709,15 @@ def plugins_cmd(config, plugins): def rollback(config, plugins): - """Rollback server configuration changes made during install.""" + """Rollback server configuration changes made during install. + + :param interfaces.IConfig config: Configuration object + :param plugins: list of plugins + + :returns: `None` + :rtype: None + + """ client.rollback(config.installer, config.checkpoints, config, plugins) @@ -619,6 +726,12 @@ def config_changes(config, unused_plugins): View checkpoints and associated configuration changes. + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + """ client.view_config_changes(config, num=config.num) @@ -627,6 +740,13 @@ def update_symlinks(config, unused_plugins): Use the information in the config file to make symlinks point to the correct archive directory. + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + """ cert_manager.update_live_symlinks(config) @@ -635,6 +755,13 @@ def rename(config, unused_plugins): Use the information in the config file to rename an existing lineage. + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + """ cert_manager.rename_lineage(config) @@ -643,16 +770,37 @@ def delete(config, unused_plugins): Use the information in the config file to delete an existing lineage. + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + """ cert_manager.delete(config) def certificates(config, unused_plugins): """Display information about certs configured with Certbot + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None """ cert_manager.certificates(config) def revoke(config, unused_plugins): # TODO: coop with renewal config - """Revoke a previously obtained certificate.""" + """Revoke a previously obtained certificate. + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` returns string indicating error in case of error + :rtype: None or str + + """ # For user-agent construction config.installer = config.authenticator = "None" if config.key_path is not None: # revocation by cert key @@ -678,7 +826,15 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals - """Obtain a certificate and install.""" + """Obtain a certificate and install. + + :param interfaces.IConfig config: Configuration object + :param plugins: list of plugins + + :returns: `None` + :rtype: None + + """ # TODO: Make run as close to auth + install as possible # Possible difficulties: config.csr was hacked into auth try: @@ -718,6 +874,13 @@ def _csr_get_and_save_cert(config, le_client): This works differently in the CSR case (for now) because we don't have the privkey, and therefore can't construct the files for a lineage. So we just save the cert & chain to disk :/ + + :param interfaces.IConfig config: Configuration object + :param client.Client client: Client object + + :returns: `cert_path` and `fullchain_path` as absolute paths to the actual files + :rtype: tuple of str + """ csr, _ = config.actual_csr certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) @@ -730,7 +893,19 @@ def _csr_get_and_save_cert(config, le_client): return cert_path, fullchain_path def renew_cert(config, plugins, lineage): - """Renew & save an existing cert. Do not install it.""" + """Renew & save an existing cert. Do not install it. + + :param interfaces.IConfig config: Configuration object + :param plugins: TODO + :param lineage: TODO + + :returns: `None` + :rtype: None + + :raises errors.PluginSelectionError: MissingCommandlineFlag in case supplied parameters do not pass + + + """ try: # installers are used in auth mode to determine domain names installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") @@ -757,8 +932,17 @@ def renew_cert(config, plugins, lineage): def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. - This implements the 'certonly' subcommand.""" + This implements the 'certonly' subcommand. + :param interfaces.IConfig config: Configuration object + :param: TODO plugins + + :returns: `None` + :rtype: None + + :raises errors.Error: If specified plugin could not be used + + """ # SETUP: Select plugins and construct a client instance try: # installers are used in auth mode to determine domain names @@ -792,7 +976,15 @@ def certonly(config, plugins): _suggest_donation_if_appropriate(config) def renew(config, unused_plugins): - """Renew previously-obtained certificates.""" + """Renew previously-obtained certificates. + + :param interfaces.IConfig config: Configuration object + :param unused_plugins: list of plugins (deprecated) + + :returns: `None` + :rtype: None + + """ try: renewal.handle_renewal_request(config) finally: @@ -800,7 +992,14 @@ def renew(config, unused_plugins): def make_or_verify_needed_dirs(config): - """Create or verify existence of config, work, and hook directories.""" + """Create or verify existence of config, work, and hook directories. + + :param interfaces.IConfig config: Configuration object + + :returns: `None` + :rtype: None + + """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, @@ -816,7 +1015,14 @@ def make_or_verify_needed_dirs(config): def set_displayer(config): - """Set the displayer""" + """Set the displayer + + :param interfaces.IConfig config: Configuration object + + :returns: `None` + :rtype: None + + """ if config.quiet: config.noninteractive_mode = True displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) @@ -829,7 +1035,13 @@ def set_displayer(config): def main(cli_args=sys.argv[1:]): - """Command line argument parsing and main script execution.""" + """Command line argument parsing and main script execution. + + :returns: TODO + + :raises errors.Error: General operating system errors triggered by issues related to wrong permissions + + """ log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() From 4e73d7ce00ce76fa0ef5b5c653f7943e436b6a7a Mon Sep 17 00:00:00 2001 From: jonasbn Date: Tue, 7 Nov 2017 21:24:30 +0100 Subject: [PATCH 209/631] Specified the list parameters after reading up on lists as parameters Ref: https://stackoverflow.com/questions/3961007/passing-an-array-list-into-python --- certbot/main.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 77d474d5f..273cc0263 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -558,7 +558,7 @@ def unregister(config, unused_plugins): """Deactivate account on server :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -595,7 +595,7 @@ def register(config, unused_plugins): """Create or modify accounts on the server. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` or a string indicating and error :rtype: None or str @@ -652,7 +652,7 @@ def install(config, plugins): """Install a previously obtained cert in a server. :param interfaces.IConfig config: Configuration object - :param plugins: list of plugins + :param list plugins: list of plugins :returns: `None` :rtype: None @@ -676,7 +676,7 @@ def plugins_cmd(config, plugins): """List server software plugins. :param interfaces.IConfig config: Configuration object - :param plugins: list of plugins + :param list plugins: list of plugins :returns: `None` :rtype: None @@ -712,7 +712,7 @@ def rollback(config, plugins): """Rollback server configuration changes made during install. :param interfaces.IConfig config: Configuration object - :param plugins: list of plugins + :param list plugins: list of plugins :returns: `None` :rtype: None @@ -727,7 +727,7 @@ def config_changes(config, unused_plugins): View checkpoints and associated configuration changes. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -742,7 +742,7 @@ def update_symlinks(config, unused_plugins): the correct archive directory. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -757,7 +757,7 @@ def rename(config, unused_plugins): lineage. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -772,7 +772,7 @@ def delete(config, unused_plugins): lineage. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -784,7 +784,7 @@ def certificates(config, unused_plugins): """Display information about certs configured with Certbot :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None @@ -795,7 +795,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` returns string indicating error in case of error :rtype: None or str @@ -829,7 +829,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install. :param interfaces.IConfig config: Configuration object - :param plugins: list of plugins + :param list plugins: list of plugins :returns: `None` :rtype: None @@ -896,7 +896,7 @@ def renew_cert(config, plugins, lineage): """Renew & save an existing cert. Do not install it. :param interfaces.IConfig config: Configuration object - :param plugins: TODO + :param list plugins: TODO :param lineage: TODO :returns: `None` @@ -935,7 +935,7 @@ def certonly(config, plugins): This implements the 'certonly' subcommand. :param interfaces.IConfig config: Configuration object - :param: TODO plugins + :param list plugins: List of plugins :returns: `None` :rtype: None @@ -979,7 +979,7 @@ def renew(config, unused_plugins): """Renew previously-obtained certificates. :param interfaces.IConfig config: Configuration object - :param unused_plugins: list of plugins (deprecated) + :param list unused_plugins: list of plugins (deprecated) :returns: `None` :rtype: None From 89485f7463123f2a687876e5950a6f729c66e37f Mon Sep 17 00:00:00 2001 From: jonasbn Date: Tue, 7 Nov 2017 21:40:35 +0100 Subject: [PATCH 210/631] I think I figured out the authentication handler object --- certbot/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 273cc0263..6c9e377ef 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -322,7 +322,7 @@ def _find_domains_or_certname(config, installer): """Retrieve domains and certname from config or user input. :param interfaces.IConfig config: Configuration object - :param: TODO installer? + :param installer: Installer object :returns: Two-part tuple of domains and certname :rtype: tuple @@ -536,8 +536,8 @@ def _init_le_client(config, authenticator, installer): """Initialize Let's Encrypt Client :param interfaces.IConfig config: Configuration object - :param: TODO authenticator - :param: TODO installer + :param AuthHandler authenticator: Acme authentication handler + :param installer: Installer object :returns: client: Client object @@ -896,7 +896,7 @@ def renew_cert(config, plugins, lineage): """Renew & save an existing cert. Do not install it. :param interfaces.IConfig config: Configuration object - :param list plugins: TODO + :param list plugins: List of plugins :param lineage: TODO :returns: `None` From 0aa9322280f9ad7a780c470607c4da1c1ca1bb1d Mon Sep 17 00:00:00 2001 From: jonasbn Date: Tue, 7 Nov 2017 21:47:59 +0100 Subject: [PATCH 211/631] Added a shot at what might be the proper type, I need to get a better understanding of certbot's datatypes --- certbot/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index 6c9e377ef..f3096da6c 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -897,7 +897,7 @@ def renew_cert(config, plugins, lineage): :param interfaces.IConfig config: Configuration object :param list plugins: List of plugins - :param lineage: TODO + :param RenewableCert lineage: a certificate lineage object :returns: `None` :rtype: None From 1173acfaf0a377917442d7becd16dd863a1f27aa Mon Sep 17 00:00:00 2001 From: jonasbn Date: Tue, 7 Nov 2017 22:18:11 +0100 Subject: [PATCH 212/631] Making friends with the linter lint: commands succeeded congratulations :) --- certbot/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index f3096da6c..5f2803da7 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -84,8 +84,11 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N checks, and requests for user input. :param interfaces.IConfig config: Configuration object - :param list domains: domains to get a certificate. This argument is optional, if not supplied it will default to `None` - :param str certname: Name of new cert. This argument is optional, if not supplied it will default to `None` + + :param list domains: domains to get a certificate. Defaults to `None` + + :param str certname: Name of new cert. Defaults to `None` + :param storage.RenewableCert lineage: :returns: the issued certificate or `None` if doing a dry run @@ -902,8 +905,7 @@ def renew_cert(config, plugins, lineage): :returns: `None` :rtype: None - :raises errors.PluginSelectionError: MissingCommandlineFlag in case supplied parameters do not pass - + :raises errors.PluginSelectionError: MissingCommandlineFlag if supplied parameters do not pass """ try: @@ -1039,7 +1041,7 @@ def main(cli_args=sys.argv[1:]): :returns: TODO - :raises errors.Error: General operating system errors triggered by issues related to wrong permissions + :raises errors.Error: OS errors triggered by wrong permissions """ log.pre_arg_parse_setup() From 686fa36b3b8eea3ed8859c6af9d0acd7ee9b8978 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 8 Nov 2017 10:58:00 -0800 Subject: [PATCH 213/631] Install dnsmadeeasy extras from dns-lexicon (#5230) * Add tools/pip_constraints.txt to pin all Python dependencies * Use tools/pip_constraints.txt in tools/pip_install.sh * Install dnsmadeeasy extras in dnsmadeeasy plugin --- certbot-dns-dnsmadeeasy/setup.py | 4 +- tools/pip_constraints.txt | 65 ++++++++++++++++++++++++++++++++ tools/pip_install.sh | 11 +++--- 3 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 tools/pip_constraints.txt diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 88e02304e..1ddec208b 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -10,7 +10,9 @@ version = '0.20.0.dev0' install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), - 'dns-lexicon', + # new versions of lexicon require that we install dnsmadeeasy extras and + # 2.1.11 is the first version that defines them. + 'dns-lexicon[dnsmadeeasy]>=2.1.11', 'mock', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: diff --git a/tools/pip_constraints.txt b/tools/pip_constraints.txt new file mode 100644 index 000000000..8970d417c --- /dev/null +++ b/tools/pip_constraints.txt @@ -0,0 +1,65 @@ +# Specifies Python package versions for packages not specified in +# letsencrypt-auto's requirements file. We should avoid listing packages in +# both places because if both files are used as constraints for the same pip +# invocation, some constraints may be ignored due to pip's lack of dependency +# resolution. +alabaster==0.7.10 +astroid==1.3.5 +Babel==2.5.1 +backports.shutil-get-terminal-size==1.0.0 +boto3==1.4.7 +botocore==1.7.41 +cloudflare==1.8.1 +coverage==4.4.2 +decorator==4.1.2 +dns-lexicon[dnsmadeeasy]==2.1.11 +dnspython==1.15.0 +docutils==0.14 +future==0.16.0 +futures==3.1.1 +google-api-python-client==1.6.4 +httplib2==0.10.3 +imagesize==0.7.1 +ipdb==0.10.3 +ipython==5.5.0 +ipython-genutils==0.2.0 +Jinja2==2.9.6 +jmespath==0.9.3 +logilab-common==1.4.1 +MarkupSafe==1.0 +nose==1.3.7 +oauth2client==4.1.2 +pathlib2==2.3.0 +pexpect==4.2.1 +pickleshare==0.7.4 +pkg-resources==0.0.0 +pkginfo==1.4.1 +pluggy==0.5.2 +prompt-toolkit==1.0.15 +ptyprocess==0.5.2 +py==1.4.34 +pyasn1==0.3.7 +pyasn1-modules==0.1.5 +Pygments==2.2.0 +pylint==1.4.2 +python-dateutil==2.6.1 +python-digitalocean==1.12 +PyYAML==3.12 +repoze.sphinx.autointerface==0.8 +requests-file==1.4.2 +requests-toolbelt==0.8.0 +rsa==3.4.2 +s3transfer==0.1.11 +scandir==1.6 +simplegeneric==0.8.1 +snowballstemmer==1.2.1 +Sphinx==1.5.6 +sphinx-rtd-theme==0.2.4 +tldextract==2.2.0 +tox==2.9.1 +tqdm==4.19.4 +traitlets==4.3.2 +twine==1.9.1 +uritemplate==3.0.0 +virtualenv==15.1.0 +wcwidth==0.1.7 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index 438e567e4..194501c7d 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,14 +1,15 @@ #!/bin/sh -e -# pip installs packages using Certbot's requirements file as constraints +# pip installs packages using pinned package versions # get the root of the Certbot repo my_path=$("$(dirname $0)/readlink.py" $0) repo_root=$(dirname $(dirname $my_path)) requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" -constraints=$(mktemp) -trap "rm -f $constraints" EXIT +certbot_auto_constraints=$(mktemp) +trap "rm -f $certbot_auto_constraints" EXIT # extract pinned requirements without hashes -sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $constraints +sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints +dev_constraints="$(dirname $my_path)/pip_constraints.txt" # install the requested packages using the pinned requirements as constraints -pip install --constraint $constraints "$@" +pip install --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" From eb26e0aacf4ac2454fe8eb5dcb8a529b466737e1 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Sun, 12 Nov 2017 00:32:24 +0100 Subject: [PATCH 214/631] Updated parameter types for a lot of parametersm some aspects are still a bug unclear, hopefully a review can shed some light on this details --- certbot/main.py | 235 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 167 insertions(+), 68 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 5f2803da7..089149858 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -45,7 +45,8 @@ logger = logging.getLogger(__name__) def _suggest_donation_if_appropriate(config): """Potentially suggest a donation to support Certbot. - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig :returns: `None` :rtype: None @@ -64,7 +65,8 @@ def _suggest_donation_if_appropriate(config): def _report_successful_dry_run(config): """Reports on successful dry run - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig :returns: `None` :rtype: None @@ -83,13 +85,17 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N then performs that action. Includes calls to hooks, various reports, checks, and requests for user input. - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig - :param list domains: domains to get a certificate. Defaults to `None` + :param domains: domains to get a certificate. Defaults to `None` + :type domains: `list` of `str` - :param str certname: Name of new cert. Defaults to `None` + :param certname: Name of new cert. Defaults to `None` + :type certname: str - :param storage.RenewableCert lineage: + :param lineage: + :type lineage: storage.RenewableCert :returns: the issued certificate or `None` if doing a dry run :rtype: storage.RenewableCert or None @@ -122,9 +128,14 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested - :param interfaces.IConfig config: Configuration object - :param list domains: Domain names. - :param storage.RenewableCert cert: + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: domains + :type domains: `list` of `str` + + :param cert: + :type cert: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" @@ -165,8 +176,11 @@ def _handle_subset_cert_request(config, domains, cert): def _handle_identical_cert_request(config, lineage): """Figure out what to do if a lineage has the same names as a previously obtained one - :param interfaces.IConfig config: Configuration object - :param storage.RenewableCert lineage: + :param config: Configuration object + :type config: interfaces.IConfig + + :param lineage: + :type lineage: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" @@ -215,8 +229,11 @@ def _find_lineage_for_domains(config, domains): the client run if the user chooses to cancel the operation when prompted). - :param interfaces.IConfig config: Configuration object - :param list domains: Domain names. + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: domains + :type domains: `list` of `str` :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -246,9 +263,14 @@ def _find_lineage_for_domains(config, domains): def _find_cert(config, domains, certname): """Finds an existing certificate object given domains and/or a certificate name. - :param interfaces.IConfig config: Configuration object - :param list domains: Domain names. - :param str certname: Name of cert + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: domains + :type domains: `list` of `str` + + :param certname: Name of cert + :type certname: str :returns: Two-element tuple of a boolean that indicates if this function should be followed by a call to fetch a certificate from the server, and either a @@ -262,9 +284,14 @@ def _find_cert(config, domains, certname): def _find_lineage_for_domains_and_certname(config, domains, certname): """Find appropriate lineage based on given domains and/or certname. - :param interfaces.IConfig config: Configuration object - :param list domains: Domain names. - :param str certname: Name of cert + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: domains + :type domains: `list` of `str` + + :param certname: Name of cert + :type certname: str :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -296,10 +323,17 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): """Ask user to confirm update cert certname to contain new_domains. - :param interfaces.IConfig config: Configuration object - :param list new_domains: Domain names. - :param str certname: Name of cert - :param list old_domains: Domain names. + :param config: Configuration object + :type config: interfaces.IConfig + + :param new_domains: domains + :type new_domains: `list` of `str` + + :param certname: Name of cert + :type certname: str + + :param old_domains: domains + :type old_domains: `list` of `str` :returns: None :rtype: None @@ -324,8 +358,12 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): def _find_domains_or_certname(config, installer): """Retrieve domains and certname from config or user input. - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig + :param installer: Installer object + :type installer: interfaces.IInstaller + :returns: Two-part tuple of domains and certname :rtype: tuple @@ -359,11 +397,14 @@ def _find_domains_or_certname(config, installer): def _report_new_cert(config, cert_path, fullchain_path, key_path=None): """Reports the creation of a new certificate to the user. - :param str cert_path: path to cert - :param str fullchain_path: path to full chain - :param str key_path: path to private key, if available + :param cert_path: path to cert + :type cert_path: str + :param fullchain_path: path to full chain + :type fullchain_path: str + :param key_path: path to private key, if available + :type key_path: str - :returns: 'None' + :returns: `None` :rtype: None """ @@ -400,7 +441,8 @@ def _determine_account(config): if ``config.account`` is ``None``, it will be updated based on the user input. Same for ``config.email``. - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig :returns: Account and optionally ACME client API (biproduct of new registration). @@ -538,9 +580,13 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b def _init_le_client(config, authenticator, installer): """Initialize Let's Encrypt Client - :param interfaces.IConfig config: Configuration object - :param AuthHandler authenticator: Acme authentication handler + :param config: Configuration object + :type config: interfaces.IConfig + + :param authenticator: Acme authentication handler + :type authenticator: interfaces.IAuthenticator :param installer: Installer object + :type installer: interfaces.IInstaller :returns: client: Client object @@ -560,8 +606,11 @@ def _init_le_client(config, authenticator, installer): def unregister(config, unused_plugins): """Deactivate account on server - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -597,8 +646,11 @@ def unregister(config, unused_plugins): def register(config, unused_plugins): """Create or modify accounts on the server. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` or a string indicating and error :rtype: None or str @@ -654,8 +706,11 @@ def _install_cert(config, le_client, domains, lineage=None): def install(config, plugins): """Install a previously obtained cert in a server. - :param interfaces.IConfig config: Configuration object - :param list plugins: list of plugins + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` :returns: `None` :rtype: None @@ -678,8 +733,11 @@ def install(config, plugins): def plugins_cmd(config, plugins): """List server software plugins. - :param interfaces.IConfig config: Configuration object - :param list plugins: list of plugins + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` :returns: `None` :rtype: None @@ -714,8 +772,11 @@ def plugins_cmd(config, plugins): def rollback(config, plugins): """Rollback server configuration changes made during install. - :param interfaces.IConfig config: Configuration object - :param list plugins: list of plugins + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` :returns: `None` :rtype: None @@ -729,8 +790,11 @@ def config_changes(config, unused_plugins): View checkpoints and associated configuration changes. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -744,8 +808,11 @@ def update_symlinks(config, unused_plugins): Use the information in the config file to make symlinks point to the correct archive directory. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -759,8 +826,11 @@ def rename(config, unused_plugins): Use the information in the config file to rename an existing lineage. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -774,8 +844,11 @@ def delete(config, unused_plugins): Use the information in the config file to delete an existing lineage. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -786,8 +859,11 @@ def delete(config, unused_plugins): def certificates(config, unused_plugins): """Display information about certs configured with Certbot - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -797,10 +873,13 @@ def certificates(config, unused_plugins): def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig - :returns: `None` returns string indicating error in case of error + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` or string indicating error in case of error :rtype: None or str """ @@ -831,8 +910,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install. - :param interfaces.IConfig config: Configuration object - :param list plugins: list of plugins + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` :returns: `None` :rtype: None @@ -878,8 +960,11 @@ def _csr_get_and_save_cert(config, le_client): have the privkey, and therefore can't construct the files for a lineage. So we just save the cert & chain to disk :/ - :param interfaces.IConfig config: Configuration object - :param client.Client client: Client object + :param config: Configuration object + :type config: interfaces.IConfig + + :param client: Client object + :type client: client.Client :returns: `cert_path` and `fullchain_path` as absolute paths to the actual files :rtype: tuple of str @@ -898,9 +983,14 @@ def _csr_get_and_save_cert(config, le_client): def renew_cert(config, plugins, lineage): """Renew & save an existing cert. Do not install it. - :param interfaces.IConfig config: Configuration object - :param list plugins: List of plugins - :param RenewableCert lineage: a certificate lineage object + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` + + :param lineage: certificate lineage object + :type lineage: storage.RenewableCert :returns: `None` :rtype: None @@ -936,8 +1026,11 @@ def certonly(config, plugins): This implements the 'certonly' subcommand. - :param interfaces.IConfig config: Configuration object - :param list plugins: List of plugins + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: list of plugins + :type plugins: `list` of `str` :returns: `None` :rtype: None @@ -980,8 +1073,11 @@ def certonly(config, plugins): def renew(config, unused_plugins): """Renew previously-obtained certificates. - :param interfaces.IConfig config: Configuration object - :param list unused_plugins: list of plugins (deprecated) + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: list of plugins (deprecated) + :type unused_plugins: `list` of `str` :returns: `None` :rtype: None @@ -996,7 +1092,8 @@ def renew(config, unused_plugins): def make_or_verify_needed_dirs(config): """Create or verify existence of config, work, and hook directories. - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig :returns: `None` :rtype: None @@ -1019,7 +1116,8 @@ def make_or_verify_needed_dirs(config): def set_displayer(config): """Set the displayer - :param interfaces.IConfig config: Configuration object + :param config: Configuration object + :type config: interfaces.IConfig :returns: `None` :rtype: None @@ -1039,9 +1137,10 @@ def set_displayer(config): def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution. - :returns: TODO + :returns: result of requested command :raises errors.Error: OS errors triggered by wrong permissions + :raises errors.Error: error if plugin command is not supported """ log.pre_arg_parse_setup() From 4d60f32865ceedc7a39f0f74cfe53445f7610fb4 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Sun, 12 Nov 2017 13:03:09 +0100 Subject: [PATCH 215/631] Minor corrections to return types for improved formatting --- certbot/main.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 089149858..d30f434c0 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -139,7 +139,7 @@ def _handle_subset_cert_request(config, domains, cert): :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" - :rtype: tuple + :rtype: `tuple` of `str` """ existing = ", ".join(cert.names()) @@ -184,7 +184,7 @@ def _handle_identical_cert_request(config, lineage): :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" - :rtype: tuple + :rtype: `tuple` of `str` """ if not lineage.ensure_deployed(): @@ -238,6 +238,7 @@ def _find_lineage_for_domains(config, domains): :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either a RenewableCert instance or `None` if renewal shouldn't occur. + :rtype: `tuple` of `str` and :class:`storage.RenewableCert` or `None` :raises errors.Error: If the user would like to rerun the client again. @@ -275,6 +276,8 @@ def _find_cert(config, domains, certname): :returns: Two-element tuple of a boolean that indicates if this function should be followed by a call to fetch a certificate from the server, and either a RenewableCert instance or None. + :rtype: `tuple` of `bool` and :class:`storage.RenewableCert` or `None` + """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) if action == "reinstall": @@ -297,6 +300,8 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): a string token ("reinstall", "renew", or "newcert"), plus either a RenewableCert instance or None if renewal should not occur. + :rtype: `tuple` of `str` and :class:`storage.RenewableCert` or `None` + :raises errors.Error: If the user would like to rerun the client again. """ @@ -366,7 +371,7 @@ def _find_domains_or_certname(config, installer): :returns: Two-part tuple of domains and certname - :rtype: tuple + :rtype: `tuple` of list of `str` and `str` :raises errors.Error: Usage message, if parameters are not used correctly @@ -446,7 +451,7 @@ def _determine_account(config): :returns: Account and optionally ACME client API (biproduct of new registration). - :rtype: tuple of certbot.account.Account and acme.client.Client + :rtype: tuple of :class:`certbot.account.Account` and :class:`acme.client.Client` :raises errors.Error: If unable to register an account with ACME server @@ -498,6 +503,9 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b :param `configuration.NamespaceConfig` config: parsed command line arguments + :returns: `None` + :rtype: None + :raises errors.Error: If anything goes wrong, including bad user input, if an overlapping archive dir is found for the specified lineage, etc ... """ @@ -589,6 +597,7 @@ def _init_le_client(config, authenticator, installer): :type installer: interfaces.IInstaller :returns: client: Client object + :rtype: client.Client """ if authenticator is not None: @@ -867,6 +876,7 @@ def certificates(config, unused_plugins): :returns: `None` :rtype: None + """ cert_manager.certificates(config) @@ -967,7 +977,7 @@ def _csr_get_and_save_cert(config, le_client): :type client: client.Client :returns: `cert_path` and `fullchain_path` as absolute paths to the actual files - :rtype: tuple of str + :rtype: `tuple` of `str` """ csr, _ = config.actual_csr From 0b843bb8515822bd7f6564180630699d01d3abdb Mon Sep 17 00:00:00 2001 From: jonasbn Date: Wed, 15 Nov 2017 07:23:34 +0100 Subject: [PATCH 216/631] Added some missing documentation --- certbot/main.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/certbot/main.py b/certbot/main.py index d30f434c0..aeb147e86 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -705,6 +705,24 @@ def register(config, unused_plugins): add_msg("Your e-mail address was updated to {0}.".format(config.email)) def _install_cert(config, le_client, domains, lineage=None): + """Install a cert + + :param config: Configuration object + :type config: interfaces.IConfig + + :param le_client: Client object + :type le_client: client.Client + + :param plugins: list of domains + :type plugins: `list` of `str` + + :param lineage: certificate lineage object + :type lineage: storage.RenewableCert + + :returns: `None` + :rtype: None + + """ path_provider = lineage if lineage else config assert path_provider.cert_path is not None From 02126c0961817ba9b87b0864a75c92800fff8e08 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Wed, 15 Nov 2017 07:24:54 +0100 Subject: [PATCH 217/631] Minor improvement to newly added documentation section --- certbot/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index aeb147e86..7359b88d5 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -716,7 +716,7 @@ def _install_cert(config, le_client, domains, lineage=None): :param plugins: list of domains :type plugins: `list` of `str` - :param lineage: certificate lineage object + :param lineage: certificate lineage object. Defaults to `None` :type lineage: storage.RenewableCert :returns: `None` From e795a79547d65f0965aa4091123214800d40949d Mon Sep 17 00:00:00 2001 From: jonasbn Date: Wed, 15 Nov 2017 07:38:09 +0100 Subject: [PATCH 218/631] Lots of minor small cosmetic changes and addressing the feedback on uniformity (in the file) from @SwartzCr --- certbot/main.py | 71 ++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 7359b88d5..82063d0db 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -88,13 +88,13 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N :param config: Configuration object :type config: interfaces.IConfig - :param domains: domains to get a certificate. Defaults to `None` + :param domains: List of domain names to get a certificate. Defaults to `None` :type domains: `list` of `str` - :param certname: Name of new cert. Defaults to `None` + :param certname: Name of new certificate. Defaults to `None` :type certname: str - :param lineage: + :param lineage: Certificate lineage object. Defaults to `None` :type lineage: storage.RenewableCert :returns: the issued certificate or `None` if doing a dry run @@ -131,10 +131,10 @@ def _handle_subset_cert_request(config, domains, cert): :param config: Configuration object :type config: interfaces.IConfig - :param domains: domains + :param domains: List of domain names :type domains: `list` of `str` - :param cert: + :param cert: Certificate object :type cert: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname @@ -179,7 +179,7 @@ def _handle_identical_cert_request(config, lineage): :param config: Configuration object :type config: interfaces.IConfig - :param lineage: + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname @@ -232,7 +232,7 @@ def _find_lineage_for_domains(config, domains): :param config: Configuration object :type config: interfaces.IConfig - :param domains: domains + :param domains: List of domain names :type domains: `list` of `str` :returns: Two-element tuple containing desired new-certificate behavior as @@ -267,10 +267,10 @@ def _find_cert(config, domains, certname): :param config: Configuration object :type config: interfaces.IConfig - :param domains: domains + :param domains: List of domain names :type domains: `list` of `str` - :param certname: Name of cert + :param certname: Name of certificate :type certname: str :returns: Two-element tuple of a boolean that indicates if this function should be @@ -290,10 +290,10 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): :param config: Configuration object :type config: interfaces.IConfig - :param domains: domains + :param domains: List of domain names :type domains: `list` of `str` - :param certname: Name of cert + :param certname: Name of certificate :type certname: str :returns: Two-element tuple containing desired new-certificate behavior as @@ -331,13 +331,13 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): :param config: Configuration object :type config: interfaces.IConfig - :param new_domains: domains + :param new_domains: List of new domain names :type new_domains: `list` of `str` - :param certname: Name of cert + :param certname: Name of certificate :type certname: str - :param old_domains: domains + :param old_domains: List of old domain names :type old_domains: `list` of `str` :returns: None @@ -402,10 +402,12 @@ def _find_domains_or_certname(config, installer): def _report_new_cert(config, cert_path, fullchain_path, key_path=None): """Reports the creation of a new certificate to the user. - :param cert_path: path to cert + :param cert_path: path to certificate :type cert_path: str + :param fullchain_path: path to full chain :type fullchain_path: str + :param key_path: path to private key, if available :type key_path: str @@ -501,7 +503,8 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b deleting happens automatically, unless if both `--cert-name` and `--cert-path` were specified with conflicting values. - :param `configuration.NamespaceConfig` config: parsed command line arguments + :param config: parsed command line arguments + :type config: interfaces.IConfig :returns: `None` :rtype: None @@ -618,7 +621,7 @@ def unregister(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -658,7 +661,7 @@ def register(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` or a string indicating and error @@ -713,10 +716,10 @@ def _install_cert(config, le_client, domains, lineage=None): :param le_client: Client object :type le_client: client.Client - :param plugins: list of domains + :param plugins: List of domains :type plugins: `list` of `str` - :param lineage: certificate lineage object. Defaults to `None` + :param lineage: Certificate lineage object. Defaults to `None` :type lineage: storage.RenewableCert :returns: `None` @@ -736,7 +739,7 @@ def install(config, plugins): :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` :returns: `None` @@ -763,7 +766,7 @@ def plugins_cmd(config, plugins): :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` :returns: `None` @@ -802,7 +805,7 @@ def rollback(config, plugins): :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` :returns: `None` @@ -820,7 +823,7 @@ def config_changes(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -838,7 +841,7 @@ def update_symlinks(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -856,7 +859,7 @@ def rename(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -874,7 +877,7 @@ def delete(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -889,7 +892,7 @@ def certificates(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` @@ -904,7 +907,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` or string indicating error in case of error @@ -941,7 +944,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` :returns: `None` @@ -1014,10 +1017,10 @@ def renew_cert(config, plugins, lineage): :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` - :param lineage: certificate lineage object + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert :returns: `None` @@ -1057,7 +1060,7 @@ def certonly(config, plugins): :param config: Configuration object :type config: interfaces.IConfig - :param plugins: list of plugins + :param plugins: List of plugins :type plugins: `list` of `str` :returns: `None` @@ -1104,7 +1107,7 @@ def renew(config, unused_plugins): :param config: Configuration object :type config: interfaces.IConfig - :param unused_plugins: list of plugins (deprecated) + :param unused_plugins: List of plugins (deprecated) :type unused_plugins: `list` of `str` :returns: `None` From cdd89998e3bb20d9337bbc3ef8d2d7fad9a43454 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 27 Nov 2017 14:49:19 -0800 Subject: [PATCH 219/631] Add nginx to these weird instructions (#5243) These are probably made obsolete by the instruction generator, and they don't include Ubuntu... --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index 1f6a45e07..a914586ff 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -156,7 +156,7 @@ If you run Debian Stretch or Debian Sid, you can install certbot packages. sudo apt-get install certbot python-certbot-apache If you don't want to use the Apache plugin, you can omit the -``python-certbot-apache`` package. +``python-certbot-apache`` package. Or you can install ``python-certbot-nginx`` instead. Packages exist for Debian Jessie via backports. First you'll have to follow the instructions at http://backports.debian.org/Instructions/ to enable the Jessie backports From f5ed771d4f884c2b2ca7df8cf973e633ae04be2b Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Mon, 27 Nov 2017 14:50:06 -0800 Subject: [PATCH 220/631] change some instances of help to flag (#5248) --- certbot/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 3a12f86c7..622462278 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -828,11 +828,11 @@ class HelpfulArgumentParser(object): return dict([(t, t == chosen_topic) for t in self.help_topics]) def _add_all_groups(helpful): - helpful.add_group("automation", description="Arguments for automating execution & other tweaks") + helpful.add_group("automation", description="Flags for automating execution & other tweaks") helpful.add_group("security", description="Security parameters & server settings") helpful.add_group("testing", description="The following flags are meant for testing and integration purposes only.") - helpful.add_group("paths", description="Arguments changing execution paths & servers") + helpful.add_group("paths", description="Flags for changing execution paths & servers") helpful.add_group("manage", description="Various subcommands and flags are available for managing your certificates:", verbs=["certificates", "delete", "renew", "revoke", "update_symlinks"]) From 8fd1d0d19e1be1ecbf3896ca66dbfa0c6732e6d9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 28 Nov 2017 18:22:01 -0800 Subject: [PATCH 221/631] Small Travis cleanups (#5273) * Test with no hosts. * Simplify build matrix. * Remove after_failure. --- .travis.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 48b9b43cb..55f18338d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,6 @@ before_install: before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' -# using separate envs with different TOXENVs creates 4x1 Travis build -# matrix, which allows us to clearly distinguish which component under -# test has failed matrix: include: - python: "2.7" @@ -22,23 +19,14 @@ matrix: - python: "2.7" env: TOXENV=py27-oldest BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "2.6" env: TOXENV=py26 BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "2.7" env: TOXENV=py27_install BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - sudo: required env: TOXENV=apache_compat @@ -81,30 +69,18 @@ matrix: - python: "3.3" env: TOXENV=py33 BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=1 sudo: required - after_failure: - - sudo cat /var/log/mysql/error.log - - ps aux | grep mysql services: docker - python: "2.7" env: TOXENV=nginxroundtrip @@ -130,17 +106,6 @@ branches: sudo: false addons: - # Custom /etc/hosts required for simple verification of http-01 - # and tls-sni-01, and for certbot_test_nginx - hosts: - - le.wtf - - le1.wtf - - le2.wtf - - le3.wtf - - nginx.wtf - - boulder - - boulder-mysql - - boulder-rabbitmq apt: sources: - augeas From d246ba78c76e686e252a3b190710f7563d87c025 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 29 Nov 2017 13:09:25 -0800 Subject: [PATCH 222/631] Use pip3 if pip isn't available (#5277) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 55f18338d..d331dc980 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,7 +125,7 @@ addons: - libapache2-mod-wsgi - libapache2-mod-macro -install: "travis_retry pip install tox coveralls" +install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' From 20bca1942033c2c0164fd7b98be9243b323c7067 Mon Sep 17 00:00:00 2001 From: Eccenux Date: Thu, 30 Nov 2017 20:24:49 +0100 Subject: [PATCH 223/631] Show a diff when re-creating certificate instead of full list of domains #5274 --- certbot/main.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 9e2850891..11f7ddab7 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -253,18 +253,39 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): "Use -d to specify domains, or run certbot --certificates to see " "possible certificate names.".format(certname)) +def _get_added_removed(after, before): + """Get lists of items removed from `before` + and a lists of items added to `after` + """ + added = list(set(after) - set(before)) + removed = list(set(before) - set(after)) + added.sort() + removed.sort() + return added, removed + +def _format_list(character, list): + """Format list with given character + """ + formatted = "{br}{ch} " + "{br}{ch} ".join(list) + return formatted.format( + ch=character, + br=os.linesep + ) + def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): """Ask user to confirm update cert certname to contain new_domains. """ if config.renew_with_new_domains: return - msg = ("You are updating certificate {0} to include domains: {1}{br}{br}" - "It previously included domains: {2}{br}{br}" + added, removed = _get_added_removed(new_domains, old_domains) + + msg = ("You are updating certificate {0} to include new domain(s): {1}{br}{br}" + "You are also removing previously included domain(s): {2}{br}{br}" "Did you intend to make this change?".format( certname, - ", ".join(new_domains), - ", ".join(old_domains), + _format_list("+", added), + _format_list("-", removed), br=os.linesep)) obj = zope.component.getUtility(interfaces.IDisplay) if not obj.yesno(msg, "Update cert", "Cancel", default=True): From 48173ed1cb075b73ac1b913cd3f3f872ba700c03 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Dec 2017 10:59:55 -0800 Subject: [PATCH 224/631] Switch from nose to pytest (#5282) * Use pipstrap to install a good version of pip * Use pytest in cb-auto tests * Remove nose usage in auto_test.py * remove nose dev dep * use pytest in test_tests * Use pytest in tox * Update dev dependency pinnings * remove nose multiprocess lines * Use pytest for coverage * Use older py and pytest for old python versions * Add test for Error.__str__ * pin pytest in oldest test * Fix tests for DNS-DO plugin on py26 * Work around bug for Python 3.3 * Clarify dockerfile comments --- .coveragerc | 3 +-- acme/acme/crypto_util_test.py | 3 --- acme/acme/messages_test.py | 6 ++++++ acme/acme/standalone_test.py | 7 ------- acme/setup.py | 3 ++- .../tests/augeas_configurator_test.py | 1 - .../certbot_apache/tests/configurator_test.py | 2 -- .../dns_digitalocean_test.py | 5 ++--- .../certbot_nginx/tests/configurator_test.py | 1 - certbot/tests/cli_test.py | 4 ---- certbot/tests/ocsp_test.py | 1 - certbot/tests/storage_test.py | 1 - letsencrypt-auto-source/Dockerfile.centos6 | 10 ++++++--- letsencrypt-auto-source/Dockerfile.precise | 11 ++++++---- letsencrypt-auto-source/Dockerfile.trusty | 10 ++++++--- letsencrypt-auto-source/Dockerfile.wheezy | 12 ++++++----- letsencrypt-auto-source/tests/auto_test.py | 15 ++++++------- setup.cfg | 6 ------ setup.py | 4 +++- tests/letstest/scripts/test_tests.sh | 4 ++-- tools/install_and_test.sh | 6 +++++- tools/pip_constraints.txt | 7 ++++++- tools/release.sh | 6 +++--- tox.cover.sh | 21 +++++++------------ tox.ini | 4 +--- 25 files changed, 74 insertions(+), 79 deletions(-) diff --git a/.coveragerc b/.coveragerc index 087900105..1a87ab2da 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,2 @@ [report] -# show lines missing coverage in output -show_missing = True +omit = */setup.py diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 4046aa197..da433c5a2 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -18,7 +18,6 @@ from acme import test_util class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - _multiprocess_can_split_ = True def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') @@ -69,7 +68,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" - _multiprocess_can_split_ = True @classmethod def _call(cls, loader, name): @@ -140,7 +138,6 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): class RandomSnTest(unittest.TestCase): """Test for random certificate serial numbers.""" - _multiprocess_can_split_ = True def setUp(self): self.cert_count = 5 diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index b4ce19a08..631f0ce4d 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -71,6 +71,12 @@ class ErrorTest(unittest.TestCase): self.assertTrue(is_acme_error(Error.with_code('badCSR'))) self.assertRaises(ValueError, Error.with_code, 'not an ACME error code') + def test_str(self): + self.assertEqual( + str(self.error), + u"{0.typ} :: {0.description} :: {0.detail} :: {0.title}" + .format(self.error)) + class ConstantTest(unittest.TestCase): """Tests for acme.messages._Constant.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 16669680c..48e13d0b6 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -23,7 +23,6 @@ from acme import test_util class TLSServerTest(unittest.TestCase): """Tests for acme.standalone.TLSServer.""" - _multiprocess_can_split_ = True def test_bind(self): # pylint: disable=no-self-use from acme.standalone import TLSServer @@ -42,7 +41,6 @@ class TLSServerTest(unittest.TestCase): class TLSSNI01ServerTest(unittest.TestCase): """Test for acme.standalone.TLSSNI01Server.""" - _multiprocess_can_split_ = True def setUp(self): self.certs = {b'localhost': ( @@ -70,7 +68,6 @@ class TLSSNI01ServerTest(unittest.TestCase): class HTTP01ServerTest(unittest.TestCase): """Tests for acme.standalone.HTTP01Server.""" - _multiprocess_can_split_ = True def setUp(self): self.account_key = jose.JWK.load( @@ -124,7 +121,6 @@ class HTTP01ServerTest(unittest.TestCase): class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" - _multiprocess_can_split_ = True class SingleProtocolServer(socketserver.TCPServer): """Server that only serves on a single protocol. FreeBSD has this behavior for AF_INET6.""" @@ -174,7 +170,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase): class TLSSNI01DualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.TLSSNI01DualNetworkedServers.""" - _multiprocess_can_split_ = True def setUp(self): self.certs = {b'localhost': ( @@ -202,7 +197,6 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase): class HTTP01DualNetworkedServersTest(unittest.TestCase): """Tests for acme.standalone.HTTP01DualNetworkedServers.""" - _multiprocess_can_split_ = True def setUp(self): self.account_key = jose.JWK.load( @@ -254,7 +248,6 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): class TestSimpleTLSSNI01Server(unittest.TestCase): """Tests for acme.standalone.simple_tls_sni_01_server.""" - _multiprocess_can_split_ = True def setUp(self): # mirror ../examples/standalone diff --git a/acme/setup.py b/acme/setup.py index c86a72b9d..c28e0c152 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -31,7 +31,8 @@ if sys.version_info < (2, 7): ]) dev_extras = [ - 'nose', + 'pytest', + 'pytest-xdist', 'tox', ] diff --git a/certbot-apache/certbot_apache/tests/augeas_configurator_test.py b/certbot-apache/certbot_apache/tests/augeas_configurator_test.py index 742afce2d..c121ecdf3 100644 --- a/certbot-apache/certbot_apache/tests/augeas_configurator_test.py +++ b/certbot-apache/certbot_apache/tests/augeas_configurator_test.py @@ -13,7 +13,6 @@ from certbot_apache.tests import util class AugeasConfiguratorTest(util.ApacheTest): """Test for Augeas Configurator base class.""" - _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ super(AugeasConfiguratorTest, self).setUp() diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7c6b071da..90561d6ad 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -30,7 +30,6 @@ from certbot_apache.tests import util class MultipleVhostsTest(util.ApacheTest): """Test two standard well-configured HTTP vhosts.""" - _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ super(MultipleVhostsTest, self).setUp() @@ -1369,7 +1368,6 @@ class MultipleVhostsTest(util.ApacheTest): class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access - _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ td = "debian_apache_2_4/augeas_vhosts" diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py index 11c8c57d5..0fdacf4ad 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -5,7 +5,6 @@ import unittest import digitalocean import mock -import six from certbot import errors from certbot.plugins import dns_test_common @@ -134,8 +133,8 @@ class DigitalOceanClientTest(unittest.TestCase): correct_record_mock.destroy.assert_called() - six.assertCountEqual(self, first_record_mock.destroy.call_args_list, []) - six.assertCountEqual(self, last_record_mock.destroy.call_args_list, []) + self.assertFalse(first_record_mock.destroy.call_args_list) + self.assertFalse(last_record_mock.destroy.call_args_list) def test_del_txt_record_error_finding_domain(self): self.manager.get_all_domains.side_effect = API_ERROR diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index aa94abecb..996bd238b 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -24,7 +24,6 @@ from certbot_nginx.tests import util class NginxConfiguratorTest(util.NginxTest): """Test a semi complex vhost configuration.""" - _multiprocess_can_split_ = True def setUp(self): super(NginxConfiguratorTest, self).setUp() diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index ba5e2775f..2fce412e2 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -26,7 +26,6 @@ PLUGINS = disco.PluginsRegistry.find_all() class TestReadFile(TempDirTestCase): '''Test cli.read_file''' - _multiprocess_can_split_ = True def test_read_file(self): rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo')) @@ -46,7 +45,6 @@ class TestReadFile(TempDirTestCase): class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods '''Test the cli args entrypoint''' - _multiprocess_can_split_ = True def setUp(self): reload_module(cli) @@ -418,7 +416,6 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" - _multiprocess_can_split_ = True def setUp(self): # pylint: disable=protected-access @@ -439,7 +436,6 @@ class DefaultTest(unittest.TestCase): class SetByCliTest(unittest.TestCase): """Tests for certbot.set_by_cli and related functions.""" - _multiprocess_can_split_ = True def setUp(self): reload_module(cli) diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 91dd6f8d6..2d54274f0 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -13,7 +13,6 @@ ocsp: Use -help for summary. class OCSPTest(unittest.TestCase): - _multiprocess_can_split_ = True def setUp(self): from certbot import ocsp diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index df6391758..6c8f775e2 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -43,7 +43,6 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): your test. Check :class:`.cli_test.DuplicateCertTest` for an example. """ - _multiprocess_can_split_ = True def setUp(self): from certbot import storage diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index e1280109b..8c1a4b353 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -5,9 +5,13 @@ FROM centos:6 RUN yum install -y epel-release -# Install pip, sudo and nose: +# Install pip and sudo: RUN yum install -y python-pip sudo -RUN pip install nose +# Use pipstrap to update to a stable and tested version of pip +COPY ./pieces/pipstrap.py /opt +RUN /opt/pipstrap.py +# Pin pytest version for increased stability +RUN pip install pytest==3.2.5 # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea @@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.precise index 5ee32c7cc..71d572315 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.precise @@ -6,13 +6,16 @@ FROM ubuntu:precise # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea -# Install pip, sudo, openssl, and nose: +# Install pip, sudo, and openssl: RUN apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean -ENV PIP_INDEX_URL https://pypi.python.org/simple -RUN pip install nose +# Use pipstrap to update to a stable and tested version of pip +COPY ./pieces/pipstrap.py /opt +RUN /opt/pipstrap.py +# Pin pytest version for increased stability +RUN pip install pytest==3.2.5 # Let that user sudo: RUN sed -i.bkp -e \ @@ -30,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/Dockerfile.trusty b/letsencrypt-auto-source/Dockerfile.trusty index 23e8f26de..e0aacd118 100644 --- a/letsencrypt-auto-source/Dockerfile.trusty +++ b/letsencrypt-auto-source/Dockerfile.trusty @@ -11,11 +11,15 @@ RUN sed -i.bkp -e \ 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \ /etc/sudoers -# Install pip and nose: +# Install pip: RUN apt-get update && \ apt-get -q -y install python-pip && \ apt-get clean -RUN pip install nose +# Use pipstrap to update to a stable and tested version of pip +COPY ./pieces/pipstrap.py /opt +RUN /opt/pipstrap.py +# Pin pytest version for increased stability +RUN pip install pytest==3.2.5 RUN mkdir -p /home/lea/certbot @@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.wheezy index acdb791a4..56948d22a 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.wheezy @@ -6,13 +6,15 @@ FROM debian:wheezy # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea -# Install pip, sudo, openssl, and nose: +# Install pip, sudo, and openssl: RUN apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean - -ENV PIP_INDEX_URL https://pypi.python.org/simple -RUN pip install nose +# Use pipstrap to update to a stable and tested version of pip +COPY ./pieces/pipstrap.py /opt +RUN /opt/pipstrap.py +# Pin pytest version for increased stability +RUN pip install pytest==3.2.5 # Let that user sudo: RUN sed -i.bkp -e \ @@ -30,4 +32,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 5c63325ee..2fa03105d 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -17,10 +17,10 @@ from tempfile import mkdtemp from threading import Thread from unittest import TestCase -from nose.tools import eq_, nottest, ok_ +from pytest import mark -@nottest +@mark.skip def tests_dir(): """Return a path to the "tests" directory.""" return dirname(abspath(__file__)) @@ -279,8 +279,8 @@ class AutoTests(TestCase): # installed, and pip hashes verify: install_le_auto(build_le_auto(version='50.0.0'), le_auto_path) out, err = run_letsencrypt_auto() - ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', - err.strip().splitlines()[-1])) + self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', + err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: self.assertTrue('Upgrading certbot-auto ' in out) self.assertTrue('Creating virtual environment...' in out) @@ -327,7 +327,7 @@ class AutoTests(TestCase): try: out, err = run_le_auto(le_auto_path, venv_dir, base_url) except CalledProcessError as exc: - eq_(exc.returncode, 1) + self.assertEqual(exc.returncode, 1) self.assertTrue("Couldn't verify signature of downloaded " "certbot-auto." in exc.output) else: @@ -348,10 +348,11 @@ class AutoTests(TestCase): try: out, err = run_le_auto(le_auto_path, venv_dir, base_url) except CalledProcessError as exc: - eq_(exc.returncode, 1) + self.assertEqual(exc.returncode, 1) self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES " "FROM THE REQUIREMENTS FILE" in exc.output) - ok_(not exists(venv_dir), + self.assertFalse( + exists(venv_dir), msg="The virtualenv was left around, even though " "installation didn't succeed. We shouldn't do " "this, as it foils our detection of whether we " diff --git a/setup.cfg b/setup.cfg index 3b4dbaf87..a21bab793 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,9 +3,3 @@ universal = 1 [easy_install] zip_ok = false - -[nosetests] -nocapture=1 -cover-package=certbot,acme,certbot_apache,certbot_nginx -cover-erase=1 -cover-tests=1 diff --git a/setup.py b/setup.py index 6ffc9a134..ee108c514 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,9 @@ dev_extras = [ 'astroid==1.3.5', 'coverage', 'ipdb', - 'nose', + 'pytest', + 'pytest-cov', + 'pytest-xdist', 'pylint==1.4.2', # upstream #248 'tox', 'twine', diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 6d67bdf2e..4bed2dd3a 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -10,9 +10,9 @@ LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-boots # change to an empty directory to ensure CWD doesn't affect tests cd $(mktemp -d) -pip install nose +pip install pytest==3.2.5 for module in $MODULES ; do echo testing $module - nosetests -v $module + pytest -v --pyargs $module done diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0de2ea3f8..0edb41c53 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -16,6 +16,10 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" + else + # Work around a bug in pytest/importlib for the deprecated Python 3.3. + # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. + pkg=$(echo "$pkg" | tr - _) fi - nosetests -v $pkg --processes=-1 --process-timeout=100 + pytest --numprocesses auto --quiet --pyargs $pkg done diff --git a/tools/pip_constraints.txt b/tools/pip_constraints.txt index 8970d417c..f2fd3d836 100644 --- a/tools/pip_constraints.txt +++ b/tools/pip_constraints.txt @@ -4,6 +4,7 @@ # invocation, some constraints may be ignored due to pip's lack of dependency # resolution. alabaster==0.7.10 +apipkg==1.4 astroid==1.3.5 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 @@ -15,6 +16,7 @@ decorator==4.1.2 dns-lexicon[dnsmadeeasy]==2.1.11 dnspython==1.15.0 docutils==0.14 +execnet==1.5.0 future==0.16.0 futures==3.1.1 google-api-python-client==1.6.4 @@ -27,7 +29,6 @@ Jinja2==2.9.6 jmespath==0.9.3 logilab-common==1.4.1 MarkupSafe==1.0 -nose==1.3.7 oauth2client==4.1.2 pathlib2==2.3.0 pexpect==4.2.1 @@ -42,6 +43,10 @@ pyasn1==0.3.7 pyasn1-modules==0.1.5 Pygments==2.2.0 pylint==1.4.2 +pytest==3.2.5 +pytest-cov==2.5.1 +pytest-forked==0.2 +pytest-xdist==1.20.1 python-dateutil==2.6.1 python-digitalocean==1.12 PyYAML==3.12 diff --git a/tools/release.sh b/tools/release.sh index 2a8e00aa1..a8de208b5 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -55,7 +55,7 @@ SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs -# - it causes problems when running nosetests - the latter tries to +# - it causes problems when running pytest - the latter tries to # run everything that matches test*, while there are no unittests # there @@ -166,10 +166,10 @@ fi mkdir kgs kgs="kgs/$version" pip freeze | tee $kgs -pip install nose +pip install pytest for module in $subpkgs_modules ; do echo testing $module - nosetests $module + pytest --pyargs $module done cd ~- diff --git a/tox.cover.sh b/tox.cover.sh index fc0c9f476..3f0a5f72e 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -16,7 +16,7 @@ fi cover () { if [ "$1" = "certbot" ]; then - min=98 + min=97 elif [ "$1" = "acme" ]; then min=100 elif [ "$1" = "certbot_apache" ]; then @@ -24,23 +24,23 @@ cover () { elif [ "$1" = "certbot_dns_cloudflare" ]; then min=98 elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=99 + min=98 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 elif [ "$1" = "certbot_dns_dnsimple" ]; then min=98 elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=99 + min=98 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_dns_luadns" ]; then min=98 elif [ "$1" = "certbot_dns_nsone" ]; then - min=99 + min=98 elif [ "$1" = "certbot_dns_rfc2136" ]; then min=99 elif [ "$1" = "certbot_dns_route53" ]; then - min=99 + min=91 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then @@ -50,17 +50,10 @@ cover () { exit 1 fi - # "-c /dev/null" makes sure setup.cfg is not loaded (multiple - # --with-cover add up, --cover-erase must not be set for coveralls - # to get all the data); --with-cover scopes coverage to only - # specific package, positional argument scopes tests only to - # specific package directory; --cover-tests makes sure every tests - # is run (c.f. #403) - nosetests -c /dev/null --with-cover --cover-tests --cover-package \ - "$1" --cover-min-percentage="$min" "$1" + pytest --cov "$1" --cov-report term-missing \ + --cov-fail-under "$min" --numprocesses auto --pyargs "$1" } -rm -f .coverage # --cover-erase is off, make sure stats are correct for pkg in $pkgs do cover $pkg diff --git a/tox.ini b/tox.ini index dee14b8b3..c6dc61155 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,6 @@ skipsdist = true envlist = modification,py{26,33,34,35,36},cover,lint -# nosetest -v => more verbose output, allows to detect busy waiting -# loops, especially on Travis - [base] # pip installs the requested packages in editable mode pip_install = {toxinidir}/tools/pip_install_editable.sh @@ -96,6 +93,7 @@ deps = pyasn1==0.1.9 pyparsing==1.5.6 pyrfc3339==1.0 + pytest==3.2.5 python-augeas==0.4.1 pytz==2012c requests[security]==2.6.0 From b9b329ecf7f3a285758a2f0e69c5f35e4c1e0259 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Dec 2017 13:20:27 -0800 Subject: [PATCH 225/631] pin pkging tools that have dropped support (#5281) --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tox.ini b/tox.ini index c6dc61155..bb421daa5 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,9 @@ source_paths = commands = {[base]install_and_test} {[base]py26_packages} python tests/lock_test.py +deps = + setuptools==36.8.0 + wheel==0.29.0 [testenv] commands = @@ -68,6 +71,12 @@ setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 +[testenv:py33] +commands = + {[testenv]commands} +deps = + wheel==0.29.0 + [testenv:py27-oldest] commands = {[testenv]commands} From 8ce6ee5f3e976c7cc795ba3049b3f77367d3b557 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 1 Dec 2017 16:10:16 -0800 Subject: [PATCH 226/631] Remove all but one BOULDER_INTEGRATION, and macOS (#5270) These tests are retained in the test-everything branch, which has a Travis cron job to run nightly. Removing these speeds up the Certbot Travis builds dramatically for two reasons: - The Boulder integration tests are slow (10-12 minutes), and it's exceedingly rare for them to fail on one Python environment but not another. - The macOS tests take a very long time to run, because they need to wait for build slots on the limited number of macOS instances, which are often in high demand. --- .travis.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index d331dc980..359801622 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,15 +13,15 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=cover + env: TOXENV=cover FYI="this also tests py27" - python: "2.7" env: TOXENV=lint - python: "2.7" - env: TOXENV=py27-oldest BOULDER_INTEGRATION=1 + env: TOXENV=py27-oldest sudo: required services: docker - python: "2.6" - env: TOXENV=py26 BOULDER_INTEGRATION=1 + env: TOXENV=py26 sudo: required services: docker - python: "2.7" @@ -67,29 +67,23 @@ matrix: env: TOXENV=apacheconftest sudo: required - python: "3.3" - env: TOXENV=py33 BOULDER_INTEGRATION=1 + env: TOXENV=py33 sudo: required services: docker - python: "3.4" - env: TOXENV=py34 BOULDER_INTEGRATION=1 + env: TOXENV=py34 sudo: required services: docker - python: "3.5" - env: TOXENV=py35 BOULDER_INTEGRATION=1 + env: TOXENV=py35 sudo: required services: docker - python: "3.6" - env: TOXENV=py36 BOULDER_INTEGRATION=1 + env: TOXENV=py36 sudo: required services: docker - python: "2.7" env: TOXENV=nginxroundtrip - - language: generic - env: TOXENV=py27 - os: osx - - language: generic - env: TOXENV=py36 - os: osx # Only build pushes to the master branch, PRs, and branches beginning with From 394dafd38c91818582ec55372f887108762f32eb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Dec 2017 17:00:24 -0800 Subject: [PATCH 227/631] Revert requiring dnsmadeeasy extras for lexicon (#5291) Fixes failures at https://travis-ci.org/certbot/certbot/jobs/310248574#L1558. Additional context can be found at #5230 and https://github.com/AnalogJ/lexicon/commit/604584521a487dd0b8814320b3f1af52b82ad205#diff-2eeaed663bd0d25b7e608891384b7298. --- certbot-dns-dnsmadeeasy/setup.py | 4 +--- tools/pip_constraints.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 1ddec208b..88e02304e 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -10,9 +10,7 @@ version = '0.20.0.dev0' install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), - # new versions of lexicon require that we install dnsmadeeasy extras and - # 2.1.11 is the first version that defines them. - 'dns-lexicon[dnsmadeeasy]>=2.1.11', + 'dns-lexicon', 'mock', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: diff --git a/tools/pip_constraints.txt b/tools/pip_constraints.txt index f2fd3d836..cacec37d6 100644 --- a/tools/pip_constraints.txt +++ b/tools/pip_constraints.txt @@ -13,7 +13,7 @@ botocore==1.7.41 cloudflare==1.8.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon[dnsmadeeasy]==2.1.11 +dns-lexicon==2.1.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 From 7319cc975a1bb41102a948399cd1a09b4c90b17b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 1 Dec 2017 23:40:09 -0800 Subject: [PATCH 228/631] Quiet pip install output. (#5288) pip install generates a lot of lines of output that make it harder to see what tox is running in general. This adds the -q flag to pip install. At the same time, add `set -x` in install_and_test.sh and pip_install.sh so they echo the commands they are running. This makes it a little clearer what's going on in tests. I didn't put `set -x` at the top or in the shebang, because moving it lower lets us avoid echoing some of the messy if/then setup statements in these scripts, which focussed attention on the pip install command. --- tools/install_and_test.sh | 3 ++- tools/pip_install.sh | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0edb41c53..d57f0974e 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -6,11 +6,12 @@ # constraints. if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -e" + pip_install="pip install -q -e" else pip_install="$(dirname $0)/pip_install_editable.sh" fi +set -x for requirement in "$@" ; do $pip_install $requirement pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] diff --git a/tools/pip_install.sh b/tools/pip_install.sh index 194501c7d..fafd58e54 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -11,5 +11,7 @@ trap "rm -f $certbot_auto_constraints" EXIT sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints dev_constraints="$(dirname $my_path)/pip_constraints.txt" +set -x + # install the requested packages using the pinned requirements as constraints -pip install --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" +pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" From abdde886fa3d6361336a2fb1d7e091b53be4b6f2 Mon Sep 17 00:00:00 2001 From: Eccenux Date: Sat, 2 Dec 2017 12:25:58 +0100 Subject: [PATCH 229/631] code style --- certbot/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index 11f7ddab7..cee381cbd 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -254,7 +254,7 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): "possible certificate names.".format(certname)) def _get_added_removed(after, before): - """Get lists of items removed from `before` + """Get lists of items removed from `before` and a lists of items added to `after` """ added = list(set(after) - set(before)) From 840c94371111ead7f20be925688ad2fb6d931551 Mon Sep 17 00:00:00 2001 From: Eccenux Date: Sat, 2 Dec 2017 12:28:53 +0100 Subject: [PATCH 230/631] W:266,28: Redefining built-in 'list' (redefined-builtin) --- certbot/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index cee381cbd..f61c70b05 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -263,10 +263,10 @@ def _get_added_removed(after, before): removed.sort() return added, removed -def _format_list(character, list): +def _format_list(character, strings): """Format list with given character """ - formatted = "{br}{ch} " + "{br}{ch} ".join(list) + formatted = "{br}{ch} " + "{br}{ch} ".join(strings) return formatted.format( ch=character, br=os.linesep From 73ba9af4422b61ad2b45ed9d3f1e8d19b5f9167a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 4 Dec 2017 11:20:53 -0800 Subject: [PATCH 231/631] Don't echo Boulder logs on failure. (#5290) The extensive logs made it hard to spot the actual failure. --- tests/boulder-integration.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 7afba19df..1e0b7754b 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -20,14 +20,6 @@ cleanup_and_exit() { echo Kill server subprocess, left running by abnormal exit kill $SERVER_STILL_RUNNING fi - # Dump boulder logs in case they contain useful debugging information. - : "------------------ ------------------ ------------------" - : "------------------ begin boulder logs ------------------" - : "------------------ ------------------ ------------------" - docker logs boulder_boulder_1 - : "------------------ ------------------ ------------------" - : "------------------ end boulder logs ------------------" - : "------------------ ------------------ ------------------" if [ -f "$HOOK_DIRS_TEST" ]; then rm -f "$HOOK_DIRS_TEST" fi From dc78fd731ee2b5b88e140f99242b4267f2fd0f27 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Dec 2017 21:49:18 +0200 Subject: [PATCH 232/631] Distribution specific override functionality based on class inheritance (#5202) Class inheritance based approach to distro specific overrides. How it works: The certbot-apache plugin entrypoint has been changed to entrypoint.ENTRYPOINT which is a variable containing appropriate override class for system, if available. Override classes register themselves using decorator override.register() which takes a list of distribution fingerprints (ID & LIKE variables in /etc/os-release, or platform.linux_distribution() as a fallback). These end up as keys in dict override.OVERRIDE_CLASSES and values for the keys are references to the class that called the decorator, hence allowing self-registration of override classes when they are imported. The only file importing these override classes is entrypoint.py, so adding new override classes would need only one import in addition to the actual override class file. Generic changes: Parser initialization has been moved to separate class method, allowing easy override where needed. Cleaned up configurator.py a bit, and moved some helper functions to newly created apache_util.py Split Debian specific code from configurator.py to debian_override.py Changed define_cmd to apache_cmd because the parameters are for every distribution supporting this behavior, and we're able to use the value to build the additional configuration dump commands. Moved add_parser_mod() from configurator to parser add_mod() Added two new configuration dump parsing methods to update_runtime_variables() in parser: update_includes() and update_modules(). Changed init_modules() in parser to accommodate the changes above. (ie. don't throw existing self.modules out). Moved OS based constants to their respective override classes. Refactored configurator class discovery in tests to help easier test case creation using distribution based override configurator class. tests.util.get_apache_configurator() now takes keyword argument os_info which is string of the desired mock OS fingerprint response that's used for picking the right override class. This PR includes two major generic additions that should vastly improve our parsing accuracy and quality: Includes are parsed from config dump from httpd binary. This is mandatory for some distributions (Like OpenSUSE) to get visibility over the whole configuration tree because of Include statements passed on in command line, and not via root httpd.conf file. Modules are parsed from config dump from httpd binary. This lets us jump into correct IfModule directives if for some reason we have missed the module availability (because of one being included on command line or such). Distribution specific changes Because of the generic changes, there are two distributions (or distribution families) that do not provide such functionality, so it had to be overridden in their respective override files. These distributions are: CentOS, because it deliberately limits httpd binary stdout using SELinux as a feature. We are doing opportunistic config dumps here however, in case SELinux enforcing is off. Gentoo, because it does not provide a way to invoke httpd with command line parsed from its specific configuration file. Gentoo relies heavily on Define statements that are passed over from APACHE2_OPTS variable /etc/conf.d/apache2 file and most of the configuration in root Apache configuration are dependent on these values. Debian Moved the Debian specific parts from configurator.py to Debian specific override. CentOS Parsing of /etc/sysconfig/httpd file for additional Define statements. This could hold other parameters too, but parsing everything off it would require a full Apache lexer. For CLI parameters, I think Defines are the most common ones. This is done in addition of opportunistic parsing of httpd binary config dump. Added CentOS default Apache configuration tree for realistic test cases. Gentoo Parsing Defines from /etc/conf.d/apache2 variable APACHE2_OPTS, which holds additional Define statements to enable certain functionalities, enabling parts of the configuration in the Apache2 DOM. This is done instead of trying to parse httpd binary configuration dumps. Added default Apache configuration from Gentoo to testdata, including /etc/conf.d/apache2 file for realistic test cases. * Distribution specific override functionality based on class inheritance * Need to patch get_systemd_os_like to as travis has proper os-release * Added pydoc * Move parser initialization to a method and fix Python 3 __new__ errors * Parser changes to parse HTTPD config * Try to get modules and includes from httpd process for better visibility over the configuration * Had to disable duplicate-code because of test setup (PyCQA/pylint/issues/214) * CentOS tests and linter fixes * Gentoo override, tests and linter fixes * Mock the process call in all the tests that require it * Fix CentOS test mock * Restore reseting modules list functionality for cleanup * Move OS fingerprinting and constant mocks to parent class * Fixes requested in review * New entrypoint structure and started moving OS constants to override classes * OS constants move continued, test and linter fixes * Removed dead code * Apache compatibility test changest to reflect OS constant restructure * Test fix * Requested changes * Moved Debian specific tests to own test file * Removed decorator based override class registration in favor of entrypoint dict * Fix for update_includes for some versions of Augeas * Take fedora fix into account in tests * Review fixes --- .pylintrc | 2 +- certbot-apache/certbot_apache/apache_util.py | 96 +++++ certbot-apache/certbot_apache/configurator.py | 282 ++++--------- certbot-apache/certbot_apache/constants.py | 181 -------- certbot-apache/certbot_apache/entrypoint.py | 47 +++ .../certbot_apache/override_arch.py | 31 ++ .../certbot_apache/override_centos.py | 59 +++ .../certbot_apache/override_darwin.py | 31 ++ .../certbot_apache/override_debian.py | 144 +++++++ .../certbot_apache/override_gentoo.py | 58 +++ .../certbot_apache/override_suse.py | 31 ++ certbot-apache/certbot_apache/parser.py | 123 ++++-- .../certbot_apache/tests/centos_test.py | 123 ++++++ .../tests/complex_parsing_test.py | 2 +- .../certbot_apache/tests/configurator_test.py | 329 +++++---------- .../certbot_apache/tests/constants_test.py | 44 -- .../certbot_apache/tests/debian_test.py | 209 ++++++++++ .../certbot_apache/tests/entrypoint_test.py | 41 ++ .../certbot_apache/tests/gentoo_test.py | 86 ++++ .../certbot_apache/tests/parser_test.py | 125 +++++- .../centos7_apache/apache/httpd/conf.d/README | 9 + .../apache/httpd/conf.d/autoindex.conf | 94 +++++ .../httpd/conf.d/centos.example.com.conf | 7 + .../apache/httpd/conf.d/ssl.conf | 211 ++++++++++ .../apache/httpd/conf.d/userdir.conf | 36 ++ .../apache/httpd/conf.d/welcome.conf | 22 + .../apache/httpd/conf.modules.d/00-base.conf | 77 ++++ .../apache/httpd/conf.modules.d/00-dav.conf | 3 + .../apache/httpd/conf.modules.d/00-lua.conf | 1 + .../apache/httpd/conf.modules.d/00-mpm.conf | 19 + .../apache/httpd/conf.modules.d/00-proxy.conf | 16 + .../apache/httpd/conf.modules.d/00-ssl.conf | 1 + .../httpd/conf.modules.d/00-systemd.conf | 2 + .../apache/httpd/conf.modules.d/01-cgi.conf | 14 + .../apache/httpd/conf/httpd.conf | 353 ++++++++++++++++ .../centos7_apache/apache/httpd/conf/magic | 385 ++++++++++++++++++ .../testdata/centos7_apache/apache/sites | 1 + .../centos7_apache/apache/sysconfig/httpd | 25 ++ .../gentoo_apache/apache/apache2/httpd.conf | 157 +++++++ .../gentoo_apache/apache/apache2/magic | 385 ++++++++++++++++++ .../modules.d/.keep_www-servers_apache-2 | 0 .../modules.d/00_default_settings.conf | 131 ++++++ .../apache2/modules.d/00_error_documents.conf | 57 +++ .../apache2/modules.d/00_languages.conf | 133 ++++++ .../apache2/modules.d/00_mod_autoindex.conf | 85 ++++ .../apache/apache2/modules.d/00_mod_info.conf | 10 + .../apache2/modules.d/00_mod_log_config.conf | 35 ++ .../apache/apache2/modules.d/00_mod_mime.conf | 46 +++ .../apache2/modules.d/00_mod_status.conf | 15 + .../apache2/modules.d/00_mod_userdir.conf | 32 ++ .../apache/apache2/modules.d/00_mpm.conf | 99 +++++ .../apache2/modules.d/10_mod_mem_cache.conf | 10 + .../apache/apache2/modules.d/40_mod_ssl.conf | 67 +++ .../apache2/modules.d/41_mod_http2.conf | 9 + .../apache/apache2/modules.d/45_mod_dav.conf | 19 + .../apache/apache2/modules.d/46_mod_ldap.conf | 18 + .../vhosts.d/.keep_www-servers_apache-2 | 0 .../vhosts.d/00_default_ssl_vhost.conf | 191 +++++++++ .../apache2/vhosts.d/00_default_vhost.conf | 45 ++ .../apache2/vhosts.d/default_vhost.include | 71 ++++ .../apache2/vhosts.d/gentoo.example.com.conf | 7 + .../gentoo_apache/apache/conf.d/apache2 | 74 ++++ .../tests/testdata/gentoo_apache/apache/sites | 3 + .../certbot_apache/tests/tls_sni_01_test.py | 16 +- certbot-apache/certbot_apache/tests/util.py | 70 +++- certbot-apache/setup.py | 2 +- .../configurators/apache/common.py | 10 +- certbot/util.py | 14 +- 68 files changed, 4383 insertions(+), 748 deletions(-) create mode 100644 certbot-apache/certbot_apache/apache_util.py create mode 100644 certbot-apache/certbot_apache/entrypoint.py create mode 100644 certbot-apache/certbot_apache/override_arch.py create mode 100644 certbot-apache/certbot_apache/override_centos.py create mode 100644 certbot-apache/certbot_apache/override_darwin.py create mode 100644 certbot-apache/certbot_apache/override_debian.py create mode 100644 certbot-apache/certbot_apache/override_gentoo.py create mode 100644 certbot-apache/certbot_apache/override_suse.py create mode 100644 certbot-apache/certbot_apache/tests/centos_test.py delete mode 100644 certbot-apache/certbot_apache/tests/constants_test.py create mode 100644 certbot-apache/certbot_apache/tests/debian_test.py create mode 100644 certbot-apache/certbot_apache/tests/entrypoint_test.py create mode 100644 certbot-apache/certbot_apache/tests/gentoo_test.py create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/README create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/autoindex.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/centos.example.com.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/ssl.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/userdir.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/welcome.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-base.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-dav.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-lua.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-mpm.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-proxy.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-ssl.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-systemd.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/01-cgi.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/httpd.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/magic create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sites create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/httpd.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/magic create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_default_settings.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_error_documents.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_languages.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_autoindex.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_info.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_log_config.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_mime.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_status.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_userdir.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mpm.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/10_mod_mem_cache.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/40_mod_ssl.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/41_mod_http2.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/45_mod_dav.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/46_mod_ldap.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_ssl_vhost.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_vhost.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/default_vhost.include create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/gentoo.example.com.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/conf.d/apache2 create mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/sites diff --git a/.pylintrc b/.pylintrc index d510fddfd..36d8c286f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -41,7 +41,7 @@ load-plugins=linter_plugin # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import +disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code # abstract-class-not-used cannot be disabled locally (at least in # pylint 1.4.1), same for abstract-class-little-used diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py new file mode 100644 index 000000000..b4a24f137 --- /dev/null +++ b/certbot-apache/certbot_apache/apache_util.py @@ -0,0 +1,96 @@ +""" Utility functions for certbot-apache plugin """ +import os + +from certbot import util + +def get_mod_deps(mod_name): + """Get known module dependencies. + + .. note:: This does not need to be accurate in order for the client to + run. This simply keeps things clean if the user decides to revert + changes. + .. warning:: If all deps are not included, it may cause incorrect parsing + behavior, due to enable_mod's shortcut for updating the parser's + currently defined modules (`.ApacheParser.add_mod`) + This would only present a major problem in extremely atypical + configs that use ifmod for the missing deps. + + """ + deps = { + "ssl": ["setenvif", "mime"] + } + return deps.get(mod_name, []) + + +def get_file_path(vhost_path): + """Get file path from augeas_vhost_path. + + Takes in Augeas path and returns the file name + + :param str vhost_path: Augeas virtual host path + + :returns: filename of vhost + :rtype: str + + """ + if not vhost_path or not vhost_path.startswith("/files/"): + return None + + return _split_aug_path(vhost_path)[0] + + +def get_internal_aug_path(vhost_path): + """Get the Augeas path for a vhost with the file path removed. + + :param str vhost_path: Augeas virtual host path + + :returns: Augeas path to vhost relative to the containing file + :rtype: str + + """ + return _split_aug_path(vhost_path)[1] + + +def _split_aug_path(vhost_path): + """Splits an Augeas path into a file path and an internal path. + + After removing "/files", this function splits vhost_path into the + file path and the remaining Augeas path. + + :param str vhost_path: Augeas virtual host path + + :returns: file path and internal Augeas path + :rtype: `tuple` of `str` + + """ + # Strip off /files + file_path = vhost_path[6:] + internal_path = [] + + # Remove components from the end of file_path until it becomes valid + while not os.path.exists(file_path): + file_path, _, internal_path_part = file_path.rpartition("/") + internal_path.append(internal_path_part) + + return file_path, "/".join(reversed(internal_path)) + + +def parse_define_file(filepath, varname): + """ Parses Defines from a variable in configuration file + + :param str filepath: Path of file to parse + :param str varname: Name of the variable + + :returns: Dict of Define:Value pairs + :rtype: `dict` + + """ + return_vars = {} + # Get list of words in the variable + a_opts = util.get_var_from_file(varname, filepath).split() + for i, v in enumerate(a_opts): + # Handle Define statements and make sure it has an argument + if v == "-D" and len(a_opts) >= i+2: + var_parts = a_opts[i+1].partition("=") + return_vars[var_parts[0]] = var_parts[2] + return return_vars diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8b6c578d6..5a33346ea 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -3,6 +3,7 @@ import fnmatch import logging import os +import pkg_resources import re import socket import time @@ -19,6 +20,7 @@ from certbot import util from certbot.plugins import common from certbot.plugins.util import path_surgery +from certbot_apache import apache_util from certbot_apache import augeas_configurator from certbot_apache import constants from certbot_apache import display_ops @@ -85,27 +87,50 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): description = "Apache Web Server plugin - Beta" + OS_DEFAULTS = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + logs_root="/var/log/apache2", + version_cmd=['apache2ctl', '-v'], + apache_cmd="apache2ctl", + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) + + def constant(self, key): + """Get constant for OS_DEFAULTS""" + return self.OS_DEFAULTS.get(key) + @classmethod def add_parser_arguments(cls, add): - add("enmod", default=constants.os_constant("enmod"), + add("enmod", default=cls.OS_DEFAULTS["enmod"], help="Path to the Apache 'a2enmod' binary.") - add("dismod", default=constants.os_constant("dismod"), + add("dismod", default=cls.OS_DEFAULTS["dismod"], help="Path to the Apache 'a2dismod' binary.") - add("le-vhost-ext", default=constants.os_constant("le_vhost_ext"), + add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], help="SSL vhost configuration extension.") - add("server-root", default=constants.os_constant("server_root"), + add("server-root", default=cls.OS_DEFAULTS["server_root"], help="Apache server root directory.") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=constants.os_constant("logs_root"), + add("logs-root", default=cls.OS_DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", - default=constants.os_constant("challenge_location"), + default=cls.OS_DEFAULTS["challenge_location"], help="Directory path for challenge configuration.") - add("handle-modules", default=constants.os_constant("handle_mods"), + add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], help="Let installer handle enabling required modules for you." + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=constants.os_constant("handle_sites"), + add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], help="Let installer handle enabling sites for you." + "(Only Ubuntu/Debian currently)") util.add_deprecated_argument(add, argument_name="ctl", nargs=1) @@ -166,7 +191,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.NoInstallationError("Problem in Augeas installation") # Verify Apache is installed - restart_cmd = constants.os_constant("restart_cmd")[0] + restart_cmd = self.constant("restart_cmd")[0] if not util.exe_exists(restart_cmd): if not path_surgery(restart_cmd): raise errors.NoInstallationError( @@ -192,20 +217,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Parse vhost-root if defined on cli if not self.conf("vhost-root"): - self.vhostroot = constants.os_constant("vhost_root") + self.vhostroot = self.constant("vhost_root") else: self.vhostroot = os.path.abspath(self.conf("vhost-root")) - self.parser = parser.ApacheParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), - self.version, configurator=self) + self.parser = self.get_parser() + # Check for errors in parsing files with Augeas self.check_parsing_errors("httpd.aug") # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() - install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest) + self.install_ssl_options_conf(self.mod_ssl_conf, + self.updated_mod_ssl_conf_digest) # Prevent two Apache plugins from modifying a config at once try: @@ -230,6 +255,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug.remove("/test/path") return matches + def get_parser(self): + """Initializes the ApacheParser""" + return parser.ApacheParser( + self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.version, configurator=self) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -585,7 +616,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if addr.get_port() == "443": is_ssl = True - filename = get_file_path(self.aug.get("/augeas/files%s/path" % get_file_path(path))) + filename = apache_util.get_file_path( + self.aug.get("/augeas/files%s/path" % apache_util.get_file_path(path))) if filename is None: return None @@ -624,7 +656,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): new_vhost = self._create_vhost(path) if not new_vhost: continue - internal_path = get_internal_aug_path(new_vhost.path) + internal_path = apache_util.get_internal_aug_path(new_vhost.path) realpath = os.path.realpath(new_vhost.filep) if realpath not in file_paths: file_paths[realpath] = new_vhost.filep @@ -640,7 +672,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for v in vhs: if v.filep == file_paths[realpath]: internal_paths[realpath].remove( - get_internal_aug_path(v.path)) + apache_util.get_internal_aug_path(v.path)) else: new_vhs.append(v) vhs = new_vhs @@ -828,7 +860,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``certbot_apache.constants.os_constant("le_vhost_ext")`` + ``self.constant("le_vhost_ext")`` .. note:: This function saves the configuration @@ -1702,17 +1734,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return redirects - def enable_site(self, vhost): """Enables an available site, Apache reload required. .. note:: Does not make sure that the site correctly works or that all modules are enabled appropriately. - - .. todo:: This function should number subdomains before the domain - vhost - - .. todo:: Make sure link is not broken... + .. note:: The distribution specific override replaces functionality + of this method where available. :param vhost: vhost to enable :type vhost: :class:`~certbot_apache.obj.VirtualHost` @@ -1724,39 +1752,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if vhost.enabled: return - # Handle non-debian systems - if not self.conf("handle-sites"): - if not self.parser.parsed_in_original(vhost.filep): - # Add direct include to root conf - self.parser.add_include(self.parser.loc["default"], vhost.filep) - vhost.enabled = True - return + if not self.parser.parsed_in_original(vhost.filep): + # Add direct include to root conf + logger.info("Enabling site %s by adding Include to root configuration", + vhost.filep) + self.save_notes += "Enabled site %s\n" % vhost.filep + self.parser.add_include(self.parser.loc["default"], vhost.filep) + vhost.enabled = True + return - enabled_path = ("%s/sites-enabled/%s" % - (self.parser.root, os.path.basename(vhost.filep))) - self.reverter.register_file_creation(False, enabled_path) - try: - os.symlink(vhost.filep, enabled_path) - except OSError as err: - if os.path.islink(enabled_path) and os.path.realpath( - enabled_path) == vhost.filep: - # Already in shape - vhost.enabled = True - return - else: - logger.warning( - "Could not symlink %s to %s, got error: %s", enabled_path, - vhost.filep, err.strerror) - errstring = ("Encountered error while trying to enable a " + - "newly created VirtualHost located at {0} by " + - "linking to it from {1}") - raise errors.NotSupportedError(errstring.format(vhost.filep, - enabled_path)) - vhost.enabled = True - logger.info("Enabling available site: %s", vhost.filep) - self.save_notes += "Enabled site %s\n" % vhost.filep - - def enable_mod(self, mod_name, temp=False): + def enable_mod(self, mod_name, temp=False): # pylint: disable=unused-argument """Enables module in Apache. Both enables and reloads Apache so module is active. @@ -1764,64 +1769,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str mod_name: Name of the module to enable. (e.g. 'ssl') :param bool temp: Whether or not this is a temporary action. - :raises .errors.NotSupportedError: If the filesystem layout is not - supported. - :raises .errors.MisconfigurationError: If a2enmod or a2dismod cannot be - run. + .. note:: The distribution specific override replaces functionality + of this method where available. + + :raises .errors.MisconfigurationError: We cannot enable modules in + generic fashion. """ - # Support Debian specific setup - avail_path = os.path.join(self.parser.root, "mods-available") - enabled_path = os.path.join(self.parser.root, "mods-enabled") - if not os.path.isdir(avail_path) or not os.path.isdir(enabled_path): - raise errors.NotSupportedError( - "Unsupported directory layout. You may try to enable mod %s " - "and try again." % mod_name) - - deps = _get_mod_deps(mod_name) - - # Enable all dependencies - for dep in deps: - if (dep + "_module") not in self.parser.modules: - self._enable_mod_debian(dep, temp) - self._add_parser_mod(dep) - - note = "Enabled dependency of %s module - %s" % (mod_name, dep) - if not temp: - self.save_notes += note + os.linesep - logger.debug(note) - - # Enable actual module - self._enable_mod_debian(mod_name, temp) - self._add_parser_mod(mod_name) - - if not temp: - self.save_notes += "Enabled %s module in Apache\n" % mod_name - logger.info("Enabled Apache %s module", mod_name) - - # Modules can enable additional config files. Variables may be defined - # within these new configuration sections. - # Reload is not necessary as DUMP_RUN_CFG uses latest config. - self.parser.update_runtime_variables() - - def _add_parser_mod(self, mod_name): - """Shortcut for updating parser modules.""" - self.parser.modules.add(mod_name + "_module") - self.parser.modules.add("mod_" + mod_name + ".c") - - def _enable_mod_debian(self, mod_name, temp): - """Assumes mods-available, mods-enabled layout.""" - # Generate reversal command. - # Try to be safe here... check that we can probably reverse before - # applying enmod command - if not util.exe_exists(self.conf("dismod")): - raise errors.MisconfigurationError( - "Unable to find a2dismod, please make sure a2enmod and " - "a2dismod are configured correctly for certbot.") - - self.reverter.register_undo_command( - temp, [self.conf("dismod"), mod_name]) - util.run_script([self.conf("enmod"), mod_name]) + mod_message = ("Apache needs to have module \"{0}\" active for the " + + "requested installation options. Unfortunately Certbot is unable " + + "to install or enable it for you. Please install the module, and " + + "run Certbot again.") + raise errors.MisconfigurationError(mod_message.format(mod_name)) def restart(self): """Runs a config test and reloads the Apache server. @@ -1840,7 +1799,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - util.run_script(constants.os_constant("restart_cmd")) + util.run_script(self.constant("restart_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1851,7 +1810,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - util.run_script(constants.os_constant("conftest_cmd")) + util.run_script(self.constant("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1867,11 +1826,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = util.run_script(constants.os_constant("version_cmd")) + stdout, _ = util.run_script(self.constant("version_cmd")) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - constants.os_constant("version_cmd")) + self.constant("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) @@ -1943,86 +1902,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self._chall_out: self.revert_challenge_config() self.restart() - self.parser.init_modules() + self.parser.reset_modules() + + def install_ssl_options_conf(self, options_ssl, options_ssl_digest): + """Copy Certbot's SSL options file into the system's config dir if required.""" + + # XXX if we ever try to enforce a local privilege boundary (eg, running + # certbot for unprivileged users via setuid), this function will need + # to be modified. + return common.install_version_controlled_file(options_ssl, options_ssl_digest, + self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) -def _get_mod_deps(mod_name): - """Get known module dependencies. - - .. note:: This does not need to be accurate in order for the client to - run. This simply keeps things clean if the user decides to revert - changes. - .. warning:: If all deps are not included, it may cause incorrect parsing - behavior, due to enable_mod's shortcut for updating the parser's - currently defined modules (`.ApacheConfigurator._add_parser_mod`) - This would only present a major problem in extremely atypical - configs that use ifmod for the missing deps. - - """ - deps = { - "ssl": ["setenvif", "mime"] - } - return deps.get(mod_name, []) - - -def get_file_path(vhost_path): - """Get file path from augeas_vhost_path. - - Takes in Augeas path and returns the file name - - :param str vhost_path: Augeas virtual host path - - :returns: filename of vhost - :rtype: str - - """ - if not vhost_path or not vhost_path.startswith("/files/"): - return None - - return _split_aug_path(vhost_path)[0] - - -def get_internal_aug_path(vhost_path): - """Get the Augeas path for a vhost with the file path removed. - - :param str vhost_path: Augeas virtual host path - - :returns: Augeas path to vhost relative to the containing file - :rtype: str - - """ - return _split_aug_path(vhost_path)[1] - - -def _split_aug_path(vhost_path): - """Splits an Augeas path into a file path and an internal path. - - After removing "/files", this function splits vhost_path into the - file path and the remaining Augeas path. - - :param str vhost_path: Augeas virtual host path - - :returns: file path and internal Augeas path - :rtype: `tuple` of `str` - - """ - # Strip off /files - file_path = vhost_path[6:] - internal_path = [] - - # Remove components from the end of file_path until it becomes valid - while not os.path.exists(file_path): - file_path, _, internal_path_part = file_path.rpartition("/") - internal_path.append(internal_path_part) - - return file_path, "/".join(reversed(internal_path)) - - -def install_ssl_options_conf(options_ssl, options_ssl_digest): - """Copy Certbot's SSL options file into the system's config dir if required.""" - - # XXX if we ever try to enforce a local privilege boundary (eg, running - # certbot for unprivileged users via setuid), this function will need - # to be modified. - return common.install_version_controlled_file(options_ssl, options_ssl_digest, - constants.os_constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index 0c5f7263a..a13ca04a6 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -1,151 +1,6 @@ """Apache plugin constants.""" import pkg_resources -from certbot import util -CLI_DEFAULTS_DEFAULT = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - version_cmd=['apache2ctl', '-v'], - define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS_DEBIAN = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - version_cmd=['apache2ctl', '-v'], - define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod="a2enmod", - dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", - handle_mods=True, - handle_sites=True, - challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS_CENTOS = dict( - server_root="/etc/httpd", - vhost_root="/etc/httpd/conf.d", - vhost_files="*.conf", - logs_root="/var/log/httpd", - version_cmd=['apachectl', '-v'], - define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apachectl', 'graceful'], - conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "centos-options-ssl-apache.conf") -) -CLI_DEFAULTS_GENTOO = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/vhosts.d", - vhost_files="*.conf", - logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/apache2', '-v'], - define_cmd=['apache2ctl', 'virtualhosts'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS_DARWIN = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/other", - vhost_files="*.conf", - logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/httpd', '-v'], - define_cmd=['/usr/sbin/httpd', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apachectl', 'graceful'], - conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/apache2/other", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS_SUSE = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/vhosts.d", - vhost_files="*.conf", - logs_root="/var/log/apache2", - version_cmd=['apache2ctl', '-v'], - define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod="a2enmod", - dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS_ARCH = dict( - server_root="/etc/httpd", - vhost_root="/etc/httpd/conf", - vhost_files="*.conf", - logs_root="/var/log/httpd", - version_cmd=['apachectl', '-v'], - define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], - restart_cmd=['apachectl', 'graceful'], - conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_mods=False, - handle_sites=False, - challenge_location="/etc/httpd/conf", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", "options-ssl-apache.conf") -) -CLI_DEFAULTS = { - "default": CLI_DEFAULTS_DEFAULT, - "debian": CLI_DEFAULTS_DEBIAN, - "ubuntu": CLI_DEFAULTS_DEBIAN, - "centos": CLI_DEFAULTS_CENTOS, - "centos linux": CLI_DEFAULTS_CENTOS, - "fedora": CLI_DEFAULTS_CENTOS, - "red hat enterprise linux server": CLI_DEFAULTS_CENTOS, - "rhel": CLI_DEFAULTS_CENTOS, - "amazon": CLI_DEFAULTS_CENTOS, - "gentoo": CLI_DEFAULTS_GENTOO, - "gentoo base system": CLI_DEFAULTS_GENTOO, - "darwin": CLI_DEFAULTS_DARWIN, - "opensuse": CLI_DEFAULTS_SUSE, - "suse": CLI_DEFAULTS_SUSE, - "arch": CLI_DEFAULTS_ARCH, -} -"""CLI defaults.""" MOD_SSL_CONF_DEST = "options-ssl-apache.conf" """Name of the mod_ssl config file as saved in `IConfig.config_dir`.""" @@ -191,39 +46,3 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy", HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, "Upgrade-Insecure-Requests": UIR_ARGS} - - -def os_constant(key): - """ - Get a constant value for operating system - - :param key: name of cli constant - :return: value of constant for active os - """ - - os_info = util.get_os_info() - try: - constants = CLI_DEFAULTS[os_info[0].lower()] - except KeyError: - constants = os_like_constants() - if not constants: - constants = CLI_DEFAULTS["default"] - return constants[key] - - -def os_like_constants(): - """ - Try to get constants for distribution with - similar layout and configuration, indicated by - /etc/os-release variable "LIKE" - - :returns: Constants dictionary - :rtype: `dict` - """ - - os_like = util.get_systemd_os_like() - if os_like: - for os_name in os_like: - if os_name in CLI_DEFAULTS.keys(): - return CLI_DEFAULTS[os_name] - return {} diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py new file mode 100644 index 000000000..4267398d5 --- /dev/null +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -0,0 +1,47 @@ +""" Entry point for Apache Plugin """ +from certbot import util + +from certbot_apache import configurator +from certbot_apache import override_arch +from certbot_apache import override_darwin +from certbot_apache import override_debian +from certbot_apache import override_centos +from certbot_apache import override_gentoo +from certbot_apache import override_suse + +OVERRIDE_CLASSES = { + "arch": override_arch.ArchConfigurator, + "darwin": override_darwin.DarwinConfigurator, + "debian": override_debian.DebianConfigurator, + "ubuntu": override_debian.DebianConfigurator, + "centos": override_centos.CentOSConfigurator, + "centos linux": override_centos.CentOSConfigurator, + "fedora": override_centos.CentOSConfigurator, + "red hat enterprise linux server": override_centos.CentOSConfigurator, + "rhel": override_centos.CentOSConfigurator, + "amazon": override_centos.CentOSConfigurator, + "gentoo": override_gentoo.GentooConfigurator, + "gentoo base system": override_gentoo.GentooConfigurator, + "opensuse": override_suse.OpenSUSEConfigurator, + "suse": override_suse.OpenSUSEConfigurator, +} + +def get_configurator(): + """ Get correct configurator class based on the OS fingerprint """ + os_info = util.get_os_info() + override_class = None + try: + override_class = OVERRIDE_CLASSES[os_info[0].lower()] + except KeyError: + # OS not found in the list + os_like = util.get_systemd_os_like() + if os_like: + for os_name in os_like: + if os_name in OVERRIDE_CLASSES.keys(): + override_class = OVERRIDE_CLASSES[os_name] + if not override_class: + # No override class found, return the generic configurator + override_class = configurator.ApacheConfigurator + return override_class + +ENTRYPOINT = get_configurator() diff --git a/certbot-apache/certbot_apache/override_arch.py b/certbot-apache/certbot_apache/override_arch.py new file mode 100644 index 000000000..ea5155a3c --- /dev/null +++ b/certbot-apache/certbot_apache/override_arch.py @@ -0,0 +1,31 @@ +""" Distribution specific override class for Arch Linux """ +import pkg_resources + +import zope.interface + +from certbot import interfaces + +from certbot_apache import configurator + +@zope.interface.provider(interfaces.IPluginFactory) +class ArchConfigurator(configurator.ApacheConfigurator): + """Arch Linux specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/httpd", + vhost_root="/etc/httpd/conf", + vhost_files="*.conf", + logs_root="/var/log/httpd", + version_cmd=['apachectl', '-v'], + apache_cmd="apachectl", + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/httpd/conf", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py new file mode 100644 index 000000000..db6cd6fba --- /dev/null +++ b/certbot-apache/certbot_apache/override_centos.py @@ -0,0 +1,59 @@ +""" Distribution specific override class for CentOS family (RHEL, Fedora) """ +import pkg_resources + +import zope.interface + +from certbot import interfaces + +from certbot_apache import apache_util +from certbot_apache import configurator +from certbot_apache import parser + +@zope.interface.provider(interfaces.IPluginFactory) +class CentOSConfigurator(configurator.ApacheConfigurator): + """CentOS specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/httpd", + vhost_root="/etc/httpd/conf.d", + vhost_files="*.conf", + logs_root="/var/log/httpd", + version_cmd=['apachectl', '-v'], + apache_cmd="apachectl", + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/httpd/conf.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "centos-options-ssl-apache.conf") + ) + + def get_parser(self): + """Initializes the ApacheParser""" + return CentOSParser( + self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.version, configurator=self) + + +class CentOSParser(parser.ApacheParser): + """CentOS specific ApacheParser override class""" + def __init__(self, *args, **kwargs): + # CentOS specific configuration file for Apache + self.sysconfig_filep = "/etc/sysconfig/httpd" + super(CentOSParser, self).__init__(*args, **kwargs) + + def update_runtime_variables(self, *args, **kwargs): + """ Override for update_runtime_variables for custom parsing """ + # Opportunistic, works if SELinux not enforced + super(CentOSParser, self).update_runtime_variables(*args, **kwargs) + self.parse_sysconfig_var() + + def parse_sysconfig_var(self): + """ Parses Apache CLI options from CentOS configuration file """ + defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS") + for k in defines.keys(): + self.variables[k] = defines[k] diff --git a/certbot-apache/certbot_apache/override_darwin.py b/certbot-apache/certbot_apache/override_darwin.py new file mode 100644 index 000000000..53741d504 --- /dev/null +++ b/certbot-apache/certbot_apache/override_darwin.py @@ -0,0 +1,31 @@ +""" Distribution specific override class for macOS """ +import pkg_resources + +import zope.interface + +from certbot import interfaces + +from certbot_apache import configurator + +@zope.interface.provider(interfaces.IPluginFactory) +class DarwinConfigurator(configurator.ApacheConfigurator): + """macOS specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/other", + vhost_files="*.conf", + logs_root="/var/log/apache2", + version_cmd=['/usr/sbin/httpd', '-v'], + apache_cmd="/usr/sbin/httpd", + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2/other", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py new file mode 100644 index 000000000..6e2e34ba9 --- /dev/null +++ b/certbot-apache/certbot_apache/override_debian.py @@ -0,0 +1,144 @@ +""" Distribution specific override class for Debian family (Ubuntu/Debian) """ +import logging +import os +import pkg_resources + +import zope.interface + +from certbot import errors +from certbot import interfaces +from certbot import util + +from certbot_apache import apache_util +from certbot_apache import configurator + +logger = logging.getLogger(__name__) + +@zope.interface.provider(interfaces.IPluginFactory) +class DebianConfigurator(configurator.ApacheConfigurator): + """Debian specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + logs_root="/var/log/apache2", + version_cmd=['apache2ctl', '-v'], + apache_cmd="apache2ctl", + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], + enmod="a2enmod", + dismod="a2dismod", + le_vhost_ext="-le-ssl.conf", + handle_mods=True, + handle_sites=True, + challenge_location="/etc/apache2", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) + + def enable_site(self, vhost): + """Enables an available site, Apache reload required. + + .. note:: Does not make sure that the site correctly works or that all + modules are enabled appropriately. + + :param vhost: vhost to enable + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :raises .errors.NotSupportedError: If filesystem layout is not + supported. + + """ + if vhost.enabled: + return + + enabled_path = ("%s/sites-enabled/%s" % + (self.parser.root, + os.path.basename(vhost.filep))) + if not os.path.isdir(os.path.dirname(enabled_path)): + # For some reason, sites-enabled / sites-available do not exist + # Call the parent method + return super(DebianConfigurator, self).enable_site(vhost) + self.reverter.register_file_creation(False, enabled_path) + try: + os.symlink(vhost.filep, enabled_path) + except OSError as err: + if os.path.islink(enabled_path) and os.path.realpath( + enabled_path) == vhost.filep: + # Already in shape + vhost.enabled = True + return + else: + logger.warning( + "Could not symlink %s to %s, got error: %s", enabled_path, + vhost.filep, err.strerror) + errstring = ("Encountered error while trying to enable a " + + "newly created VirtualHost located at {0} by " + + "linking to it from {1}") + raise errors.NotSupportedError(errstring.format(vhost.filep, + enabled_path)) + vhost.enabled = True + logger.info("Enabling available site: %s", vhost.filep) + self.save_notes += "Enabled site %s\n" % vhost.filep + + def enable_mod(self, mod_name, temp=False): + # pylint: disable=unused-argument + """Enables module in Apache. + + Both enables and reloads Apache so module is active. + + :param str mod_name: Name of the module to enable. (e.g. 'ssl') + :param bool temp: Whether or not this is a temporary action. + + :raises .errors.NotSupportedError: If the filesystem layout is not + supported. + :raises .errors.MisconfigurationError: If a2enmod or a2dismod cannot be + run. + + """ + avail_path = os.path.join(self.parser.root, "mods-available") + enabled_path = os.path.join(self.parser.root, "mods-enabled") + if not os.path.isdir(avail_path) or not os.path.isdir(enabled_path): + raise errors.NotSupportedError( + "Unsupported directory layout. You may try to enable mod %s " + "and try again." % mod_name) + + deps = apache_util.get_mod_deps(mod_name) + + # Enable all dependencies + for dep in deps: + if (dep + "_module") not in self.parser.modules: + self._enable_mod_debian(dep, temp) + self.parser.add_mod(dep) + note = "Enabled dependency of %s module - %s" % (mod_name, dep) + if not temp: + self.save_notes += note + os.linesep + logger.debug(note) + + # Enable actual module + self._enable_mod_debian(mod_name, temp) + self.parser.add_mod(mod_name) + + if not temp: + self.save_notes += "Enabled %s module in Apache\n" % mod_name + logger.info("Enabled Apache %s module", mod_name) + + # Modules can enable additional config files. Variables may be defined + # within these new configuration sections. + # Reload is not necessary as DUMP_RUN_CFG uses latest config. + self.parser.update_runtime_variables() + + def _enable_mod_debian(self, mod_name, temp): + """Assumes mods-available, mods-enabled layout.""" + # Generate reversal command. + # Try to be safe here... check that we can probably reverse before + # applying enmod command + if not util.exe_exists(self.conf("dismod")): + raise errors.MisconfigurationError( + "Unable to find a2dismod, please make sure a2enmod and " + "a2dismod are configured correctly for certbot.") + + self.reverter.register_undo_command( + temp, [self.conf("dismod"), mod_name]) + util.run_script([self.conf("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py new file mode 100644 index 000000000..d4d4e96b9 --- /dev/null +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -0,0 +1,58 @@ +""" Distribution specific override class for Gentoo Linux """ +import pkg_resources + +import zope.interface + +from certbot import interfaces + +from certbot_apache import apache_util +from certbot_apache import configurator +from certbot_apache import parser + +@zope.interface.provider(interfaces.IPluginFactory) +class GentooConfigurator(configurator.ApacheConfigurator): + """Gentoo specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/vhosts.d", + vhost_files="*.conf", + logs_root="/var/log/apache2", + version_cmd=['/usr/sbin/apache2', '-v'], + apache_cmd="apache2ctl", + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2/vhosts.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) + + def get_parser(self): + """Initializes the ApacheParser""" + return GentooParser( + self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.version, configurator=self) + + +class GentooParser(parser.ApacheParser): + """Gentoo specific ApacheParser override class""" + def __init__(self, *args, **kwargs): + # Gentoo specific configuration file for Apache2 + self.apacheconfig_filep = "/etc/conf.d/apache2" + super(GentooParser, self).__init__(*args, **kwargs) + + def update_runtime_variables(self): + """ Override for update_runtime_variables for custom parsing """ + self.parse_sysconfig_var() + + def parse_sysconfig_var(self): + """ Parses Apache CLI options from Gentoo configuration file """ + defines = apache_util.parse_define_file(self.apacheconfig_filep, + "APACHE2_OPTS") + for k in defines.keys(): + self.variables[k] = defines[k] diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py new file mode 100644 index 000000000..a67054b5b --- /dev/null +++ b/certbot-apache/certbot_apache/override_suse.py @@ -0,0 +1,31 @@ +""" Distribution specific override class for OpenSUSE """ +import pkg_resources + +import zope.interface + +from certbot import interfaces + +from certbot_apache import configurator + +@zope.interface.provider(interfaces.IPluginFactory) +class OpenSUSEConfigurator(configurator.ApacheConfigurator): + """OpenSUSE specific ApacheConfigurator override class""" + + OS_DEFAULTS = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/vhosts.d", + vhost_files="*.conf", + logs_root="/var/log/apache2", + version_cmd=['apache2ctl', '-v'], + apache_cmd="apache2ctl", + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], + enmod="a2enmod", + dismod="a2dismod", + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2/vhosts.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "certbot_apache", "options-ssl-apache.conf") + ) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index b15608d61..7715d2c35 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -11,8 +11,6 @@ import six from certbot import errors -from certbot_apache import constants - logger = logging.getLogger(__name__) @@ -40,14 +38,9 @@ class ApacheParser(object): # issues with aug.load() after adding new files / defines to parse tree self.configurator = configurator - # This uses the binary, so it can be done first. - # https://httpd.apache.org/docs/2.4/mod/core.html#define - # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine - # This only handles invocation parameters and Define directives! + self.modules = set() self.parser_paths = {} self.variables = {} - if version >= (2, 4): - self.update_runtime_variables() self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -55,24 +48,26 @@ class ApacheParser(object): self.loc = {"root": self._find_config_root()} self.parse_file(self.loc["root"]) + if version >= (2, 4): + # Look up variables from httpd and add to DOM if not already parsed + self.update_runtime_variables() + # This problem has been fixed in Augeas 1.0 self.standardize_excl() - # Temporarily set modules to be empty, so that find_dirs can work - # https://httpd.apache.org/docs/2.4/mod/core.html#ifmodule - # This needs to come before locations are set. - self.modules = set() - self.init_modules() + # Parse LoadModule directives from configuration files + self.parse_modules() # Set up rest of locations self.loc.update(self._set_locations()) + # list of the active include paths, before modifications self.existing_paths = copy.deepcopy(self.parser_paths) # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - constants.os_constant("vhost_files")) + self.configurator.constant("vhost_files")) # check to see if there were unparsed define statements if version < (2, 4): @@ -103,50 +98,61 @@ class ApacheParser(object): # Create a new path self.existing_paths[new_dir] = [new_file] - def init_modules(self): + def add_mod(self, mod_name): + """Shortcut for updating parser modules.""" + if mod_name + "_module" not in self.modules: + self.modules.add(mod_name + "_module") + if "mod_" + mod_name + ".c" not in self.modules: + self.modules.add("mod_" + mod_name + ".c") + + def reset_modules(self): + """Reset the loaded modules list. This is called from cleanup to clear + temporarily loaded modules.""" + self.modules = set() + self.update_modules() + self.parse_modules() + + def parse_modules(self): """Iterates on the configuration until no new modules are loaded. ..todo:: This should be attempted to be done with a binary to avoid the iteration issue. Else... parse and enable mods at same time. """ - # Since modules are being initiated... clear existing set. - self.modules = set() + mods = set() matches = self.find_dir("LoadModule") - iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration prev_size = -1 - while len(self.modules) != prev_size: - prev_size = len(self.modules) + while len(mods) != prev_size: + prev_size = len(mods) for match_name, match_filename in six.moves.zip( iterator, iterator): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) if mod_name and mod_filename: - self.modules.add(mod_name) - self.modules.add(os.path.basename(mod_filename)[:-2] + "c") + mods.add(mod_name) + mods.add(os.path.basename(mod_filename)[:-2] + "c") else: logger.debug("Could not read LoadModule directive from " + "Augeas path: {0}".format(match_name[6:])) + self.modules.update(mods) def update_runtime_variables(self): - """" + """Update Includes, Defines and Includes from httpd config dump data""" + self.update_defines() + self.update_includes() + self.update_modules() - .. note:: Compile time variables (apache2ctl -V) are not used within - the dynamic configuration files. These should not be parsed or - interpreted. - - .. todo:: Create separate compile time variables... - simply for arg_get() - - """ - stdout = self._get_runtime_cfg() + def update_defines(self): + """Get Defines from httpd process""" variables = dict() - matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) + define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + "DUMP_RUN_CFG"] + matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") try: matches.remove("DUMP_RUN_CFG") except ValueError: @@ -163,15 +169,54 @@ class ApacheParser(object): self.variables = variables - def _get_runtime_cfg(self): # pylint: disable=no-self-use - """Get runtime configuration info. + def update_includes(self): + """Get includes from httpd process, and add them to DOM if needed""" - :returns: stdout from DUMP_RUN_CFG + # Find_dir iterates over configuration for Include and IncludeOptional + # directives to make sure we see the full include tree present in the + # configuration files + _ = self.find_dir("Include") + + inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + "DUMP_INCLUDES"] + matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)") + if matches: + for i in matches: + if not self.parsed_in_current(i): + self.parse_file(i) + + def update_modules(self): + """Get loaded modules from httpd process, and add them to DOM""" + + mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + "DUMP_MODULES"] + matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") + for mod in matches: + self.add_mod(mod.strip()) + + def parse_from_subprocess(self, command, regexp): + """Get values from stdout of subprocess command + + :param list command: Command to run + :param str regexp: Regexp for parsing + + :returns: list parsed from command output + :rtype: list + + """ + stdout = self._get_runtime_cfg(command) + return re.compile(regexp).findall(stdout) + + def _get_runtime_cfg(self, command): # pylint: disable=no-self-use + """Get runtime configuration info. + :param command: Command to run + + :returns: stdout from command """ try: proc = subprocess.Popen( - constants.os_constant("define_cmd"), + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) @@ -180,10 +225,10 @@ class ApacheParser(object): except (OSError, ValueError): logger.error( "Error running command %s for runtime parameters!%s", - constants.os_constant("define_cmd"), os.linesep) + command, os.linesep) raise errors.MisconfigurationError( "Error accessing loaded Apache parameters: %s", - constants.os_constant("define_cmd")) + command) # Small errors that do not impede if proc.returncode != 0: logger.warning("Error in checking parameter list: %s", stderr) diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py new file mode 100644 index 000000000..7ca47a4d5 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -0,0 +1,123 @@ +"""Test for certbot_apache.configurator for Centos overrides""" +import os +import unittest + +import mock + +from certbot_apache import obj +from certbot_apache import override_centos +from certbot_apache.tests import util + +def get_vh_truth(temp_dir, config_name): + """Return the ground truth for the specified directory.""" + prefix = os.path.join( + temp_dir, config_name, "httpd/conf.d") + + aug_pre = "/files" + prefix + vh_truth = [ + obj.VirtualHost( + os.path.join(prefix, "centos.example.com.conf"), + os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"), + set([obj.Addr.fromstring("*:80")]), + False, True, "centos.example.com"), + obj.VirtualHost( + os.path.join(prefix, "ssl.conf"), + os.path.join(aug_pre, "ssl.conf/VirtualHost"), + set([obj.Addr.fromstring("_default_:443")]), + True, True, None) + ] + return vh_truth + +class MultipleVhostsTestCentOS(util.ApacheTest): + """Multiple vhost tests for CentOS / RHEL family of distros""" + + _multiprocess_can_split_ = True + + def setUp(self): # pylint: disable=arguments-differ + test_dir = "centos7_apache/apache" + config_root = "centos7_apache/apache/httpd" + vhost_root = "centos7_apache/apache/httpd/conf.d" + super(MultipleVhostsTestCentOS, self).setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="centos") + self.vh_truth = get_vh_truth( + self.temp_dir, "centos7_apache/apache") + + def test_get_parser(self): + self.assertTrue(isinstance(self.config.parser, + override_centos.CentOSParser)) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_opportunistic_httpd_runtime_parsing(self, mock_get): + define_val = ( + 'Define: TEST1\n' + 'Define: TEST2\n' + 'Define: DUMP_RUN_CFG\n' + ) + mod_val = ( + 'Loaded Modules:\n' + ' mock_module (static)\n' + ' another_module (static)\n' + ) + def mock_get_cfg(command): + """Mock httpd process stdout""" + if command == ['apachectl', '-t', '-D', 'DUMP_RUN_CFG']: + return define_val + elif command == ['apachectl', '-t', '-D', 'DUMP_MODULES']: + return mod_val + return "" + mock_get.side_effect = mock_get_cfg + self.config.parser.modules = set() + self.config.parser.variables = {} + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("centos", "7") + self.config.parser.update_runtime_variables() + + self.assertEquals(mock_get.call_count, 3) + self.assertEquals(len(self.config.parser.modules), 4) + self.assertEquals(len(self.config.parser.variables), 2) + self.assertTrue("TEST2" in self.config.parser.variables.keys()) + self.assertTrue("mod_another.c" in self.config.parser.modules) + + def test_get_virtual_hosts(self): + """Make sure all vhosts are being properly found.""" + vhs = self.config.get_virtual_hosts() + self.assertEqual(len(vhs), 2) + found = 0 + + for vhost in vhs: + for centos_truth in self.vh_truth: + if vhost == centos_truth: + found += 1 + break + else: + raise Exception("Missed: %s" % vhost) # pragma: no cover + self.assertEqual(found, 2) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_get_sysconfig_vars(self, mock_cfg): + """Make sure we read the sysconfig OPTIONS variable correctly""" + # Return nothing for the process calls + mock_cfg.return_value = "" + self.config.parser.sysconfig_filep = os.path.realpath( + os.path.join(self.config.parser.root, "../sysconfig/httpd")) + self.config.parser.variables = {} + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("centos", "7") + self.config.parser.update_runtime_variables() + + self.assertTrue("mock_define" in self.config.parser.variables.keys()) + self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) + self.assertTrue("mock_value" in self.config.parser.variables.keys()) + self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/complex_parsing_test.py b/certbot-apache/certbot_apache/tests/complex_parsing_test.py index 079d7e95f..a296fb0eb 100644 --- a/certbot-apache/certbot_apache/tests/complex_parsing_test.py +++ b/certbot-apache/certbot_apache/tests/complex_parsing_test.py @@ -18,7 +18,7 @@ class ComplexParserTest(util.ParserTest): self.setup_variables() # This needs to happen after due to setup_variables not being run # until after - self.parser.init_modules() # pylint: disable=protected-access + self.parser.parse_modules() # pylint: disable=protected-access def tearDown(self): shutil.rmtree(self.temp_dir) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 90561d6ad..4f85e1e3f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -3,12 +3,12 @@ import os import shutil import socket +import tempfile import unittest import mock # six is used in mock.patch() import six # pylint: disable=unused-import -import tempfile from acme import challenges @@ -19,7 +19,7 @@ from certbot import errors from certbot.tests import acme_util from certbot.tests import util as certbot_util -from certbot_apache import configurator +from certbot_apache import apache_util from certbot_apache import constants from certbot_apache import parser from certbot_apache import obj @@ -34,39 +34,24 @@ class MultipleVhostsTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ super(MultipleVhostsTest, self).setUp() - from certbot_apache.constants import os_constant - orig_os_constant = os_constant - def mock_os_constant(key, vhost_path=self.vhost_path): - """Mock default vhost path""" - if key == "vhost_root": - return vhost_path - else: - return orig_os_constant(key) - - with mock.patch("certbot_apache.constants.os_constant") as mock_c: - mock_c.side_effect = mock_os_constant - self.config = util.get_apache_configurator( - self.config_path, None, self.config_dir, self.work_dir) - self.config = self.mock_deploy_cert(self.config) + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") def mock_deploy_cert(self, config): """A test for a mock deploy cert""" - self.config.real_deploy_cert = self.config.deploy_cert + config.real_deploy_cert = self.config.deploy_cert def mocked_deploy_cert(*args, **kwargs): """a helper to mock a deployed cert""" - with mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod"): + g_mod = "certbot_apache.configurator.ApacheConfigurator.enable_mod" + with mock.patch(g_mod): config.real_deploy_cert(*args, **kwargs) self.config.deploy_cert = mocked_deploy_cert return self.config - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - @mock.patch("certbot_apache.configurator.ApacheConfigurator.init_augeas") @mock.patch("certbot_apache.configurator.path_surgery") def test_prepare_no_install(self, mock_surgery, _init_augeas): @@ -130,6 +115,10 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_constant(self): + self.assertEqual(self.config.constant("server_root"), "/etc/apache2") + self.assertEqual(self.config.constant("nonexistent"), None) + @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): mock_utility = mock_getutility() @@ -163,13 +152,12 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue("certbot.demo" in names) def test_get_bad_path(self): - from certbot_apache.configurator import get_file_path - self.assertEqual(get_file_path(None), None) - self.assertEqual(get_file_path("nonexistent"), None) + self.assertEqual(apache_util.get_file_path(None), None) + self.assertEqual(apache_util.get_file_path("nonexistent"), None) self.assertEqual(self.config._create_vhost("nonexistent"), None) # pylint: disable=protected-access def test_get_aug_internal_path(self): - from certbot_apache.configurator import get_internal_aug_path + from certbot_apache.apache_util import get_internal_aug_path internal_paths = [ "Virtualhost", "IfModule/VirtualHost", "VirtualHost", "VirtualHost", "Macro/VirtualHost", "IfModule/VirtualHost", "VirtualHost", @@ -319,190 +307,23 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.assertEqual(len(self.config._non_default_vhosts()), 8) - @mock.patch("certbot.util.run_script") - @mock.patch("certbot.util.exe_exists") - @mock.patch("certbot_apache.parser.subprocess.Popen") - def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script): - mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") - mock_popen().returncode = 0 - mock_exe_exists.return_value = True - - self.config.enable_mod("ssl") - self.assertTrue("ssl_module" in self.config.parser.modules) - self.assertTrue("mod_ssl.c" in self.config.parser.modules) - - self.assertTrue(mock_run_script.called) - - def test_enable_mod_unsupported_dirs(self): - shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled")) - self.assertRaises( - errors.NotSupportedError, self.config.enable_mod, "ssl") - - @mock.patch("certbot.util.exe_exists") - def test_enable_mod_no_disable(self, mock_exe_exists): - mock_exe_exists.return_value = False - self.assertRaises( - errors.MisconfigurationError, self.config.enable_mod, "ssl") - - def test_enable_site_already_enabled(self): - self.assertTrue(self.vh_truth[1].enabled) - self.config.enable_site(self.vh_truth[1]) - - def test_enable_site_failure(self): - self.config.parser.root = "/tmp/nonexistent" - self.assertRaises( - errors.NotSupportedError, - self.config.enable_site, - obj.VirtualHost("asdf", "afsaf", set(), False, False)) - - def test_enable_site_nondebian(self): - mock_c = "certbot_apache.configurator.ApacheConfigurator.conf" - def conf_side_effect(arg): - """ Mock function for ApacheConfigurator.conf """ - confvars = {"handle-sites": False} - if arg in confvars: - return confvars[arg] - inc_path = "/path/to/wherever" - vhost = self.vh_truth[0] - with mock.patch(mock_c) as mock_conf: - mock_conf.side_effect = conf_side_effect - vhost.enabled = False - vhost.filep = inc_path - self.assertFalse(self.config.parser.find_dir("Include", inc_path)) - self.assertFalse( - os.path.dirname(inc_path) in self.config.parser.existing_paths) - self.config.enable_site(vhost) - self.assertTrue(self.config.parser.find_dir("Include", inc_path)) - self.assertTrue( - os.path.dirname(inc_path) in self.config.parser.existing_paths) - self.assertTrue( - os.path.basename(inc_path) in self.config.parser.existing_paths[ - os.path.dirname(inc_path)]) - def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") + self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( "encryption-example.demo", "example/cert.pem", "example/key.pem", "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) - # Make sure that we don't error out if symlink already exists - ssl_vhost.enabled = False - self.assertFalse(ssl_vhost.enabled) - self.config.deploy_cert( - "encryption-example.demo", "example/cert.pem", "example/key.pem", - "example/cert_chain.pem", "example/fullchain.pem") - self.assertTrue(ssl_vhost.enabled) - - def test_deploy_cert_newssl(self): - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, - self.work_dir, version=(2, 4, 16)) - - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - - # Get the default 443 vhost - self.config.assoc["random.demo"] = self.vh_truth[1] - self.config = self.mock_deploy_cert(self.config) - self.config.deploy_cert( - "random.demo", "example/cert.pem", "example/key.pem", - "example/cert_chain.pem", "example/fullchain.pem") - self.config.save() - - # Verify ssl_module was enabled. - self.assertTrue(self.vh_truth[1].enabled) - self.assertTrue("ssl_module" in self.config.parser.modules) - - loc_cert = self.config.parser.find_dir( - "sslcertificatefile", "example/fullchain.pem", - self.vh_truth[1].path) - loc_key = self.config.parser.find_dir( - "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path) - - # Verify one directive was found in the correct file - self.assertEqual(len(loc_cert), 1) - self.assertEqual( - configurator.get_file_path(loc_cert[0]), - self.vh_truth[1].filep) - - self.assertEqual(len(loc_key), 1) - self.assertEqual( - configurator.get_file_path(loc_key[0]), - self.vh_truth[1].filep) - - def test_deploy_cert_newssl_no_fullchain(self): - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, - self.work_dir, version=(2, 4, 16)) - self.config = self.mock_deploy_cert(self.config) - - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - - # Get the default 443 vhost - self.config.assoc["random.demo"] = self.vh_truth[1] - self.assertRaises(errors.PluginError, - lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", - "example/key.pem")) - - def test_deploy_cert_old_apache_no_chain(self): - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, - self.work_dir, version=(2, 4, 7)) - self.config = self.mock_deploy_cert(self.config) - - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - - # Get the default 443 vhost - self.config.assoc["random.demo"] = self.vh_truth[1] - self.assertRaises(errors.PluginError, - lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", - "example/key.pem")) - - def test_deploy_cert_not_parsed_path(self): - # Make sure that we add include to root config for vhosts when - # handle-sites is false - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - tmp_path = os.path.realpath(tempfile.mkdtemp("vhostroot")) - os.chmod(tmp_path, 0o755) - mock_p = "certbot_apache.configurator.ApacheConfigurator._get_ssl_vhost_path" - mock_a = "certbot_apache.parser.ApacheParser.add_include" - mock_c = "certbot_apache.configurator.ApacheConfigurator.conf" - orig_conf = self.config.conf - def conf_side_effect(arg): - """ Mock function for ApacheConfigurator.conf """ - confvars = {"handle-sites": False} - if arg in confvars: - return confvars[arg] - else: - return orig_conf("arg") - - with mock.patch(mock_c) as mock_conf: - mock_conf.side_effect = conf_side_effect - with mock.patch(mock_p) as mock_path: - mock_path.return_value = os.path.join(tmp_path, "whatever.conf") - with mock.patch(mock_a) as mock_add: - self.config.deploy_cert( - "encryption-example.demo", - "example/cert.pem", "example/key.pem", - "example/cert_chain.pem") - # Test that we actually called add_include - self.assertTrue(mock_add.called) - shutil.rmtree(tmp_path) - def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") - + self.config.parser.modules.add("socache_shmcb_module") # Patch _add_dummy_ssl_directives to make sure we write them correctly # pylint: disable=protected-access orig_add_dummy = self.config._add_dummy_ssl_directives @@ -531,7 +352,6 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue( "insert_key_file_path" in find_args(vhostpath, "SSLCertificateKeyFile")) - # pylint: disable=protected-access self.config._add_dummy_ssl_directives = mock_add_dummy_ssl @@ -557,17 +377,17 @@ class MultipleVhostsTest(util.ApacheTest): # Verify one directive was found in the correct file self.assertEqual(len(loc_cert), 1) self.assertEqual( - configurator.get_file_path(loc_cert[0]), + apache_util.get_file_path(loc_cert[0]), self.vh_truth[1].filep) self.assertEqual(len(loc_key), 1) self.assertEqual( - configurator.get_file_path(loc_key[0]), + apache_util.get_file_path(loc_key[0]), self.vh_truth[1].filep) self.assertEqual(len(loc_chain), 1) self.assertEqual( - configurator.get_file_path(loc_chain[0]), + apache_util.get_file_path(loc_chain[0]), self.vh_truth[1].filep) # One more time for chain directive setting @@ -877,7 +697,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_restart.call_count, 1) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_cleanup(self, mock_restart): + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_cleanup(self, mock_cfg, mock_restart): + mock_cfg.return_value = "" _, achall1, achall2 = self.get_achalls() self.config._chall_out.add(achall1) # pylint: disable=protected-access @@ -890,7 +712,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(mock_restart.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_cleanup_no_errors(self, mock_restart): + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_cleanup_no_errors(self, mock_cfg, mock_restart): + mock_cfg.return_value = "" _, achall1, achall2 = self.get_achalls() self.config._chall_out.add(achall1) # pylint: disable=protected-access @@ -951,10 +775,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(isinstance(self.config.get_chall_pref(""), list)) def test_install_ssl_options_conf(self): - from certbot_apache.configurator import install_ssl_options_conf path = os.path.join(self.work_dir, "test_it") other_path = os.path.join(self.work_dir, "other_test_it") - install_ssl_options_conf(path, other_path) + self.config.install_ssl_options_conf(path, other_path) self.assertTrue(os.path.isfile(path)) self.assertTrue(os.path.isfile(other_path)) @@ -994,20 +817,17 @@ class MultipleVhostsTest(util.ApacheTest): errors.PluginError, self.config.enhance, "certbot.demo", "unknown_enhancement") - @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") - def test_ocsp_stapling(self, mock_exe, mock_run_script): + def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True # This will create an ssl vhost for certbot.demo self.config.enhance("certbot.demo", "staple-ocsp") - self.assertTrue("socache_shmcb_module" in self.config.parser.modules) - self.assertTrue(mock_run_script.called) - # Get the ssl vhost for certbot.demo ssl_vhost = self.config.assoc["certbot.demo"] @@ -1077,14 +897,13 @@ class MultipleVhostsTest(util.ApacheTest): def test_http_header_hsts(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") mock_exe.return_value = True # This will create an ssl vhost for certbot.demo self.config.enhance("certbot.demo", "ensure-http-header", "Strict-Transport-Security") - self.assertTrue("headers_module" in self.config.parser.modules) - # Get the ssl vhost for certbot.demo ssl_vhost = self.config.assoc["certbot.demo"] @@ -1115,6 +934,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_http_header_uir(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -1151,6 +972,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_well_formed_http(self, mock_exe, _): + self.config.parser.modules.add("rewrite_module") self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) @@ -1173,8 +995,6 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path[:-3])) self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path[:-3])) - self.assertTrue("rewrite_module" in self.config.parser.modules) - def test_rewrite_rule_exists(self): # Skip the enable mod self.config.parser.modules.add("rewrite_module") @@ -1196,6 +1016,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_existing_rewrite(self, mock_exe, _): + self.config.parser.modules.add("rewrite_module") self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1228,6 +1049,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_old_https_redirection(self, mock_exe, _): + self.config.parser.modules.add("rewrite_module") self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1365,6 +1187,57 @@ class MultipleVhostsTest(util.ApacheTest): self.config.aug.match.side_effect = RuntimeError self.assertFalse(self.config._check_aug_version()) + def test_enable_site_nondebian(self): + inc_path = "/path/to/wherever" + vhost = self.vh_truth[0] + vhost.enabled = False + vhost.filep = inc_path + self.assertFalse(self.config.parser.find_dir("Include", inc_path)) + self.assertFalse( + os.path.dirname(inc_path) in self.config.parser.existing_paths) + self.config.enable_site(vhost) + self.assertTrue(self.config.parser.find_dir("Include", inc_path)) + self.assertTrue( + os.path.dirname(inc_path) in self.config.parser.existing_paths) + self.assertTrue( + os.path.basename(inc_path) in self.config.parser.existing_paths[ + os.path.dirname(inc_path)]) + + def test_deploy_cert_not_parsed_path(self): + # Make sure that we add include to root config for vhosts when + # handle-sites is false + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") + tmp_path = os.path.realpath(tempfile.mkdtemp("vhostroot")) + os.chmod(tmp_path, 0o755) + mock_p = "certbot_apache.configurator.ApacheConfigurator._get_ssl_vhost_path" + mock_a = "certbot_apache.parser.ApacheParser.add_include" + + with mock.patch(mock_p) as mock_path: + mock_path.return_value = os.path.join(tmp_path, "whatever.conf") + with mock.patch(mock_a) as mock_add: + self.config.deploy_cert( + "encryption-example.demo", + "example/cert.pem", "example/key.pem", + "example/cert_chain.pem") + # Test that we actually called add_include + self.assertTrue(mock_add.called) + shutil.rmtree(tmp_path) + + @mock.patch("certbot_apache.parser.ApacheParser.parsed_in_original") + def test_choose_vhost_and_servername_addition_parsed(self, mock_parsed): + ret_vh = self.vh_truth[8] + ret_vh.enabled = True + self.config.enable_site(ret_vh) + # Make sure that we return early + self.assertFalse(mock_parsed.called) + + def test_enable_mod_unsupported(self): + self.assertRaises(errors.MisconfigurationError, + self.config.enable_mod, + "whatever") + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access @@ -1378,12 +1251,8 @@ class AugeasVhostsTest(util.ApacheTest): vhost_root=vr) self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir) - - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir) def test_choosevhost_with_illegal_name(self): self.config.aug = mock.MagicMock() @@ -1461,15 +1330,11 @@ class MultiVhostsTest(util.ApacheTest): vhost_root=vr) self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config_path, self.vhost_path, + self.config_dir, self.work_dir, conf_vhost_path=self.vhost_path) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multi_vhosts") - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) @@ -1569,11 +1434,11 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir) def _call(self): - from certbot_apache.configurator import install_ssl_options_conf - install_ssl_options_conf(self.config.mod_ssl_conf, self.config.updated_mod_ssl_conf_digest) + self.config.install_ssl_options_conf(self.config.mod_ssl_conf, + self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(constants.os_constant("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC")) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1608,7 +1473,8 @@ class InstallSslOptionsConfTest(util.ApacheTest): self._call() self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) - self.assertEqual(crypto_util.sha256sum(constants.os_constant("MOD_SSL_CONF_SRC")), + self.assertEqual(crypto_util.sha256sum( + self.config.constant("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1623,7 +1489,8 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertEqual(mock_logger.warning.call_args[0][0], "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") - self.assertEqual(crypto_util.sha256sum(constants.os_constant("MOD_SSL_CONF_SRC")), + self.assertEqual(crypto_util.sha256sum( + self.config.constant("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: diff --git a/certbot-apache/certbot_apache/tests/constants_test.py b/certbot-apache/certbot_apache/tests/constants_test.py deleted file mode 100644 index 5ab324101..000000000 --- a/certbot-apache/certbot_apache/tests/constants_test.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Test for certbot_apache.configurator.""" - -import mock -import unittest - -from certbot_apache import constants - - -class ConstantsTest(unittest.TestCase): - - @mock.patch("certbot.util.get_os_info") - def test_get_debian_value(self, os_info): - os_info.return_value = ('Debian', '', '') - self.assertEqual(constants.os_constant("vhost_root"), - "/etc/apache2/sites-available") - - @mock.patch("certbot.util.get_os_info") - def test_get_centos_value(self, os_info): - os_info.return_value = ('CentOS Linux', '', '') - self.assertEqual(constants.os_constant("vhost_root"), - "/etc/httpd/conf.d") - - @mock.patch("certbot.util.get_systemd_os_like") - @mock.patch("certbot.util.get_os_info") - def test_get_default_values(self, os_info, os_like): - os_info.return_value = ('Nonexistent Linux', '', '') - os_like.return_value = {} - self.assertFalse(constants.os_constant("handle_mods")) - self.assertEqual(constants.os_constant("server_root"), "/etc/apache2") - self.assertEqual(constants.os_constant("vhost_root"), - "/etc/apache2/sites-available") - - @mock.patch("certbot.util.get_systemd_os_like") - @mock.patch("certbot.util.get_os_info") - def test_get_darwin_like_values(self, os_info, os_like): - os_info.return_value = ('Nonexistent Linux', '', '') - os_like.return_value = ["something", "nonexistent", "darwin"] - self.assertFalse(constants.os_constant("enmod")) - self.assertEqual(constants.os_constant("vhost_root"), - "/etc/apache2/other") - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py new file mode 100644 index 000000000..a648101e9 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -0,0 +1,209 @@ +"""Test for certbot_apache.configurator for Debian overrides""" +import os +import shutil +import unittest + +import mock + +from certbot import errors + +from certbot_apache import apache_util +from certbot_apache import obj +from certbot_apache.tests import util + + +class MultipleVhostsTestDebian(util.ApacheTest): + """Multiple vhost tests for Debian family of distros""" + + _multiprocess_can_split_ = True + + def setUp(self): # pylint: disable=arguments-differ + super(MultipleVhostsTestDebian, self).setUp() + self.config = util.get_apache_configurator( + self.config_path, None, self.config_dir, self.work_dir, + os_info="debian") + self.config = self.mock_deploy_cert(self.config) + self.vh_truth = util.get_vh_truth(self.temp_dir, + "debian_apache_2_4/multiple_vhosts") + + def mock_deploy_cert(self, config): + """A test for a mock deploy cert""" + config.real_deploy_cert = self.config.deploy_cert + + def mocked_deploy_cert(*args, **kwargs): + """a helper to mock a deployed cert""" + g_mod = "certbot_apache.configurator.ApacheConfigurator.enable_mod" + d_mod = "certbot_apache.override_debian.DebianConfigurator.enable_mod" + with mock.patch(g_mod): + with mock.patch(d_mod): + config.real_deploy_cert(*args, **kwargs) + self.config.deploy_cert = mocked_deploy_cert + return self.config + + def test_enable_mod_unsupported_dirs(self): + shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled")) + self.assertRaises( + errors.NotSupportedError, self.config.enable_mod, "ssl") + + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + @mock.patch("certbot_apache.parser.subprocess.Popen") + def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script): + mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + mock_popen().returncode = 0 + mock_exe_exists.return_value = True + + self.config.enable_mod("ssl") + self.assertTrue("ssl_module" in self.config.parser.modules) + self.assertTrue("mod_ssl.c" in self.config.parser.modules) + + self.assertTrue(mock_run_script.called) + + def test_deploy_cert_enable_new_vhost(self): + # Create + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + self.assertFalse(ssl_vhost.enabled) + self.config.deploy_cert( + "encryption-example.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.assertTrue(ssl_vhost.enabled) + # Make sure that we don't error out if symlink already exists + ssl_vhost.enabled = False + self.assertFalse(ssl_vhost.enabled) + self.config.deploy_cert( + "encryption-example.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.assertTrue(ssl_vhost.enabled) + + def test_enable_site_failure(self): + self.config.parser.root = "/tmp/nonexistent" + with mock.patch("os.path.isdir") as mock_dir: + mock_dir.return_value = True + with mock.patch("os.path.islink") as mock_link: + mock_link.return_value = False + self.assertRaises( + errors.NotSupportedError, + self.config.enable_site, + obj.VirtualHost("asdf", "afsaf", set(), False, False)) + + def test_deploy_cert_newssl(self): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) + self.config = self.mock_deploy_cert(self.config) + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + + # Get the default 443 vhost + self.config.assoc["random.demo"] = self.vh_truth[1] + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.config.save() + + # Verify ssl_module was enabled. + self.assertTrue(self.vh_truth[1].enabled) + self.assertTrue("ssl_module" in self.config.parser.modules) + + loc_cert = self.config.parser.find_dir( + "sslcertificatefile", "example/fullchain.pem", + self.vh_truth[1].path) + loc_key = self.config.parser.find_dir( + "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path) + + # Verify one directive was found in the correct file + self.assertEqual(len(loc_cert), 1) + self.assertEqual( + apache_util.get_file_path(loc_cert[0]), + self.vh_truth[1].filep) + + self.assertEqual(len(loc_key), 1) + self.assertEqual( + apache_util.get_file_path(loc_key[0]), + self.vh_truth[1].filep) + + def test_deploy_cert_newssl_no_fullchain(self): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) + self.config = self.mock_deploy_cert(self.config) + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + + # Get the default 443 vhost + self.config.assoc["random.demo"] = self.vh_truth[1] + self.assertRaises(errors.PluginError, + lambda: self.config.deploy_cert( + "random.demo", "example/cert.pem", + "example/key.pem")) + + def test_deploy_cert_old_apache_no_chain(self): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 7)) + self.config = self.mock_deploy_cert(self.config) + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + + # Get the default 443 vhost + self.config.assoc["random.demo"] = self.vh_truth[1] + self.assertRaises(errors.PluginError, + lambda: self.config.deploy_cert( + "random.demo", "example/cert.pem", + "example/key.pem")) + + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_ocsp_stapling_enable_mod(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + self.config.parser.modules.add("mod_ssl.c") + self.config.get_version = mock.Mock(return_value=(2, 4, 7)) + mock_exe.return_value = True + self.config.enhance("certbot.demo", "staple-ocsp") + self.assertTrue("socache_shmcb_module" in self.config.parser.modules) + + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_ensure_http_header_enable_mod(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + self.config.parser.modules.add("mod_ssl.c") + mock_exe.return_value = True + + # This will create an ssl vhost for certbot.demo + self.config.enhance("certbot.demo", "ensure-http-header", + "Strict-Transport-Security") + self.assertTrue("headers_module" in self.config.parser.modules) + + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_redirect_enable_mod(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + mock_exe.return_value = True + self.config.get_version = mock.Mock(return_value=(2, 2)) + # This will create an ssl vhost for certbot.demo + self.config.enhance("certbot.demo", "redirect") + self.assertTrue("rewrite_module" in self.config.parser.modules) + + def test_enable_site_already_enabled(self): + self.assertTrue(self.vh_truth[1].enabled) + self.config.enable_site(self.vh_truth[1]) + + def test_enable_site_call_parent(self): + with mock.patch( + "certbot_apache.configurator.ApacheConfigurator.enable_site") as e_s: + self.config.parser.root = "/tmp/nonexistent" + vh = self.vh_truth[0] + vh.enabled = False + self.config.enable_site(vh) + self.assertTrue(e_s.called) + + @mock.patch("certbot.util.exe_exists") + def test_enable_mod_no_disable(self, mock_exe_exists): + mock_exe_exists.return_value = False + self.assertRaises( + errors.MisconfigurationError, self.config.enable_mod, "ssl") + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/entrypoint_test.py b/certbot-apache/certbot_apache/tests/entrypoint_test.py new file mode 100644 index 000000000..c04611465 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/entrypoint_test.py @@ -0,0 +1,41 @@ +"""Test for certbot_apache.entrypoint for override class resolution""" +import unittest + +import mock + +from certbot_apache import configurator +from certbot_apache import entrypoint + +class EntryPointTest(unittest.TestCase): + """Entrypoint tests""" + + _multiprocess_can_split_ = True + + def test_get_configurator(self): + + with mock.patch("certbot.util.get_os_info") as mock_info: + for distro in entrypoint.OVERRIDE_CLASSES.keys(): + mock_info.return_value = (distro, "whatever") + self.assertEqual(entrypoint.get_configurator(), + entrypoint.OVERRIDE_CLASSES[distro]) + + def test_nonexistent_like(self): + with mock.patch("certbot.util.get_os_info") as mock_info: + mock_info.return_value = ("nonexistent", "irrelevant") + with mock.patch("certbot.util.get_systemd_os_like") as mock_like: + for like in entrypoint.OVERRIDE_CLASSES.keys(): + mock_like.return_value = [like] + self.assertEqual(entrypoint.get_configurator(), + entrypoint.OVERRIDE_CLASSES[like]) + + def test_nonexistent_generic(self): + with mock.patch("certbot.util.get_os_info") as mock_info: + mock_info.return_value = ("nonexistent", "irrelevant") + with mock.patch("certbot.util.get_systemd_os_like") as mock_like: + mock_like.return_value = ["unknonwn"] + self.assertEqual(entrypoint.get_configurator(), + configurator.ApacheConfigurator) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py new file mode 100644 index 000000000..0f2b96818 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -0,0 +1,86 @@ +"""Test for certbot_apache.configurator for Gentoo overrides""" +import os +import unittest + +from certbot_apache import override_gentoo +from certbot_apache import obj +from certbot_apache.tests import util + +def get_vh_truth(temp_dir, config_name): + """Return the ground truth for the specified directory.""" + prefix = os.path.join( + temp_dir, config_name, "apache2/vhosts.d") + + aug_pre = "/files" + prefix + vh_truth = [ + obj.VirtualHost( + os.path.join(prefix, "gentoo.example.com.conf"), + os.path.join(aug_pre, "gentoo.example.com.conf/VirtualHost"), + set([obj.Addr.fromstring("*:80")]), + False, True, "gentoo.example.com"), + obj.VirtualHost( + os.path.join(prefix, "00_default_vhost.conf"), + os.path.join(aug_pre, "00_default_vhost.conf/IfDefine/VirtualHost"), + set([obj.Addr.fromstring("*:80")]), + False, True, "localhost"), + obj.VirtualHost( + os.path.join(prefix, "00_default_ssl_vhost.conf"), + os.path.join(aug_pre, + "00_default_ssl_vhost.conf" + + "/IfDefine/IfDefine/IfModule/VirtualHost"), + set([obj.Addr.fromstring("_default_:443")]), + True, True, "localhost") + ] + return vh_truth + +class MultipleVhostsTestGentoo(util.ApacheTest): + """Multiple vhost tests for non-debian distro""" + + _multiprocess_can_split_ = True + + def setUp(self): # pylint: disable=arguments-differ + test_dir = "gentoo_apache/apache" + config_root = "gentoo_apache/apache/apache2" + vhost_root = "gentoo_apache/apache/apache2/vhosts.d" + super(MultipleVhostsTestGentoo, self).setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") + self.vh_truth = get_vh_truth( + self.temp_dir, "gentoo_apache/apache") + + def test_get_parser(self): + self.assertTrue(isinstance(self.config.parser, + override_gentoo.GentooParser)) + + def test_get_virtual_hosts(self): + """Make sure all vhosts are being properly found.""" + vhs = self.config.get_virtual_hosts() + self.assertEqual(len(vhs), 3) + found = 0 + + for vhost in vhs: + for gentoo_truth in self.vh_truth: + if vhost == gentoo_truth: + found += 1 + break + else: + raise Exception("Missed: %s" % vhost) # pragma: no cover + self.assertEqual(found, 3) + + def test_get_sysconfig_vars(self): + """Make sure we read the Gentoo APACHE2_OPTS variable correctly""" + defines = ['DEFAULT_VHOST', 'INFO', + 'SSL', 'SSL_DEFAULT_VHOST', 'LANGUAGE'] + self.config.parser.apacheconfig_filep = os.path.realpath( + os.path.join(self.config.parser.root, "../conf.d/apache2")) + self.config.parser.variables = {} + self.config.parser.update_runtime_variables() + for define in defines: + self.assertTrue(define in self.config.parser.variables.keys()) + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index 38c5b29c7..a9eb129c2 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -120,17 +120,18 @@ class BasicParserTest(util.ParserTest): @mock.patch("certbot_apache.parser.ApacheParser.find_dir") @mock.patch("certbot_apache.parser.ApacheParser.get_arg") - def test_init_modules_bad_syntax(self, mock_arg, mock_find): + def test_parse_modules_bad_syntax(self, mock_arg, mock_find): mock_find.return_value = ["1", "2", "3", "4", "5", "6", "7", "8"] mock_arg.return_value = None with mock.patch("certbot_apache.parser.logger") as mock_logger: - self.parser.init_modules() + self.parser.parse_modules() # Make sure that we got None return value and logged the file self.assertTrue(mock_logger.debug.called) + @mock.patch("certbot_apache.parser.ApacheParser.find_dir") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") - def test_update_runtime_variables(self, mock_cfg): - mock_cfg.return_value = ( + def test_update_runtime_variables(self, mock_cfg, _): + define_val = ( 'ServerRoot: "/etc/apache2"\n' 'Main DocumentRoot: "/var/www"\n' 'Main ErrorLog: "/var/log/apache2/error.log"\n' @@ -147,11 +148,113 @@ class BasicParserTest(util.ParserTest): 'User: name="www-data" id=33 not_used\n' 'Group: name="www-data" id=33 not_used\n' ) + inc_val = ( + 'Included configuration files:\n' + ' (*) /etc/apache2/apache2.conf\n' + ' (146) /etc/apache2/mods-enabled/access_compat.load\n' + ' (146) /etc/apache2/mods-enabled/alias.load\n' + ' (146) /etc/apache2/mods-enabled/auth_basic.load\n' + ' (146) /etc/apache2/mods-enabled/authn_core.load\n' + ' (146) /etc/apache2/mods-enabled/authn_file.load\n' + ' (146) /etc/apache2/mods-enabled/authz_core.load\n' + ' (146) /etc/apache2/mods-enabled/authz_host.load\n' + ' (146) /etc/apache2/mods-enabled/authz_user.load\n' + ' (146) /etc/apache2/mods-enabled/autoindex.load\n' + ' (146) /etc/apache2/mods-enabled/deflate.load\n' + ' (146) /etc/apache2/mods-enabled/dir.load\n' + ' (146) /etc/apache2/mods-enabled/env.load\n' + ' (146) /etc/apache2/mods-enabled/filter.load\n' + ' (146) /etc/apache2/mods-enabled/mime.load\n' + ' (146) /etc/apache2/mods-enabled/mpm_event.load\n' + ' (146) /etc/apache2/mods-enabled/negotiation.load\n' + ' (146) /etc/apache2/mods-enabled/reqtimeout.load\n' + ' (146) /etc/apache2/mods-enabled/setenvif.load\n' + ' (146) /etc/apache2/mods-enabled/socache_shmcb.load\n' + ' (146) /etc/apache2/mods-enabled/ssl.load\n' + ' (146) /etc/apache2/mods-enabled/status.load\n' + ' (147) /etc/apache2/mods-enabled/alias.conf\n' + ' (147) /etc/apache2/mods-enabled/autoindex.conf\n' + ' (147) /etc/apache2/mods-enabled/deflate.conf\n' + ) + mod_val = ( + 'Loaded Modules:\n' + ' core_module (static)\n' + ' so_module (static)\n' + ' watchdog_module (static)\n' + ' http_module (static)\n' + ' log_config_module (static)\n' + ' logio_module (static)\n' + ' version_module (static)\n' + ' unixd_module (static)\n' + ' access_compat_module (shared)\n' + ' alias_module (shared)\n' + ' auth_basic_module (shared)\n' + ' authn_core_module (shared)\n' + ' authn_file_module (shared)\n' + ' authz_core_module (shared)\n' + ' authz_host_module (shared)\n' + ' authz_user_module (shared)\n' + ' autoindex_module (shared)\n' + ' deflate_module (shared)\n' + ' dir_module (shared)\n' + ' env_module (shared)\n' + ' filter_module (shared)\n' + ' mime_module (shared)\n' + ' mpm_event_module (shared)\n' + ' negotiation_module (shared)\n' + ' reqtimeout_module (shared)\n' + ' setenvif_module (shared)\n' + ' socache_shmcb_module (shared)\n' + ' ssl_module (shared)\n' + ' status_module (shared)\n' + ) + + def mock_get_vars(cmd): + """Mock command output""" + if cmd[-1] == "DUMP_RUN_CFG": + return define_val + elif cmd[-1] == "DUMP_INCLUDES": + return inc_val + elif cmd[-1] == "DUMP_MODULES": + return mod_val + + mock_cfg.side_effect = mock_get_vars + expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.update_runtime_variables() - self.assertEqual(self.parser.variables, expected_vars) + self.parser.modules = set() + with mock.patch( + "certbot_apache.parser.ApacheParser.parse_file") as mock_parse: + self.parser.update_runtime_variables() + self.assertEqual(self.parser.variables, expected_vars) + self.assertEqual(len(self.parser.modules), 58) + # None of the includes in inc_val should be in parsed paths. + # Make sure we tried to include them all. + self.assertEqual(mock_parse.call_count, 25) + + @mock.patch("certbot_apache.parser.ApacheParser.find_dir") + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_update_runtime_variables_alt_values(self, mock_cfg, _): + inc_val = ( + 'Included configuration files:\n' + ' (*) {0}\n' + ' (146) /etc/apache2/mods-enabled/access_compat.load\n' + ' (146) {1}/mods-enabled/alias.load\n' + ).format(self.parser.loc["root"], + os.path.dirname(self.parser.loc["root"])) + + mock_cfg.return_value = inc_val + self.parser.modules = set() + + with mock.patch( + "certbot_apache.parser.ApacheParser.parse_file") as mock_parse: + self.parser.update_runtime_variables() + # No matching modules should have been found + self.assertEqual(len(self.parser.modules), 0) + # Only one of the three includes do not exist in already parsed + # path derived from root configuration Include statements + self.assertEqual(mock_parse.call_count, 1) @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_vars_bad_output(self, mock_cfg): @@ -162,7 +265,7 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache.constants.os_constant") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") @mock.patch("certbot_apache.parser.subprocess.Popen") def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): mock_popen.side_effect = OSError @@ -198,7 +301,7 @@ class ParserInitTest(util.ApacheTest): self.assertRaises( errors.PluginError, ApacheParser, self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath", version=(2, 2, 22)) + "/dummy/vhostpath", version=(2, 2, 22), configurator=self.config) def test_root_normalized(self): from certbot_apache.parser import ApacheParser @@ -210,7 +313,7 @@ class ParserInitTest(util.ApacheTest): "debian_apache_2_4/////multiple_vhosts/../multiple_vhosts/apache2") parser = ApacheParser(self.aug, path, - "/dummy/vhostpath") + "/dummy/vhostpath", configurator=self.config) self.assertEqual(parser.root, self.config_path) @@ -220,7 +323,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath") + "/dummy/vhostpath", configurator=self.config) self.assertEqual(parser.root, self.config_path) @@ -230,7 +333,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, self.config_path + os.path.sep, - "/dummy/vhostpath") + "/dummy/vhostpath", configurator=self.config) self.assertEqual(parser.root, self.config_path) diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/README b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/README new file mode 100644 index 000000000..f5e96615a --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/README @@ -0,0 +1,9 @@ + +This directory holds configuration files for the Apache HTTP Server; +any files in this directory which have the ".conf" extension will be +processed as httpd configuration files. The directory is used in +addition to the directory /etc/httpd/conf.modules.d/, which contains +configuration files necessary to load modules. + +Files are processed in alphabetical order. + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/autoindex.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/autoindex.conf new file mode 100644 index 000000000..a85cf5dca --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/autoindex.conf @@ -0,0 +1,94 @@ +# +# Directives controlling the display of server-generated directory listings. +# +# Required modules: mod_authz_core, mod_authz_host, +# mod_autoindex, mod_alias +# +# To see the listing of a directory, the Options directive for the +# directory must include "Indexes", and the directory must not contain +# a file matching those listed in the DirectoryIndex directive. +# + +# +# IndexOptions: Controls the appearance of server-generated directory +# listings. +# +IndexOptions FancyIndexing HTMLTable VersionSort + +# We include the /icons/ alias for FancyIndexed directory listings. If +# you do not use FancyIndexing, you may comment this out. +# +Alias /icons/ "/usr/share/httpd/icons/" + + + Options Indexes MultiViews FollowSymlinks + AllowOverride None + Require all granted + + +# +# AddIcon* directives tell the server which icon to show for different +# files or filename extensions. These are only displayed for +# FancyIndexed directories. +# +AddIconByEncoding (CMP,/icons/compressed.gif) x-compress x-gzip + +AddIconByType (TXT,/icons/text.gif) text/* +AddIconByType (IMG,/icons/image2.gif) image/* +AddIconByType (SND,/icons/sound2.gif) audio/* +AddIconByType (VID,/icons/movie.gif) video/* + +AddIcon /icons/binary.gif .bin .exe +AddIcon /icons/binhex.gif .hqx +AddIcon /icons/tar.gif .tar +AddIcon /icons/world2.gif .wrl .wrl.gz .vrml .vrm .iv +AddIcon /icons/compressed.gif .Z .z .tgz .gz .zip +AddIcon /icons/a.gif .ps .ai .eps +AddIcon /icons/layout.gif .html .shtml .htm .pdf +AddIcon /icons/text.gif .txt +AddIcon /icons/c.gif .c +AddIcon /icons/p.gif .pl .py +AddIcon /icons/f.gif .for +AddIcon /icons/dvi.gif .dvi +AddIcon /icons/uuencoded.gif .uu +AddIcon /icons/script.gif .conf .sh .shar .csh .ksh .tcl +AddIcon /icons/tex.gif .tex +AddIcon /icons/bomb.gif /core +AddIcon /icons/bomb.gif */core.* + +AddIcon /icons/back.gif .. +AddIcon /icons/hand.right.gif README +AddIcon /icons/folder.gif ^^DIRECTORY^^ +AddIcon /icons/blank.gif ^^BLANKICON^^ + +# +# DefaultIcon is which icon to show for files which do not have an icon +# explicitly set. +# +DefaultIcon /icons/unknown.gif + +# +# AddDescription allows you to place a short description after a file in +# server-generated indexes. These are only displayed for FancyIndexed +# directories. +# Format: AddDescription "description" filename +# +#AddDescription "GZIP compressed document" .gz +#AddDescription "tar archive" .tar +#AddDescription "GZIP compressed tar archive" .tgz + +# +# ReadmeName is the name of the README file the server will look for by +# default, and append to directory listings. +# +# HeaderName is the name of a file which should be prepended to +# directory indexes. +ReadmeName README.html +HeaderName HEADER.html + +# +# IndexIgnore is a set of filenames which directory indexing should ignore +# and not include in the listing. Shell-style wildcarding is permitted. +# +IndexIgnore .??* *~ *# HEADER* README* RCS CVS *,v *,t + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/centos.example.com.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/centos.example.com.conf new file mode 100644 index 000000000..de7ac2777 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/centos.example.com.conf @@ -0,0 +1,7 @@ + + ServerName centos.example.com + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/ssl.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/ssl.conf new file mode 100644 index 000000000..6e2502e9a --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/ssl.conf @@ -0,0 +1,211 @@ +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +Listen 443 https + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/run/httpd/sslcache(512000) +SSLSessionCacheTimeout 300 + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + + + +# General setup for the virtual host, inherited from global configuration +#DocumentRoot "/var/www/html" +#ServerName www.example.com:443 + +# Use separate log files for the SSL virtual host; note that LogLevel +# is not inherited from httpd.conf. +ErrorLog logs/ssl_error_log +TransferLog logs/ssl_access_log +LogLevel warn + +# SSL Engine Switch: +# Enable/Disable SSL for this virtual host. +SSLEngine on + +# SSL Protocol support: +# List the enable protocol levels with which clients will be able to +# connect. Disable SSLv2 access by default: +SSLProtocol all -SSLv2 + +# SSL Cipher Suite: +# List the ciphers that the client is permitted to negotiate. +# See the mod_ssl documentation for a complete list. +SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!SEED:!IDEA + +# Speed-optimized SSL Cipher configuration: +# If speed is your main concern (on busy HTTPS servers e.g.), +# you might want to force clients to specific, performance +# optimized ciphers. In this case, prepend those ciphers +# to the SSLCipherSuite list, and enable SSLHonorCipherOrder. +# Caveat: by giving precedence to RC4-SHA and AES128-SHA +# (as in the example below), most connections will no longer +# have perfect forward secrecy - if the server's key is +# compromised, captures of past or future traffic must be +# considered compromised, too. +#SSLCipherSuite RC4-SHA:AES128-SHA:HIGH:MEDIUM:!aNULL:!MD5 +#SSLHonorCipherOrder on + +# Server Certificate: +# Point SSLCertificateFile at a PEM encoded certificate. If +# the certificate is encrypted, then you will be prompted for a +# pass phrase. Note that a kill -HUP will prompt again. A new +# certificate can be generated using the genkey(1) command. + +# Server Private Key: +# If the key is not combined with the certificate, use this +# directive to point at the key file. Keep in mind that if +# you've both a RSA and a DSA private key you can configure +# both in parallel (to also allow the use of DSA ciphers, etc.) + +# Server Certificate Chain: +# Point SSLCertificateChainFile at a file containing the +# concatenation of PEM encoded CA certificates which form the +# certificate chain for the server certificate. Alternatively +# the referenced file can be the same as SSLCertificateFile +# when the CA certificates are directly appended to the server +# certificate for convinience. +#SSLCertificateChainFile /etc/pki/tls/certs/server-chain.crt + +# Certificate Authority (CA): +# Set the CA certificate verification path where to find CA +# certificates for client authentication or alternatively one +# huge file containing all of them (file must be PEM encoded) +#SSLCACertificateFile /etc/pki/tls/certs/ca-bundle.crt + +# Client Authentication (Type): +# Client certificate verification type and depth. Types are +# none, optional, require and optional_no_ca. Depth is a +# number which specifies how deeply to verify the certificate +# issuer chain before deciding the certificate is not valid. +#SSLVerifyClient require +#SSLVerifyDepth 10 + +# Access Control: +# With SSLRequire you can do per-directory access control based +# on arbitrary complex boolean expressions containing server +# variable checks and other lookup directives. The syntax is a +# mixture between C and Perl. See the mod_ssl documentation +# for more details. +# +#SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ +# and %{SSL_CLIENT_S_DN_O} eq "Snake Oil, Ltd." \ +# and %{SSL_CLIENT_S_DN_OU} in {"Staff", "CA", "Dev"} \ +# and %{TIME_WDAY} >= 1 and %{TIME_WDAY} <= 5 \ +# and %{TIME_HOUR} >= 8 and %{TIME_HOUR} <= 20 ) \ +# or %{REMOTE_ADDR} =~ m/^192\.76\.162\.[0-9]+$/ +# + +# SSL Engine Options: +# Set various options for the SSL engine. +# o FakeBasicAuth: +# Translate the client X.509 into a Basic Authorisation. This means that +# the standard Auth/DBMAuth methods can be used for access control. The +# user name is the `one line' version of the client's X.509 certificate. +# Note that no password is obtained from the user. Every entry in the user +# file needs this password: `xxj31ZMTZzkVA'. +# o ExportCertData: +# This exports two additional environment variables: SSL_CLIENT_CERT and +# SSL_SERVER_CERT. These contain the PEM-encoded certificates of the +# server (always existing) and the client (only existing when client +# authentication is used). This can be used to import the certificates +# into CGI scripts. +# o StdEnvVars: +# This exports the standard SSL/TLS related `SSL_*' environment variables. +# Per default this exportation is switched off for performance reasons, +# because the extraction step is an expensive operation and is usually +# useless for serving static content. So one usually enables the +# exportation for CGI and SSI requests only. +# o StrictRequire: +# This denies access when "SSLRequireSSL" or "SSLRequire" applied even +# under a "Satisfy any" situation, i.e. when it applies access is denied +# and no other module can change it. +# o OptRenegotiate: +# This enables optimized SSL connection renegotiation handling when SSL +# directives are used in per-directory context. +#SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire + + SSLOptions +StdEnvVars + + + SSLOptions +StdEnvVars + + +# SSL Protocol Adjustments: +# The safe and default but still SSL/TLS standard compliant shutdown +# approach is that mod_ssl sends the close notify alert but doesn't wait for +# the close notify alert from client. When you need a different shutdown +# approach you can use one of the following variables: +# o ssl-unclean-shutdown: +# This forces an unclean shutdown when the connection is closed, i.e. no +# SSL close notify alert is send or allowed to received. This violates +# the SSL/TLS standard but is needed for some brain-dead browsers. Use +# this when you receive I/O errors because of the standard approach where +# mod_ssl sends the close notify alert. +# o ssl-accurate-shutdown: +# This forces an accurate shutdown when the connection is closed, i.e. a +# SSL close notify alert is send and mod_ssl waits for the close notify +# alert of the client. This is 100% SSL/TLS standard compliant, but in +# practice often causes hanging connections with brain-dead browsers. Use +# this only for browsers where you know that their SSL implementation +# works correctly. +# Notice: Most problems of broken clients are also related to the HTTP +# keep-alive facility, so you usually additionally want to disable +# keep-alive for those clients, too. Use variable "nokeepalive" for this. +# Similarly, one has to force some clients to use HTTP/1.0 to workaround +# their broken HTTP/1.1 implementation. Use variables "downgrade-1.0" and +# "force-response-1.0" for this. +BrowserMatch "MSIE [2-5]" nokeepalive ssl-unclean-shutdown downgrade-1.0 force-response-1.0 + +# Per-Server Logging: +# The home of a custom SSL log file. Use this when you want a +# compact non-error SSL logfile on a virtual host basis. +CustomLog logs/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/userdir.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/userdir.conf new file mode 100644 index 000000000..b5d7a49ef --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/userdir.conf @@ -0,0 +1,36 @@ +# +# UserDir: The name of the directory that is appended onto a user's home +# directory if a ~user request is received. +# +# The path to the end user account 'public_html' directory must be +# accessible to the webserver userid. This usually means that ~userid +# must have permissions of 711, ~userid/public_html must have permissions +# of 755, and documents contained therein must be world-readable. +# Otherwise, the client will only receive a "403 Forbidden" message. +# + + # + # UserDir is disabled by default since it can confirm the presence + # of a username on the system (depending on home directory + # permissions). + # + UserDir disabled + + # + # To enable requests to /~user/ to serve the user's public_html + # directory, remove the "UserDir disabled" line above, and uncomment + # the following line instead: + # + #UserDir public_html + + +# +# Control access to UserDir directories. The following is an example +# for a site where these directories are restricted to read-only. +# + + AllowOverride FileInfo AuthConfig Limit Indexes + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + Require method GET POST OPTIONS + + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/welcome.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/welcome.conf new file mode 100644 index 000000000..c1b6c11d9 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.d/welcome.conf @@ -0,0 +1,22 @@ +# +# This configuration file enables the default "Welcome" page if there +# is no default index page present for the root URL. To disable the +# Welcome page, comment out all the lines below. +# +# NOTE: if this file is removed, it will be restored on upgrades. +# + + Options -Indexes + ErrorDocument 403 /.noindex.html + + + + AllowOverride None + Require all granted + + +Alias /.noindex.html /usr/share/httpd/noindex/index.html +Alias /noindex/css/bootstrap.min.css /usr/share/httpd/noindex/css/bootstrap.min.css +Alias /noindex/css/open-sans.css /usr/share/httpd/noindex/css/open-sans.css +Alias /images/apache_pb.gif /usr/share/httpd/noindex/images/apache_pb.gif +Alias /images/poweredby.png /usr/share/httpd/noindex/images/poweredby.png diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-base.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-base.conf new file mode 100644 index 000000000..31d979f20 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-base.conf @@ -0,0 +1,77 @@ +# +# This file loads most of the modules included with the Apache HTTP +# Server itself. +# + +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule actions_module modules/mod_actions.so +LoadModule alias_module modules/mod_alias.so +LoadModule allowmethods_module modules/mod_allowmethods.so +LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule auth_digest_module modules/mod_auth_digest.so +LoadModule authn_anon_module modules/mod_authn_anon.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authn_dbd_module modules/mod_authn_dbd.so +LoadModule authn_dbm_module modules/mod_authn_dbm.so +LoadModule authn_file_module modules/mod_authn_file.so +LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_dbm_module modules/mod_authz_dbm.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_owner_module modules/mod_authz_owner.so +LoadModule authz_user_module modules/mod_authz_user.so +LoadModule autoindex_module modules/mod_autoindex.so +LoadModule cache_module modules/mod_cache.so +LoadModule cache_disk_module modules/mod_cache_disk.so +LoadModule data_module modules/mod_data.so +LoadModule dbd_module modules/mod_dbd.so +LoadModule deflate_module modules/mod_deflate.so +LoadModule dir_module modules/mod_dir.so +LoadModule dumpio_module modules/mod_dumpio.so +LoadModule echo_module modules/mod_echo.so +LoadModule env_module modules/mod_env.so +LoadModule expires_module modules/mod_expires.so +LoadModule ext_filter_module modules/mod_ext_filter.so +LoadModule filter_module modules/mod_filter.so +LoadModule headers_module modules/mod_headers.so +LoadModule include_module modules/mod_include.so +LoadModule info_module modules/mod_info.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule logio_module modules/mod_logio.so +LoadModule mime_magic_module modules/mod_mime_magic.so +LoadModule mime_module modules/mod_mime.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule remoteip_module modules/mod_remoteip.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule slotmem_plain_module modules/mod_slotmem_plain.so +LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +LoadModule socache_dbm_module modules/mod_socache_dbm.so +LoadModule socache_memcache_module modules/mod_socache_memcache.so +LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +LoadModule status_module modules/mod_status.so +LoadModule substitute_module modules/mod_substitute.so +LoadModule suexec_module modules/mod_suexec.so +LoadModule unique_id_module modules/mod_unique_id.so +LoadModule unixd_module modules/mod_unixd.so +LoadModule userdir_module modules/mod_userdir.so +LoadModule version_module modules/mod_version.so +LoadModule vhost_alias_module modules/mod_vhost_alias.so + +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule watchdog_module modules/mod_watchdog.so +#LoadModule heartbeat_module modules/mod_heartbeat.so +#LoadModule heartmonitor_module modules/mod_heartmonitor.so +#LoadModule usertrack_module modules/mod_usertrack.so +#LoadModule dialup_module modules/mod_dialup.so +#LoadModule charset_lite_module modules/mod_charset_lite.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +#LoadModule reflector_module modules/mod_reflector.so +#LoadModule request_module modules/mod_request.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule speling_module modules/mod_speling.so + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-dav.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-dav.conf new file mode 100644 index 000000000..e6af8decd --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-dav.conf @@ -0,0 +1,3 @@ +LoadModule dav_module modules/mod_dav.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule dav_lock_module modules/mod_dav_lock.so diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-lua.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-lua.conf new file mode 100644 index 000000000..9e0d0db6e --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-lua.conf @@ -0,0 +1 @@ +LoadModule lua_module modules/mod_lua.so diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-mpm.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-mpm.conf new file mode 100644 index 000000000..7bfd1d413 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-mpm.conf @@ -0,0 +1,19 @@ +# Select the MPM module which should be used by uncommenting exactly +# one of the following LoadModule lines: + +# prefork MPM: Implements a non-threaded, pre-forking web server +# See: http://httpd.apache.org/docs/2.4/mod/prefork.html +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so + +# worker MPM: Multi-Processing Module implementing a hybrid +# multi-threaded multi-process web server +# See: http://httpd.apache.org/docs/2.4/mod/worker.html +# +#LoadModule mpm_worker_module modules/mod_mpm_worker.so + +# event MPM: A variant of the worker MPM with the goal of consuming +# threads only for connections with active processing +# See: http://httpd.apache.org/docs/2.4/mod/event.html +# +#LoadModule mpm_event_module modules/mod_mpm_event.so + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-proxy.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-proxy.conf new file mode 100644 index 000000000..cc0bca077 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-proxy.conf @@ -0,0 +1,16 @@ +# This file configures all the proxy modules: +LoadModule proxy_module modules/mod_proxy.so +LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +LoadModule proxy_connect_module modules/mod_proxy_connect.so +LoadModule proxy_express_module modules/mod_proxy_express.so +LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so +LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +LoadModule proxy_http_module modules/mod_proxy_http.so +LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-ssl.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-ssl.conf new file mode 100644 index 000000000..53235cd76 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-ssl.conf @@ -0,0 +1 @@ +LoadModule ssl_module modules/mod_ssl.so diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-systemd.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-systemd.conf new file mode 100644 index 000000000..b208c972d --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/00-systemd.conf @@ -0,0 +1,2 @@ +# This file configures systemd module: +LoadModule systemd_module modules/mod_systemd.so diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/01-cgi.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/01-cgi.conf new file mode 100644 index 000000000..5b8b9362e --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf.modules.d/01-cgi.conf @@ -0,0 +1,14 @@ +# This configuration file loads a CGI module appropriate to the MPM +# which has been configured in 00-mpm.conf. mod_cgid should be used +# with a threaded MPM; mod_cgi with the prefork MPM. + + + LoadModule cgid_module modules/mod_cgid.so + + + LoadModule cgid_module modules/mod_cgid.so + + + LoadModule cgi_module modules/mod_cgi.so + + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/httpd.conf b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/httpd.conf new file mode 100644 index 000000000..a7af0dc1e --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/httpd.conf @@ -0,0 +1,353 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so 'log/access_log' +# with ServerRoot set to '/www' will be interpreted by the +# server as '/www/log/access_log', where as '/log/access_log' will be +# interpreted as '/log/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/etc/httpd" + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +Include conf.modules.d/*.conf + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User apache +Group apache + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin root@localhost + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/var/www/html" + +# +# Relax access to content within /var/www. +# + + AllowOverride None + # Allow open access: + Require all granted + + +# Further relax access to the default document root: + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # Options FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog "logs/error_log" + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + #CustomLog "logs/access_log" common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" + + + +# +# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig /etc/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + AddType text/html .shtml + AddOutputFilter INCLUDES .shtml + + +# +# Specify a default charset for all content served; this enables +# interpretation of all content as UTF-8 by default. To use the +# default browser choice (ISO-8859-1), or to allow the META tags +# in HTML content to override this choice, comment out this +# directive: +# +AddDefaultCharset UTF-8 + + + # + # The mod_mime_magic module allows the server to use various hints from the + # contents of the file itself to determine its type. The MIMEMagicFile + # directive tells the module where the hint definitions are located. + # + MIMEMagicFile conf/magic + + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults if commented: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +EnableSendfile on + +# Supplemental configuration +# +# Load config files in the "/etc/httpd/conf.d" directory, if any. +IncludeOptional conf.d/*.conf diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/magic b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/magic new file mode 100644 index 000000000..7c56119e9 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/httpd/conf/magic @@ -0,0 +1,385 @@ +# Magic data for mod_mime_magic Apache module (originally for file(1) command) +# The module is described in /manual/mod/mod_mime_magic.html +# +# The format is 4-5 columns: +# Column #1: byte number to begin checking from, ">" indicates continuation +# Column #2: type of data to match +# Column #3: contents of data to match +# Column #4: MIME type of result +# Column #5: MIME encoding of result (optional) + +#------------------------------------------------------------------------------ +# Localstuff: file(1) magic for locally observed files +# Add any locally observed files here. + +#------------------------------------------------------------------------------ +# end local stuff +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# Java + +0 short 0xcafe +>2 short 0xbabe application/java + +#------------------------------------------------------------------------------ +# audio: file(1) magic for sound formats +# +# from Jan Nicolai Langfeldt , +# + +# Sun/NeXT audio data +0 string .snd +>12 belong 1 audio/basic +>12 belong 2 audio/basic +>12 belong 3 audio/basic +>12 belong 4 audio/basic +>12 belong 5 audio/basic +>12 belong 6 audio/basic +>12 belong 7 audio/basic + +>12 belong 23 audio/x-adpcm + +# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format +# that uses little-endian encoding and has a different magic number +# (0x0064732E in little-endian encoding). +0 lelong 0x0064732E +>12 lelong 1 audio/x-dec-basic +>12 lelong 2 audio/x-dec-basic +>12 lelong 3 audio/x-dec-basic +>12 lelong 4 audio/x-dec-basic +>12 lelong 5 audio/x-dec-basic +>12 lelong 6 audio/x-dec-basic +>12 lelong 7 audio/x-dec-basic +# compressed (G.721 ADPCM) +>12 lelong 23 audio/x-dec-adpcm + +# Bytes 0-3 of AIFF, AIFF-C, & 8SVX audio files are "FORM" +# AIFF audio data +8 string AIFF audio/x-aiff +# AIFF-C audio data +8 string AIFC audio/x-aiff +# IFF/8SVX audio data +8 string 8SVX audio/x-aiff + +# Creative Labs AUDIO stuff +# Standard MIDI data +0 string MThd audio/unknown +#>9 byte >0 (format %d) +#>11 byte >1 using %d channels +# Creative Music (CMF) data +0 string CTMF audio/unknown +# SoundBlaster instrument data +0 string SBI audio/unknown +# Creative Labs voice data +0 string Creative\ Voice\ File audio/unknown +## is this next line right? it came this way... +#>19 byte 0x1A +#>23 byte >0 - version %d +#>22 byte >0 \b.%d + +# [GRR 950115: is this also Creative Labs? Guessing that first line +# should be string instead of unknown-endian long...] +#0 long 0x4e54524b MultiTrack sound data +#0 string NTRK MultiTrack sound data +#>4 long x - version %ld + +# Microsoft WAVE format (*.wav) +# [GRR 950115: probably all of the shorts and longs should be leshort/lelong] +# Microsoft RIFF +0 string RIFF audio/unknown +# - WAVE format +>8 string WAVE audio/x-wav +# MPEG audio. +0 beshort&0xfff0 0xfff0 audio/mpeg +# C64 SID Music files, from Linus Walleij +0 string PSID audio/prs.sid + +#------------------------------------------------------------------------------ +# c-lang: file(1) magic for C programs or various scripts +# + +# XPM icons (Greg Roelofs, newt@uchicago.edu) +# ideally should go into "images", but entries below would tag XPM as C source +0 string /*\ XPM image/x-xbm 7bit + +# this first will upset you if you're a PL/1 shop... (are there any left?) +# in which case rm it; ascmagic will catch real C programs +# C or REXX program text +0 string /* text/plain +# C++ program text +0 string // text/plain + +#------------------------------------------------------------------------------ +# compress: file(1) magic for pure-compression formats (no archives) +# +# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, whap, etc. +# +# Formats for various forms of compressed data +# Formats for "compress" proper have been moved into "compress.c", +# because it tries to uncompress it to figure out what's inside. + +# standard unix compress +0 string \037\235 application/octet-stream x-compress + +# gzip (GNU zip, not to be confused with [Info-ZIP/PKWARE] zip archiver) +0 string \037\213 application/octet-stream x-gzip + +# According to gzip.h, this is the correct byte order for packed data. +0 string \037\036 application/octet-stream +# +# This magic number is byte-order-independent. +# +0 short 017437 application/octet-stream + +# XXX - why *two* entries for "compacted data", one of which is +# byte-order independent, and one of which is byte-order dependent? +# +# compacted data +0 short 0x1fff application/octet-stream +0 string \377\037 application/octet-stream +# huf output +0 short 0145405 application/octet-stream + +# Squeeze and Crunch... +# These numbers were gleaned from the Unix versions of the programs to +# handle these formats. Note that I can only uncrunch, not crunch, and +# I didn't have a crunched file handy, so the crunch number is untested. +# Keith Waclena +#0 leshort 0x76FF squeezed data (CP/M, DOS) +#0 leshort 0x76FE crunched data (CP/M, DOS) + +# Freeze +#0 string \037\237 Frozen file 2.1 +#0 string \037\236 Frozen file 1.0 (or gzip 0.5) + +# lzh? +#0 string \037\240 LZH compressed data + +#------------------------------------------------------------------------------ +# frame: file(1) magic for FrameMaker files +# +# This stuff came on a FrameMaker demo tape, most of which is +# copyright, but this file is "published" as witness the following: +# +0 string \ +# and Anna Shergold +# +0 string \ +0 string \14 byte 12 (OS/2 1.x format) +#>14 byte 64 (OS/2 2.x format) +#>14 byte 40 (Windows 3.x format) +#0 string IC icon +#0 string PI pointer +#0 string CI color icon +#0 string CP color pointer +#0 string BA bitmap array + +0 string \x89PNG image/png +0 string FWS application/x-shockwave-flash +0 string CWS application/x-shockwave-flash + +#------------------------------------------------------------------------------ +# lisp: file(1) magic for lisp programs +# +# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com) +0 string ;; text/plain 8bit +# Emacs 18 - this is always correct, but not very magical. +0 string \012( application/x-elc +# Emacs 19 +0 string ;ELC\023\000\000\000 application/x-elc + +#------------------------------------------------------------------------------ +# mail.news: file(1) magic for mail and news +# +# There are tests to ascmagic.c to cope with mail and news. +0 string Relay-Version: message/rfc822 7bit +0 string #!\ rnews message/rfc822 7bit +0 string N#!\ rnews message/rfc822 7bit +0 string Forward\ to message/rfc822 7bit +0 string Pipe\ to message/rfc822 7bit +0 string Return-Path: message/rfc822 7bit +0 string Path: message/news 8bit +0 string Xref: message/news 8bit +0 string From: message/rfc822 7bit +0 string Article message/news 8bit +#------------------------------------------------------------------------------ +# msword: file(1) magic for MS Word files +# +# Contributor claims: +# Reversed-engineered MS Word magic numbers +# + +0 string \376\067\0\043 application/msword +0 string \333\245-\0\0\0 application/msword + +# disable this one because it applies also to other +# Office/OLE documents for which msword is not correct. See PR#2608. +#0 string \320\317\021\340\241\261 application/msword + + + +#------------------------------------------------------------------------------ +# printer: file(1) magic for printer-formatted files +# + +# PostScript +0 string %! application/postscript +0 string \004%! application/postscript + +# Acrobat +# (due to clamen@cs.cmu.edu) +0 string %PDF- application/pdf + +#------------------------------------------------------------------------------ +# sc: file(1) magic for "sc" spreadsheet +# +38 string Spreadsheet application/x-sc + +#------------------------------------------------------------------------------ +# tex: file(1) magic for TeX files +# +# XXX - needs byte-endian stuff (big-endian and little-endian DVI?) +# +# From + +# Although we may know the offset of certain text fields in TeX DVI +# and font files, we can't use them reliably because they are not +# zero terminated. [but we do anyway, christos] +0 string \367\002 application/x-dvi +#0 string \367\203 TeX generic font data +#0 string \367\131 TeX packed font data +#0 string \367\312 TeX virtual font data +#0 string This\ is\ TeX, TeX transcript text +#0 string This\ is\ METAFONT, METAFONT transcript text + +# There is no way to detect TeX Font Metric (*.tfm) files without +# breaking them apart and reading the data. The following patterns +# match most *.tfm files generated by METAFONT or afm2tfm. +#2 string \000\021 TeX font metric data +#2 string \000\022 TeX font metric data +#>34 string >\0 (%s) + +# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com) +#0 string \\input\ texinfo Texinfo source text +#0 string This\ is\ Info\ file GNU Info text + +# correct TeX magic for Linux (and maybe more) +# from Peter Tobias (tobias@server.et-inf.fho-emden.de) +# +0 leshort 0x02f7 application/x-dvi + +# RTF - Rich Text Format +0 string {\\rtf application/rtf + +#------------------------------------------------------------------------------ +# animation: file(1) magic for animation/movie formats +# +# animation formats, originally from vax@ccwf.cc.utexas.edu (VaX#n8) +# MPEG file +0 string \000\000\001\263 video/mpeg +# +# The contributor claims: +# I couldn't find a real magic number for these, however, this +# -appears- to work. Note that it might catch other files, too, +# so BE CAREFUL! +# +# Note that title and author appear in the two 20-byte chunks +# at decimal offsets 2 and 22, respectively, but they are XOR'ed with +# 255 (hex FF)! DL format SUCKS BIG ROCKS. +# +# DL file version 1 , medium format (160x100, 4 images/screen) +0 byte 1 video/unknown +0 byte 2 video/unknown +# Quicktime video, from Linus Walleij +# from Apple quicktime file format documentation. +4 string moov video/quicktime +4 string mdat video/quicktime + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sites b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sites new file mode 100644 index 000000000..6af1f63fa --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sites @@ -0,0 +1 @@ +conf.d/centos.example.com.conf, centos.example.com diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd new file mode 100644 index 000000000..0bf6b176c --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -0,0 +1,25 @@ +# +# This file can be used to set additional environment variables for +# the httpd process, or pass additional options to the httpd +# executable. +# +# Note: With previous versions of httpd, the MPM could be changed by +# editing an "HTTPD" variable here. With the current version, that +# variable is now ignored. The MPM is a loadable module, and the +# choice of MPM can be changed by editing the configuration file +# /etc/httpd/conf.modules.d/00-mpm.conf. +# + +# +# To pass additional options (for instance, -D definitions) to the +# httpd binary at startup, set OPTIONS here. +# +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" + +# +# This setting ensures the httpd process is started in the "C" locale +# by default. (Some modules will not behave correctly if +# case-sensitive string comparisons are performed in a different +# locale.) +# +LANG=C diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/httpd.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/httpd.conf new file mode 100644 index 000000000..e5693ffff --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/httpd.conf @@ -0,0 +1,157 @@ +# This is a modification of the default Apache 2.4 configuration file +# for Gentoo Linux. +# +# Support: +# http://www.gentoo.org/main/en/lists.xml [mailing lists] +# http://forums.gentoo.org/ [web forums] +# irc://irc.freenode.net#gentoo-apache [irc chat] +# +# Bug Reports: +# http://bugs.gentoo.org [gentoo related bugs] +# http://httpd.apache.org/bug_report.html [apache httpd related bugs] +# +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "var/log/apache2/foo_log" +# with ServerRoot set to "/usr" will be interpreted by the +# server as "/usr/var/log/apache2/foo.log". + +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to point the LockFile directive +# at a local disk. If you wish to share the same ServerRoot for multiple +# httpd daemons, you will need to change at least LockFile and PidFile. +# Comment: The LockFile directive has been replaced by the Mutex directive +ServerRoot "/usr/lib64/apache2" + +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +# GENTOO: Automatically defined based on APACHE2_MODULES USE_EXPAND variable. +# Do not change manually, it will be overwritten on upgrade. +# +# The following modules are considered as the default configuration. +# If you wish to disable one of them, you may have to alter other +# configuration directives. +# +# Change these at your own risk! + +LoadModule actions_module modules/mod_actions.so +LoadModule alias_module modules/mod_alias.so +LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule authn_anon_module modules/mod_authn_anon.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authn_dbm_module modules/mod_authn_dbm.so +LoadModule authn_file_module modules/mod_authn_file.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule authz_dbm_module modules/mod_authz_dbm.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_owner_module modules/mod_authz_owner.so +LoadModule authz_user_module modules/mod_authz_user.so +LoadModule autoindex_module modules/mod_autoindex.so + +LoadModule cache_module modules/mod_cache.so + +LoadModule cgi_module modules/mod_cgi.so +LoadModule cgid_module modules/mod_cgid.so + +LoadModule dav_module modules/mod_dav.so + + +LoadModule dav_fs_module modules/mod_dav_fs.so + + +LoadModule dav_lock_module modules/mod_dav_lock.so + +LoadModule deflate_module modules/mod_deflate.so +LoadModule dir_module modules/mod_dir.so +LoadModule env_module modules/mod_env.so +LoadModule expires_module modules/mod_expires.so +LoadModule ext_filter_module modules/mod_ext_filter.so + +LoadModule file_cache_module modules/mod_file_cache.so + +LoadModule filter_module modules/mod_filter.so +LoadModule headers_module modules/mod_headers.so +LoadModule include_module modules/mod_include.so + +LoadModule info_module modules/mod_info.so + +LoadModule log_config_module modules/mod_log_config.so +LoadModule logio_module modules/mod_logio.so +LoadModule mime_module modules/mod_mime.so +LoadModule mime_magic_module modules/mod_mime_magic.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule setenvif_module modules/mod_setenvif.so + +LoadModule socache_shmcb_module modules/mod_socache_shmcb.so + +LoadModule speling_module modules/mod_speling.so + +LoadModule ssl_module modules/mod_ssl.so + + +LoadModule status_module modules/mod_status.so + +LoadModule unique_id_module modules/mod_unique_id.so +LoadModule unixd_module modules/mod_unixd.so + +LoadModule userdir_module modules/mod_userdir.so + +LoadModule usertrack_module modules/mod_usertrack.so +LoadModule vhost_alias_module modules/mod_vhost_alias.so + +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +User apache +Group apache + +# Supplemental configuration +# +# Most of the configuration files in the /etc/apache2/modules.d/ directory can +# be turned on using APACHE2_OPTS in /etc/conf.d/apache2 to add extra features +# or to modify the default configuration of the server. +# +# To know which flag to add to APACHE2_OPTS, look at the first line of the +# the file, which will usually be an where OPTION is the +# flag to use. + +Include modules.d/*.conf + +# Virtual-host support +# +# Gentoo has made using virtual-hosts easy. In /etc/apache2/vhosts.d/ we +# include a default vhost (enabled by adding -D DEFAULT_VHOST to +# APACHE2_OPTS in /etc/conf.d/apache2). +Include vhosts.d/*.conf + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/magic b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/magic new file mode 100644 index 000000000..7c56119e9 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/magic @@ -0,0 +1,385 @@ +# Magic data for mod_mime_magic Apache module (originally for file(1) command) +# The module is described in /manual/mod/mod_mime_magic.html +# +# The format is 4-5 columns: +# Column #1: byte number to begin checking from, ">" indicates continuation +# Column #2: type of data to match +# Column #3: contents of data to match +# Column #4: MIME type of result +# Column #5: MIME encoding of result (optional) + +#------------------------------------------------------------------------------ +# Localstuff: file(1) magic for locally observed files +# Add any locally observed files here. + +#------------------------------------------------------------------------------ +# end local stuff +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# Java + +0 short 0xcafe +>2 short 0xbabe application/java + +#------------------------------------------------------------------------------ +# audio: file(1) magic for sound formats +# +# from Jan Nicolai Langfeldt , +# + +# Sun/NeXT audio data +0 string .snd +>12 belong 1 audio/basic +>12 belong 2 audio/basic +>12 belong 3 audio/basic +>12 belong 4 audio/basic +>12 belong 5 audio/basic +>12 belong 6 audio/basic +>12 belong 7 audio/basic + +>12 belong 23 audio/x-adpcm + +# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format +# that uses little-endian encoding and has a different magic number +# (0x0064732E in little-endian encoding). +0 lelong 0x0064732E +>12 lelong 1 audio/x-dec-basic +>12 lelong 2 audio/x-dec-basic +>12 lelong 3 audio/x-dec-basic +>12 lelong 4 audio/x-dec-basic +>12 lelong 5 audio/x-dec-basic +>12 lelong 6 audio/x-dec-basic +>12 lelong 7 audio/x-dec-basic +# compressed (G.721 ADPCM) +>12 lelong 23 audio/x-dec-adpcm + +# Bytes 0-3 of AIFF, AIFF-C, & 8SVX audio files are "FORM" +# AIFF audio data +8 string AIFF audio/x-aiff +# AIFF-C audio data +8 string AIFC audio/x-aiff +# IFF/8SVX audio data +8 string 8SVX audio/x-aiff + +# Creative Labs AUDIO stuff +# Standard MIDI data +0 string MThd audio/unknown +#>9 byte >0 (format %d) +#>11 byte >1 using %d channels +# Creative Music (CMF) data +0 string CTMF audio/unknown +# SoundBlaster instrument data +0 string SBI audio/unknown +# Creative Labs voice data +0 string Creative\ Voice\ File audio/unknown +## is this next line right? it came this way... +#>19 byte 0x1A +#>23 byte >0 - version %d +#>22 byte >0 \b.%d + +# [GRR 950115: is this also Creative Labs? Guessing that first line +# should be string instead of unknown-endian long...] +#0 long 0x4e54524b MultiTrack sound data +#0 string NTRK MultiTrack sound data +#>4 long x - version %ld + +# Microsoft WAVE format (*.wav) +# [GRR 950115: probably all of the shorts and longs should be leshort/lelong] +# Microsoft RIFF +0 string RIFF audio/unknown +# - WAVE format +>8 string WAVE audio/x-wav +# MPEG audio. +0 beshort&0xfff0 0xfff0 audio/mpeg +# C64 SID Music files, from Linus Walleij +0 string PSID audio/prs.sid + +#------------------------------------------------------------------------------ +# c-lang: file(1) magic for C programs or various scripts +# + +# XPM icons (Greg Roelofs, newt@uchicago.edu) +# ideally should go into "images", but entries below would tag XPM as C source +0 string /*\ XPM image/x-xbm 7bit + +# this first will upset you if you're a PL/1 shop... (are there any left?) +# in which case rm it; ascmagic will catch real C programs +# C or REXX program text +0 string /* text/plain +# C++ program text +0 string // text/plain + +#------------------------------------------------------------------------------ +# compress: file(1) magic for pure-compression formats (no archives) +# +# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, whap, etc. +# +# Formats for various forms of compressed data +# Formats for "compress" proper have been moved into "compress.c", +# because it tries to uncompress it to figure out what's inside. + +# standard unix compress +0 string \037\235 application/octet-stream x-compress + +# gzip (GNU zip, not to be confused with [Info-ZIP/PKWARE] zip archiver) +0 string \037\213 application/octet-stream x-gzip + +# According to gzip.h, this is the correct byte order for packed data. +0 string \037\036 application/octet-stream +# +# This magic number is byte-order-independent. +# +0 short 017437 application/octet-stream + +# XXX - why *two* entries for "compacted data", one of which is +# byte-order independent, and one of which is byte-order dependent? +# +# compacted data +0 short 0x1fff application/octet-stream +0 string \377\037 application/octet-stream +# huf output +0 short 0145405 application/octet-stream + +# Squeeze and Crunch... +# These numbers were gleaned from the Unix versions of the programs to +# handle these formats. Note that I can only uncrunch, not crunch, and +# I didn't have a crunched file handy, so the crunch number is untested. +# Keith Waclena +#0 leshort 0x76FF squeezed data (CP/M, DOS) +#0 leshort 0x76FE crunched data (CP/M, DOS) + +# Freeze +#0 string \037\237 Frozen file 2.1 +#0 string \037\236 Frozen file 1.0 (or gzip 0.5) + +# lzh? +#0 string \037\240 LZH compressed data + +#------------------------------------------------------------------------------ +# frame: file(1) magic for FrameMaker files +# +# This stuff came on a FrameMaker demo tape, most of which is +# copyright, but this file is "published" as witness the following: +# +0 string \ +# and Anna Shergold +# +0 string \ +0 string \14 byte 12 (OS/2 1.x format) +#>14 byte 64 (OS/2 2.x format) +#>14 byte 40 (Windows 3.x format) +#0 string IC icon +#0 string PI pointer +#0 string CI color icon +#0 string CP color pointer +#0 string BA bitmap array + +0 string \x89PNG image/png +0 string FWS application/x-shockwave-flash +0 string CWS application/x-shockwave-flash + +#------------------------------------------------------------------------------ +# lisp: file(1) magic for lisp programs +# +# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com) +0 string ;; text/plain 8bit +# Emacs 18 - this is always correct, but not very magical. +0 string \012( application/x-elc +# Emacs 19 +0 string ;ELC\023\000\000\000 application/x-elc + +#------------------------------------------------------------------------------ +# mail.news: file(1) magic for mail and news +# +# There are tests to ascmagic.c to cope with mail and news. +0 string Relay-Version: message/rfc822 7bit +0 string #!\ rnews message/rfc822 7bit +0 string N#!\ rnews message/rfc822 7bit +0 string Forward\ to message/rfc822 7bit +0 string Pipe\ to message/rfc822 7bit +0 string Return-Path: message/rfc822 7bit +0 string Path: message/news 8bit +0 string Xref: message/news 8bit +0 string From: message/rfc822 7bit +0 string Article message/news 8bit +#------------------------------------------------------------------------------ +# msword: file(1) magic for MS Word files +# +# Contributor claims: +# Reversed-engineered MS Word magic numbers +# + +0 string \376\067\0\043 application/msword +0 string \333\245-\0\0\0 application/msword + +# disable this one because it applies also to other +# Office/OLE documents for which msword is not correct. See PR#2608. +#0 string \320\317\021\340\241\261 application/msword + + + +#------------------------------------------------------------------------------ +# printer: file(1) magic for printer-formatted files +# + +# PostScript +0 string %! application/postscript +0 string \004%! application/postscript + +# Acrobat +# (due to clamen@cs.cmu.edu) +0 string %PDF- application/pdf + +#------------------------------------------------------------------------------ +# sc: file(1) magic for "sc" spreadsheet +# +38 string Spreadsheet application/x-sc + +#------------------------------------------------------------------------------ +# tex: file(1) magic for TeX files +# +# XXX - needs byte-endian stuff (big-endian and little-endian DVI?) +# +# From + +# Although we may know the offset of certain text fields in TeX DVI +# and font files, we can't use them reliably because they are not +# zero terminated. [but we do anyway, christos] +0 string \367\002 application/x-dvi +#0 string \367\203 TeX generic font data +#0 string \367\131 TeX packed font data +#0 string \367\312 TeX virtual font data +#0 string This\ is\ TeX, TeX transcript text +#0 string This\ is\ METAFONT, METAFONT transcript text + +# There is no way to detect TeX Font Metric (*.tfm) files without +# breaking them apart and reading the data. The following patterns +# match most *.tfm files generated by METAFONT or afm2tfm. +#2 string \000\021 TeX font metric data +#2 string \000\022 TeX font metric data +#>34 string >\0 (%s) + +# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com) +#0 string \\input\ texinfo Texinfo source text +#0 string This\ is\ Info\ file GNU Info text + +# correct TeX magic for Linux (and maybe more) +# from Peter Tobias (tobias@server.et-inf.fho-emden.de) +# +0 leshort 0x02f7 application/x-dvi + +# RTF - Rich Text Format +0 string {\\rtf application/rtf + +#------------------------------------------------------------------------------ +# animation: file(1) magic for animation/movie formats +# +# animation formats, originally from vax@ccwf.cc.utexas.edu (VaX#n8) +# MPEG file +0 string \000\000\001\263 video/mpeg +# +# The contributor claims: +# I couldn't find a real magic number for these, however, this +# -appears- to work. Note that it might catch other files, too, +# so BE CAREFUL! +# +# Note that title and author appear in the two 20-byte chunks +# at decimal offsets 2 and 22, respectively, but they are XOR'ed with +# 255 (hex FF)! DL format SUCKS BIG ROCKS. +# +# DL file version 1 , medium format (160x100, 4 images/screen) +0 byte 1 video/unknown +0 byte 2 video/unknown +# Quicktime video, from Linus Walleij +# from Apple quicktime file format documentation. +4 string moov video/quicktime +4 string mdat video/quicktime + diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_default_settings.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_default_settings.conf new file mode 100644 index 000000000..38635aa9d --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_default_settings.conf @@ -0,0 +1,131 @@ +# This configuration file reflects default settings for Apache HTTP Server. +# You may change these, but chances are that you may not need to. + +# Timeout: The number of seconds before receives and sends time out. +Timeout 300 + +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +KeepAlive On + +# MaxKeepAliveRequests: The maximum number of requests to allow +# during a persistent connection. Set to 0 to allow an unlimited amount. +# We recommend you leave this number high, for maximum performance. +MaxKeepAliveRequests 100 + +# KeepAliveTimeout: Number of seconds to wait for the next request from the +# same client on the same connection. +KeepAliveTimeout 15 + +# UseCanonicalName: Determines how Apache constructs self-referencing +# URLs and the SERVER_NAME and SERVER_PORT variables. +# When set "Off", Apache will use the Hostname and Port supplied +# by the client. When set "On", Apache will use the value of the +# ServerName directive. +UseCanonicalName Off + +# AccessFileName: The name of the file to look for in each directory +# for additional configuration directives. See also the AllowOverride +# directive. +AccessFileName .htaccess + +# ServerTokens +# This directive configures what you return as the Server HTTP response +# Header. The default is 'Full' which sends information about the OS-Type +# and compiled in modules. +# Set to one of: Full | OS | Minor | Minimal | Major | Prod +# where Full conveys the most information, and Prod the least. +ServerTokens Prod + +# TraceEnable +# This directive overrides the behavior of TRACE for both the core server and +# mod_proxy. The default TraceEnable on permits TRACE requests per RFC 2616, +# which disallows any request body to accompany the request. TraceEnable off +# causes the core server and mod_proxy to return a 405 (Method not allowed) +# error to the client. +# For security reasons this is turned off by default. (bug #240680) +TraceEnable off + +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (internal error documents, FTP directory +# listings, mod_status and mod_info output etc., but not CGI generated +# documents or custom error documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +ServerSignature On + +# HostnameLookups: Log the names of clients or just their IP addresses +# e.g., www.apache.org (on) or 204.62.129.132 (off). +# The default is off because it'd be overall better for the net if people +# had to knowingly turn this feature on, since enabling it means that +# each client request will result in AT LEAST one lookup request to the +# nameserver. +HostnameLookups Off + +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall is used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +EnableMMAP On +EnableSendfile Off + +# FileETag: Configures the file attributes that are used to create +# the ETag (entity tag) response header field when the document is +# based on a static file. (The ETag value is used in cache management +# to save network bandwidth.) +FileETag MTime Size + +# ContentDigest: This directive enables the generation of Content-MD5 +# headers as defined in RFC1864 respectively RFC2616. +# The Content-MD5 header provides an end-to-end message integrity +# check (MIC) of the entity-body. A proxy or client may check this +# header for detecting accidental modification of the entity-body +# in transit. +# Note that this can cause performance problems on your server since +# the message digest is computed on every request (the values are +# not cached). +# Content-MD5 is only sent for documents served by the core, and not +# by any module. For example, SSI documents, output from CGI scripts, +# and byte range responses do not have this header. +ContentDigest Off + +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +ErrorLog /var/log/apache2/error_log + +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +LogLevel warn + +# We configure the "default" to be a very restrictive set of features. + + Options FollowSymLinks + AllowOverride None + Require all denied + + +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# +# The index.html.var file (a type-map) is used to deliver content- +# negotiated documents. The MultiViews Options can be used for the +# same purpose, but it is much slower. +# +# Do not change this entry unless you know what you are doing. + + DirectoryIndex index.html index.html.var + + +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. + + Require all denied + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_error_documents.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_error_documents.conf new file mode 100644 index 000000000..61479fa53 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_error_documents.conf @@ -0,0 +1,57 @@ +# The configuration below implements multi-language error documents through +# content-negotiation. + +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html + +# Required modules: mod_alias, mod_include, mod_negotiation +# We use Alias to redirect any /error/HTTP_.html.var response to +# our collection of by-error message multi-language collections. We use +# includes to substitute the appropriate text. +# You can modify the messages' appearance without changing any of the +# default HTTP_.html.var files by adding the line: +# Alias /error/include/ "/your/include/path/" +# which allows you to create your own set of files by starting with the +# /var/www/localhost/error/include/ files and copying them to /your/include/path/, +# even on a per-VirtualHost basis. The default include files will display +# your Apache version number and your ServerAdmin email address regardless +# of the setting of ServerSignature. + + +Alias /error/ "/usr/share/apache2/error/" + + + AllowOverride None + Options IncludesNoExec + AddOutputFilter Includes html + AddHandler type-map var + Require all granted + LanguagePriority en cs de es fr it ja ko nl pl pt-br ro sv tr + ForceLanguagePriority Prefer Fallback + + +ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var +ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var +ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var +ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var +ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var +ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var +ErrorDocument 410 /error/HTTP_GONE.html.var +ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var +ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var +ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var +ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var +ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var +ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var +ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var +ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var +ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var +ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_languages.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_languages.conf new file mode 100644 index 000000000..c429bf94c --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_languages.conf @@ -0,0 +1,133 @@ +# Settings for hosting different languages. + +# DefaultLanguage and AddLanguage allows you to specify the language of +# a document. You can then use content negotiation to give a browser a +# file in a language the user can understand. +# +# Specify a default language. This means that all data +# going out without a specific language tag (see below) will +# be marked with this one. You probably do NOT want to set +# this unless you are sure it is correct for all cases. +# +# It is generally better to not mark a page as +# being a certain language than marking it with the wrong +# language! +# +# DefaultLanguage nl +# +# Note 1: The suffix does not have to be the same as the language +# keyword --- those with documents in Polish (whose net-standard +# language code is pl) may wish to use "AddLanguage pl .po" to +# avoid the ambiguity with the common suffix for perl scripts. +# +# Note 2: The example entries below illustrate that in some cases +# the two character 'Language' abbreviation is not identical to +# the two character 'Country' code for its country, +# E.g. 'Danmark/dk' versus 'Danish/da'. +# +# Note 3: In the case of 'ltz' we violate the RFC by using a three char +# specifier. There is 'work in progress' to fix this and get +# the reference data for rfc1766 cleaned up. +# +# Catalan (ca) - Croatian (hr) - Czech (cs) - Danish (da) - Dutch (nl) +# English (en) - Esperanto (eo) - Estonian (et) - French (fr) - German (de) +# Greek-Modern (el) - Hebrew (he) - Italian (it) - Japanese (ja) +# Korean (ko) - Luxembourgeois* (ltz) - Norwegian Nynorsk (nn) +# Norwegian (no) - Polish (pl) - Portugese (pt) +# Brazilian Portuguese (pt-BR) - Russian (ru) - Swedish (sv) +# Simplified Chinese (zh-CN) - Spanish (es) - Traditional Chinese (zh-TW) +AddLanguage ca .ca +AddLanguage cs .cz .cs +AddLanguage da .dk +AddLanguage de .de +AddLanguage el .el +AddLanguage en .en +AddLanguage eo .eo +AddLanguage es .es +AddLanguage et .et +AddLanguage fr .fr +AddLanguage he .he +AddLanguage hr .hr +AddLanguage it .it +AddLanguage ja .ja +AddLanguage ko .ko +AddLanguage ltz .ltz +AddLanguage nl .nl +AddLanguage nn .nn +AddLanguage no .no +AddLanguage pl .po +AddLanguage pt .pt +AddLanguage pt-BR .pt-br +AddLanguage ru .ru +AddLanguage sv .sv +AddLanguage zh-CN .zh-cn +AddLanguage zh-TW .zh-tw + +# LanguagePriority allows you to give precedence to some languages +# in case of a tie during content negotiation. +# +# Just list the languages in decreasing order of preference. We have +# more or less alphabetized them here. You probably want to change this. +LanguagePriority en ca cs da de el eo es et fr he hr it ja ko ltz nl nn no pl pt pt-BR ru sv zh-CN zh-TW + +# ForceLanguagePriority allows you to serve a result page rather than +# MULTIPLE CHOICES (Prefer) [in case of a tie] or NOT ACCEPTABLE (Fallback) +# [in case no accepted languages matched the available variants] +ForceLanguagePriority Prefer Fallback + +# Commonly used filename extensions to character sets. You probably +# want to avoid clashes with the language extensions, unless you +# are good at carefully testing your setup after each change. +# See http://www.iana.org/assignments/character-sets for the +# official list of charset names and their respective RFCs. +AddCharset us-ascii.ascii .us-ascii +AddCharset ISO-8859-1 .iso8859-1 .latin1 +AddCharset ISO-8859-2 .iso8859-2 .latin2 .cen +AddCharset ISO-8859-3 .iso8859-3 .latin3 +AddCharset ISO-8859-4 .iso8859-4 .latin4 +AddCharset ISO-8859-5 .iso8859-5 .cyr .iso-ru +AddCharset ISO-8859-6 .iso8859-6 .arb .arabic +AddCharset ISO-8859-7 .iso8859-7 .grk .greek +AddCharset ISO-8859-8 .iso8859-8 .heb .hebrew +AddCharset ISO-8859-9 .iso8859-9 .latin5 .trk +AddCharset ISO-8859-10 .iso8859-10 .latin6 +AddCharset ISO-8859-13 .iso8859-13 +AddCharset ISO-8859-14 .iso8859-14 .latin8 +AddCharset ISO-8859-15 .iso8859-15 .latin9 +AddCharset ISO-8859-16 .iso8859-16 .latin10 +AddCharset ISO-2022-JP .iso2022-jp .jis +AddCharset ISO-2022-KR .iso2022-kr .kis +AddCharset ISO-2022-CN .iso2022-cn .cis +AddCharset Big5.Big5 .big5 .b5 +AddCharset cn-Big5 .cn-big5 +# For russian, more than one charset is used (depends on client, mostly): +AddCharset WINDOWS-1251 .cp-1251 .win-1251 +AddCharset CP866 .cp866 +AddCharset KOI8 .koi8 +AddCharset KOI8-E .koi8-e +AddCharset KOI8-r .koi8-r .koi8-ru +AddCharset KOI8-U .koi8-u +AddCharset KOI8-ru .koi8-uk .ua +AddCharset ISO-10646-UCS-2 .ucs2 +AddCharset ISO-10646-UCS-4 .ucs4 +AddCharset UTF-7 .utf7 +AddCharset UTF-8 .utf8 +AddCharset UTF-16 .utf16 +AddCharset UTF-16BE .utf16be +AddCharset UTF-16LE .utf16le +AddCharset UTF-32 .utf32 +AddCharset UTF-32BE .utf32be +AddCharset UTF-32LE .utf32le +AddCharset euc-cn .euc-cn +AddCharset euc-gb .euc-gb +AddCharset euc-jp .euc-jp +AddCharset euc-kr .euc-kr +# Not sure how euc-tw got in - IANA doesn't list it??? +AddCharset EUC-TW .euc-tw +AddCharset gb2312 .gb2312 .gb +AddCharset iso-10646-ucs-2 .ucs-2 .iso-10646-ucs-2 +AddCharset iso-10646-ucs-4 .ucs-4 .iso-10646-ucs-4 +AddCharset shift_jis .shift_jis .sjis + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_autoindex.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_autoindex.conf new file mode 100644 index 000000000..10bf48317 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_autoindex.conf @@ -0,0 +1,85 @@ + + + + +# We include the /icons/ alias for FancyIndexed directory listings. If +# you do not use FancyIndexing, you may comment this out. +Alias /icons/ "/usr/share/apache2/icons/" + + + Options Indexes MultiViews + AllowOverride None + Require all granted + + + +# Directives controlling the display of server-generated directory listings. +# +# To see the listing of a directory, the Options directive for the +# directory must include "Indexes", and the directory must not contain +# a file matching those listed in the DirectoryIndex directive. + +# IndexOptions: Controls the appearance of server-generated directory +# listings. +IndexOptions FancyIndexing VersionSort + +# AddIcon* directives tell the server which icon to show for different +# files or filename extensions. These are only displayed for +# FancyIndexed directories. +AddIconByEncoding (CMP,/icons/compressed.gif) x-compress x-gzip + +AddIconByType (TXT,/icons/text.gif) text/* +AddIconByType (IMG,/icons/image2.gif) image/* +AddIconByType (SND,/icons/sound2.gif) audio/* +AddIconByType (VID,/icons/movie.gif) video/* + +AddIcon /icons/binary.gif .bin .exe +AddIcon /icons/binhex.gif .hqx +AddIcon /icons/tar.gif .tar +AddIcon /icons/world2.gif .wrl .wrl.gz .vrml .vrm .iv +AddIcon /icons/compressed.gif .Z .z .tgz .gz .zip +AddIcon /icons/a.gif .ps .ai .eps +AddIcon /icons/layout.gif .html .shtml .htm .pdf +AddIcon /icons/text.gif .txt +AddIcon /icons/c.gif .c +AddIcon /icons/p.gif .pl .py +AddIcon /icons/f.gif .for +AddIcon /icons/dvi.gif .dvi +AddIcon /icons/uuencoded.gif .uu +AddIcon /icons/script.gif .conf .sh .shar .csh .ksh .tcl +AddIcon /icons/tex.gif .tex +AddIcon /icons/bomb.gif core + +AddIcon /icons/back.gif .. +AddIcon /icons/hand.right.gif README +AddIcon /icons/folder.gif ^^DIRECTORY^^ +AddIcon /icons/blank.gif ^^BLANKICON^^ + +# DefaultIcon is which icon to show for files which do not have an icon +# explicitly set. +DefaultIcon /icons/unknown.gif + +# AddDescription allows you to place a short description after a file in +# server-generated indexes. These are only displayed for FancyIndexed +# directories. +# Format: AddDescription "description" filename + +#AddDescription "GZIP compressed document" .gz +#AddDescription "tar archive" .tar +#AddDescription "GZIP compressed tar archive" .tgz + +# ReadmeName is the name of the README file the server will look for by +# default, and append to directory listings. + +# HeaderName is the name of a file which should be prepended to +# directory indexes. +ReadmeName README.html +HeaderName HEADER.html + +# IndexIgnore is a set of filenames which directory indexing should ignore +# and not include in the listing. Shell-style wildcarding is permitted. +IndexIgnore .??* *~ *# HEADER* README* RCS CVS *,v *,t + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_info.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_info.conf new file mode 100644 index 000000000..2cd32c477 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_info.conf @@ -0,0 +1,10 @@ + +# Allow remote server configuration reports, with the URL of +# http://servername/server-info + + SetHandler server-info + Require local + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_log_config.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_log_config.conf new file mode 100644 index 000000000..ce0238eee --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_log_config.conf @@ -0,0 +1,35 @@ + +# The following directives define some format nicknames for use with +# a CustomLog directive (see below). +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common + +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-Agent}i" agent +LogFormat "%v %h %l %u %t \"%r\" %>s %b %T" script +LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" VLOG=%{VLOG}e" vhost + + +# You need to enable mod_logio.c to use %I and %O +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio +LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" vhostio + + +# The location and format of the access logfile (Common Logfile Format). +# If you do not define any access logfiles within a +# container, they will be logged here. Contrariwise, if you *do* +# define per- access logfiles, transactions will be +# logged therein and *not* in this file. +CustomLog /var/log/apache2/access_log common + +# If you would like to have agent and referer logfiles, +# uncomment the following directives. +#CustomLog /var/log/apache2/referer_log referer +#CustomLog /var/log/apache2/agent_logs agent + +# If you prefer a logfile with access, agent, and referer information +# (Combined Logfile Format) you can use the following directive. +#CustomLog /var/log/apache2/access_log combined + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_mime.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_mime.conf new file mode 100644 index 000000000..fb8a9a5d5 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_mime.conf @@ -0,0 +1,46 @@ + +# TypesConfig points to the file containing the list of mappings from +# filename extension to MIME-type. +TypesConfig /etc/mime.types + +# AddType allows you to add to or override the MIME configuration +# file specified in TypesConfig for specific file types. +#AddType application/x-gzip .tgz + +# AddEncoding allows you to have certain browsers uncompress +# information on the fly. Note: Not all browsers support this. +#AddEncoding x-compress .Z +#AddEncoding x-gzip .gz .tgz + +# If the AddEncoding directives above are commented-out, then you +# probably should define those extensions to indicate media types: +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz + +# AddHandler allows you to map certain file extensions to "handlers": +# actions unrelated to filetype. These can be either built into the server +# or added with the Action directive (see below) + +# To use CGI scripts outside of ScriptAliased directories: +# (You will also need to add "ExecCGI" to the "Options" directive.) +#AddHandler cgi-script .cgi + +# For type maps (negotiated resources): +#AddHandler type-map var + +# Filters allow you to process content before it is sent to the client. +# +# To parse .shtml files for server-side includes (SSI): +# (You will also need to add "Includes" to the "Options" directive.) +#AddType text/html .shtml +#AddOutputFilter INCLUDES .shtml + + + +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +MIMEMagicFile /etc/apache2/magic + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_status.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_status.conf new file mode 100644 index 000000000..ed8b3c7cb --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_status.conf @@ -0,0 +1,15 @@ + +# Allow server status reports generated by mod_status, +# with the URL of http://servername/server-status + + SetHandler server-status + Require local + + +# ExtendedStatus controls whether Apache will generate "full" status +# information (ExtendedStatus On) or just basic information (ExtendedStatus +# Off) when the "server-status" handler is called. +ExtendedStatus On + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_userdir.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_userdir.conf new file mode 100644 index 000000000..0087126c4 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mod_userdir.conf @@ -0,0 +1,32 @@ +# Settings for user home directories + +# UserDir: The name of the directory that is appended onto a user's home +# directory if a ~user request is received. Note that you must also set +# the default access control for these directories, as in the example below. +UserDir public_html + +# Control access to UserDir directories. The following is an example +# for a site where these directories are restricted to read-only. + + AllowOverride FileInfo AuthConfig Limit Indexes + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + + Require all granted + + + Require all denied + + + +# Suexec isn't really required to run cgi-scripts, but it's a really good +# idea if you have multiple users serving websites... + + + Options ExecCGI + SetHandler cgi-script + + + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mpm.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mpm.conf new file mode 100644 index 000000000..bcb9b6b47 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/00_mpm.conf @@ -0,0 +1,99 @@ +# Server-Pool Management (MPM specific) + +# PidFile: The file in which the server should record its process +# identification number when it starts. +# +# DO NOT CHANGE UNLESS YOU KNOW WHAT YOU ARE DOING +PidFile /run/apache2.pid + +# The accept serialization lock file MUST BE STORED ON A LOCAL DISK. +# Mutex file:/run/apache_mpm_mutex + +# Only one of the below sections will be relevant on your +# installed httpd. Use "/usr/sbin/apache2 -l" to find out the +# active mpm. + +# common MPM configuration +# These configuration directives apply to all MPMs +# +# StartServers: Number of child server processes created at startup +# MaxRequestWorkers: Maximum number of child processes to serve requests +# MaxConnectionsPerChild: Limit on the number of connections that an individual +# child server will handle during its life + + +# prefork MPM +# This is the default MPM if USE=-threads +# +# MinSpareServers: Minimum number of idle child server processes +# MaxSpareServers: Maximum number of idle child server processes + + StartServers 5 + MinSpareServers 5 + MaxSpareServers 10 + MaxRequestWorkers 150 + MaxConnectionsPerChild 10000 + + +# worker MPM +# This is the default MPM if USE=threads +# +# MinSpareThreads: Minimum number of idle threads available to handle request spikes +# MaxSpareThreads: Maximum number of idle threads +# ThreadsPerChild: Number of threads created by each child process + + StartServers 2 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadsPerChild 25 + MaxRequestWorkers 150 + MaxConnectionsPerChild 10000 + + +# event MPM +# +# MinSpareThreads: Minimum number of idle threads available to handle request spikes +# MaxSpareThreads: Maximum number of idle threads +# ThreadsPerChild: Number of threads created by each child process + + StartServers 2 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadsPerChild 25 + MaxRequestWorkers 150 + MaxConnectionsPerChild 10000 + + +# peruser MPM +# +# MinSpareProcessors: Minimum number of idle child server processes +# MinProcessors: Minimum number of processors per virtual host +# MaxProcessors: Maximum number of processors per virtual host +# ExpireTimeout: Maximum idle time before a child is killed, 0 to disable +# Multiplexer: Specify a Multiplexer child configuration. +# Processor: Specify a user and group for a specific child process + + MinSpareProcessors 2 + MinProcessors 2 + MaxProcessors 10 + MaxRequestWorkers 150 + MaxConnectionsPerChild 1000 + ExpireTimeout 1800 + + Multiplexer nobody nobody + Processor apache apache + + +# itk MPM +# +# MinSpareServers: Minimum number of idle child server processes +# MaxSpareServers: Maximum number of idle child server processes + + StartServers 5 + MinSpareServers 5 + MaxSpareServers 10 + MaxRequestWorkers 150 + MaxConnectionsPerChild 10000 + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/10_mod_mem_cache.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/10_mod_mem_cache.conf new file mode 100644 index 000000000..520d9fd82 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/10_mod_mem_cache.conf @@ -0,0 +1,10 @@ + +# 128MB cache for objects < 2MB +CacheEnable mem / +MCacheSize 131072 +MCacheMaxObjectCount 1000 +MCacheMinObjectSize 1 +MCacheMaxObjectSize 2097152 + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/40_mod_ssl.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/40_mod_ssl.conf new file mode 100644 index 000000000..f51de4641 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/40_mod_ssl.conf @@ -0,0 +1,67 @@ +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + + + +# This is the Apache server configuration file providing SSL support. +# It contains the configuration directives to instruct the server how to +# serve pages over an https connection. For detailing information about these +# directives see + +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. + +## Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the SSL library. +# The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed startup file:/dev/urandom 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +## SSL Global Context: +# All SSL configuration in this context applies both to the main server and +# all SSL-enabled virtual hosts. + +# Some MIME-types for downloading Certificates and CRLs + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + +## Pass Phrase Dialog: +# Configure the pass phrase gathering process. The filtering dialog program +# (`builtin' is a internal terminal dialog) has to provide the pass phrase on +# stdout. +SSLPassPhraseDialog builtin + +## Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism to use and second the +# expiring timeout (in seconds). +#SSLSessionCache dbm:/run/ssl_scache +SSLSessionCache shmcb:/run/ssl_scache(512000) +SSLSessionCacheTimeout 300 + +## Semaphore: +# Configure the path to the mutual exclusion semaphore the SSL engine uses +# internally for inter-process synchronization. +Mutex file:/run/apache_ssl_mutex ssl-cache + +## SSL Compression: +# Known to be vulnerable thus disabled by default (bug #507324). +SSLCompression off + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/41_mod_http2.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/41_mod_http2.conf new file mode 100644 index 000000000..e4c9454e0 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/41_mod_http2.conf @@ -0,0 +1,9 @@ + + + # enable debugging for this module + #LogLevel http2:info + + #Enable HTTP/2 support + Protocols h2 h2c http/1.1 + + diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/45_mod_dav.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/45_mod_dav.conf new file mode 100644 index 000000000..36f6b9cca --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/45_mod_dav.conf @@ -0,0 +1,19 @@ + +DavLockDB "/var/lib/dav/lockdb" + +# The following directives disable redirects on non-GET requests for +# a directory that does not include the trailing slash. This fixes a +# problem with several clients that do not appropriately handle +# redirects for folders with DAV methods. + +BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully +BrowserMatch "MS FrontPage" redirect-carefully +BrowserMatch "^WebDrive" redirect-carefully +BrowserMatch "^WebDAVFS/1.[012345678]" redirect-carefully +BrowserMatch "^gnome-vfs/1.0" redirect-carefully +BrowserMatch "^XML Spy" redirect-carefully +BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/46_mod_ldap.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/46_mod_ldap.conf new file mode 100644 index 000000000..883061fee --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/46_mod_ldap.conf @@ -0,0 +1,18 @@ +# Examples below are taken from the online documentation +# Refer to: +# http://localhost/manual/mod/mod_ldap.html +# http://localhost/manual/mod/mod_auth_ldap.html + +LDAPSharedCacheSize 200000 +LDAPCacheEntries 1024 +LDAPCacheTTL 600 +LDAPOpCacheEntries 1024 +LDAPOpCacheTTL 600 + + + SetHandler ldap-status + Require local + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_ssl_vhost.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_ssl_vhost.conf new file mode 100644 index 000000000..bb395473c --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_ssl_vhost.conf @@ -0,0 +1,191 @@ + + + +# see bug #178966 why this is in here + +# When we also provide SSL we have to listen to the HTTPS port +# Note: Configurations that use IPv6 but not IPv4-mapped addresses need two +# Listen directives: "Listen [::]:443" and "Listen 0.0.0.0:443" +Listen 443 + + + ServerName localhost + Include /etc/apache2/vhosts.d/default_vhost.include + ErrorLog /var/log/apache2/ssl_error_log + + + TransferLog /var/log/apache2/ssl_access_log + + + ## SSL Engine Switch: + # Enable/Disable SSL for this virtual host. + SSLEngine on + + ## SSLProtocol: + # Don't use SSLv2 anymore as it's considered to be broken security-wise. + # Also disable SSLv3 as most modern browsers are capable of TLS. + SSLProtocol ALL -SSLv2 -SSLv3 + + ## SSL Cipher Suite: + # List the ciphers that the client is permitted to negotiate. + # See the mod_ssl documentation for a complete list. + # This list of ciphers is recommended by mozilla and was stripped off + # its RC4 ciphers. (bug #506924) + SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128:AES256:HIGH:!RC4:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK + + ## SSLHonorCipherOrder: + # Prefer the server's cipher preference order as the client may have a + # weak default order. + SSLHonorCipherOrder On + + ## Server Certificate: + # Point SSLCertificateFile at a PEM encoded certificate. If the certificate + # is encrypted, then you will be prompted for a pass phrase. Note that a + # kill -HUP will prompt again. Keep in mind that if you have both an RSA + # and a DSA certificate you can configure both in parallel (to also allow + # the use of DSA ciphers, etc.) + SSLCertificateFile /etc/ssl/apache2/server.crt + + ## Server Private Key: + # If the key is not combined with the certificate, use this directive to + # point at the key file. Keep in mind that if you've both a RSA and a DSA + # private key you can configure both in parallel (to also allow the use of + # DSA ciphers, etc.) + SSLCertificateKeyFile /etc/ssl/apache2/server.key + + ## Server Certificate Chain: + # Point SSLCertificateChainFile at a file containing the concatenation of + # PEM encoded CA certificates which form the certificate chain for the + # server certificate. Alternatively the referenced file can be the same as + # SSLCertificateFile when the CA certificates are directly appended to the + # server certificate for convinience. + #SSLCertificateChainFile /etc/ssl/apache2/ca.crt + + ## Certificate Authority (CA): + # Set the CA certificate verification path where to find CA certificates + # for client authentication or alternatively one huge file containing all + # of them (file must be PEM encoded). + # Note: Inside SSLCACertificatePath you need hash symlinks to point to the + # certificate files. Use the provided Makefile to update the hash symlinks + # after changes. + #SSLCACertificatePath /etc/ssl/apache2/ssl.crt + #SSLCACertificateFile /etc/ssl/apache2/ca-bundle.crt + + ## Certificate Revocation Lists (CRL): + # Set the CA revocation path where to find CA CRLs for client authentication + # or alternatively one huge file containing all of them (file must be PEM + # encoded). + # Note: Inside SSLCARevocationPath you need hash symlinks to point to the + # certificate files. Use the provided Makefile to update the hash symlinks + # after changes. + #SSLCARevocationPath /etc/ssl/apache2/ssl.crl + #SSLCARevocationFile /etc/ssl/apache2/ca-bundle.crl + + ## Client Authentication (Type): + # Client certificate verification type and depth. Types are none, optional, + # require and optional_no_ca. Depth is a number which specifies how deeply + # to verify the certificate issuer chain before deciding the certificate is + # not valid. + #SSLVerifyClient require + #SSLVerifyDepth 10 + + ## Access Control: + # With SSLRequire you can do per-directory access control based on arbitrary + # complex boolean expressions containing server variable checks and other + # lookup directives. The syntax is a mixture between C and Perl. See the + # mod_ssl documentation for more details. + # + # #SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ + # and %{SSL_CLIENT_S_DN_O} eq "Snake Oil, Ltd." \ + # and %{SSL_CLIENT_S_DN_OU} in {"Staff", "CA", "Dev"} \ + # and %{TIME_WDAY} >= 1 and %{TIME_WDAY} <= 5 \ + # and %{TIME_HOUR} >= 8 and %{TIME_HOUR} <= 20 ) \ + # or %{REMOTE_ADDR} =~ m/^192\.76\.162\.[0-9]+$/ + # + + ## SSL Engine Options: + # Set various options for the SSL engine. + + ## FakeBasicAuth: + # Translate the client X.509 into a Basic Authorisation. This means that the + # standard Auth/DBMAuth methods can be used for access control. The user + # name is the `one line' version of the client's X.509 certificate. + # Note that no password is obtained from the user. Every entry in the user + # file needs this password: `xxj31ZMTZzkVA'. + + ## ExportCertData: + # This exports two additional environment variables: SSL_CLIENT_CERT and + # SSL_SERVER_CERT. These contain the PEM-encoded certificates of the server + # (always existing) and the client (only existing when client + # authentication is used). This can be used to import the certificates into + # CGI scripts. + + ## StdEnvVars: + # This exports the standard SSL/TLS related `SSL_*' environment variables. + # Per default this exportation is switched off for performance reasons, + # because the extraction step is an expensive operation and is usually + # useless for serving static content. So one usually enables the exportation + # for CGI and SSI requests only. + + ## StrictRequire: + # This denies access when "SSLRequireSSL" or "SSLRequire" applied even under + # a "Satisfy any" situation, i.e. when it applies access is denied and no + # other module can change it. + + ## OptRenegotiate: + # This enables optimized SSL connection renegotiation handling when SSL + # directives are used in per-directory context. + #SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire + + SSLOptions +StdEnvVars + + + + SSLOptions +StdEnvVars + + + ## SSL Protocol Adjustments: + # The safe and default but still SSL/TLS standard compliant shutdown + # approach is that mod_ssl sends the close notify alert but doesn't wait + # for the close notify alert from client. When you need a different + # shutdown approach you can use one of the following variables: + + ## ssl-unclean-shutdown: + # This forces an unclean shutdown when the connection is closed, i.e. no + # SSL close notify alert is send or allowed to received. This violates the + # SSL/TLS standard but is needed for some brain-dead browsers. Use this when + # you receive I/O errors because of the standard approach where mod_ssl + # sends the close notify alert. + + ## ssl-accurate-shutdown: + # This forces an accurate shutdown when the connection is closed, i.e. a + # SSL close notify alert is send and mod_ssl waits for the close notify + # alert of the client. This is 100% SSL/TLS standard compliant, but in + # practice often causes hanging connections with brain-dead browsers. Use + # this only for browsers where you know that their SSL implementation works + # correctly. + # Notice: Most problems of broken clients are also related to the HTTP + # keep-alive facility, so you usually additionally want to disable + # keep-alive for those clients, too. Use variable "nokeepalive" for this. + # Similarly, one has to force some clients to use HTTP/1.0 to workaround + # their broken HTTP/1.1 implementation. Use variables "downgrade-1.0" and + # "force-response-1.0" for this. + + BrowserMatch ".*MSIE.*" \ + nokeepalive ssl-unclean-shutdown \ + downgrade-1.0 force-response-1.0 + + + ## Per-Server Logging: + # The home of a custom SSL log file. Use this when you want a compact + # non-error SSL logfile on a virtual host basis. + + CustomLog /var/log/apache2/ssl_request_log \ + "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + + + + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_vhost.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_vhost.conf new file mode 100644 index 000000000..b9766b5f1 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/00_default_vhost.conf @@ -0,0 +1,45 @@ +# Virtual Hosts +# +# If you want to maintain multiple domains/hostnames on your +# machine you can setup VirtualHost containers for them. Most configurations +# use only name-based virtual hosts so the server doesn't need to worry about +# IP addresses. This is indicated by the asterisks in the directives below. +# +# Please see the documentation at +# +# for further details before you try to setup virtual hosts. +# +# You may use the command line option '-S' to verify your virtual host +# configuration. + + +# see bug #178966 why this is in here + +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# When virtual hosts are enabled, the main host defined in the default +# httpd.conf configuration will go away. We redefine it here so that it is +# still available. +# +# If you disable this vhost by removing -D DEFAULT_VHOST from +# /etc/conf.d/apache2, the first defined virtual host elsewhere will be +# the default. + + ServerName localhost + Include /etc/apache2/vhosts.d/default_vhost.include + + + ServerEnvironment apache apache + + + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/default_vhost.include b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/default_vhost.include new file mode 100644 index 000000000..af6ece85b --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/default_vhost.include @@ -0,0 +1,71 @@ +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +ServerAdmin root@localhost + +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +# If you change this to something that isn't under /var/www then suexec +# will no longer work. +DocumentRoot "/var/www/localhost/htdocs" + +# This should be changed to whatever you set DocumentRoot to. + + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + Options Indexes FollowSymLinks + + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # Options FileInfo AuthConfig Limit + AllowOverride All + + # Controls who can get stuff from this server. + Require all granted + + + + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + ScriptAlias /cgi-bin/ "/var/www/localhost/cgi-bin/" + + +# "/var/www/localhost/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. + + AllowOverride None + Options None + Require all granted + + +# vim: ts=4 filetype=apache diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/gentoo.example.com.conf b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/gentoo.example.com.conf new file mode 100644 index 000000000..41de4d236 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/gentoo.example.com.conf @@ -0,0 +1,7 @@ + + ServerName gentoo.example.com + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/conf.d/apache2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/conf.d/apache2 new file mode 100644 index 000000000..b7ecb4f2a --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/conf.d/apache2 @@ -0,0 +1,74 @@ +# /etc/conf.d/apache2: config file for /etc/init.d/apache2 + +# When you install a module it is easy to activate or deactivate the modules +# and other features of apache using the APACHE2_OPTS line. Every module should +# install a configuration in /etc/apache2/modules.d. In that file will have an +# directive where NNN is the option to enable that module. +# +# Here are the options available in the default configuration: +# +# AUTH_DIGEST Enables mod_auth_digest +# AUTHNZ_LDAP Enables authentication through mod_ldap (available if USE=ldap) +# CACHE Enables mod_cache +# DAV Enables mod_dav +# ERRORDOCS Enables default error documents for many languages. +# INFO Enables mod_info, a useful module for debugging +# LANGUAGE Enables content-negotiation based on language and charset. +# LDAP Enables mod_ldap (available if USE=ldap) +# MANUAL Enables /manual/ to be the apache manual (available if USE=docs) +# MEM_CACHE Enables default configuration mod_mem_cache +# PROXY Enables mod_proxy +# SSL Enables SSL (available if USE=ssl) +# STATUS Enabled mod_status, a useful module for statistics +# SUEXEC Enables running CGI scripts (in USERDIR) through suexec. +# USERDIR Enables /~username mapping to /home/username/public_html +# +# +# The following two options provide the default virtual host for the HTTP and +# HTTPS protocol. YOU NEED TO ENABLE AT LEAST ONE OF THEM, otherwise apache +# will not listen for incomming connections on the approriate port. +# +# DEFAULT_VHOST Enables name-based virtual hosts, with the default +# virtual host being in /var/www/localhost/htdocs +# SSL_DEFAULT_VHOST Enables default vhost for SSL (you should enable this +# when you enable SSL) +# +APACHE2_OPTS="-D DEFAULT_VHOST -D INFO -D SSL -D SSL_DEFAULT_VHOST -D LANGUAGE" + +# Extended options for advanced uses of Apache ONLY +# You don't need to edit these unless you are doing crazy Apache stuff +# As not having them set correctly, or feeding in an incorrect configuration +# via them will result in Apache failing to start +# YOU HAVE BEEN WARNED. + +# PID file +#PIDFILE=/var/run/apache2.pid + +# timeout for startup/shutdown checks +#TIMEOUT=10 + +# ServerRoot setting +#SERVERROOT=/usr/lib64/apache2 + +# Configuration file location +# - If this does NOT start with a '/', then it is treated relative to +# $SERVERROOT by Apache +#CONFIGFILE=/etc/apache2/httpd.conf + +# Location to log startup errors to +# They are normally dumped to your terminal. +#STARTUPERRORLOG="/var/log/apache2/startuperror.log" + +# A command that outputs a formatted text version of the HTML at the URL +# of the command line. Designed for lynx, however other programs may work. +#LYNX="lynx -dump" + +# The URL to your server's mod_status status page. +# Required for status and fullstatus +#STATUSURL="http://localhost/server-status" + +# Method to use when reloading the server +# Valid options are 'restart' and 'graceful' +# See http://httpd.apache.org/docs/2.2/stopping.html for information on +# what they do and how they differ. +#RELOAD_TYPE="graceful" diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/sites b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/sites new file mode 100644 index 000000000..7f0b3a8b3 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/sites @@ -0,0 +1,3 @@ +vhosts.d/gentoo.example.com.conf, gentoo.example.com +vhosts.d/00_default_vhost.conf, localhost +vhosts.d/00_default_ssl_vhost.conf, localhost diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 62464d5d0..6c37c2ecc 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -23,7 +23,8 @@ class TlsSniPerformTest(util.ApacheTest): super(TlsSniPerformTest, self).setUp() config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir) config.config.tls_sni_01_port = 443 from certbot_apache import tls_sni_01 @@ -41,8 +42,8 @@ class TlsSniPerformTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") @mock.patch("certbot.util.run_script") def test_perform1(self, _, mock_exists): - mock_register = mock.Mock() - self.sni.configurator.reverter.register_undo_command = mock_register + self.sni.configurator.parser.modules.add("socache_shmcb_module") + self.sni.configurator.parser.modules.add("ssl_module") mock_exists.return_value = True self.sni.configurator.parser.update_runtime_variables = mock.Mock() @@ -55,10 +56,6 @@ class TlsSniPerformTest(util.ApacheTest): self.sni._setup_challenge_cert = mock_setup_cert responses = self.sni.perform() - - # Make sure that register_undo_command was called into temp directory. - self.assertEqual(True, mock_register.call_args[0][0]) - mock_setup_cert.assert_called_once_with(achall) # Check to make sure challenge config path is included in apache config @@ -71,7 +68,7 @@ class TlsSniPerformTest(util.ApacheTest): def test_perform2(self): # Avoid load module self.sni.configurator.parser.modules.add("ssl_module") - + self.sni.configurator.parser.modules.add("socache_shmcb_module") acme_responses = [] for achall in self.achalls: self.sni.add_chall(achall) @@ -81,7 +78,8 @@ class TlsSniPerformTest(util.ApacheTest): # pylint: disable=protected-access self.sni._setup_challenge_cert = mock_setup_cert - with mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod"): + with mock.patch( + "certbot_apache.override_debian.DebianConfigurator.enable_mod"): sni_responses = self.sni.perform() self.assertEqual(mock_setup_cert.call_count, 2) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 34d2476f7..2405110c5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -1,5 +1,6 @@ """Common utilities for certbot_apache.""" import os +import shutil import sys import unittest @@ -16,7 +17,7 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_apache import configurator -from certbot_apache import constants +from certbot_apache import entrypoint from certbot_apache import obj @@ -38,6 +39,9 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + self.config = get_apache_configurator(self.config_path, vhost_root, + self.config_dir, self.work_dir) + # Make sure all vhosts in sites-enabled are symlinks (Python packaging # does not preserve symlinks) sites_enabled = os.path.join(self.config_path, "sites-enabled") @@ -55,8 +59,13 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods os.path.pardir, "sites-available", vhost_basename) os.symlink(target, vhost) + def tearDown(self): + shutil.rmtree(self.temp_dir) + shutil.rmtree(self.config_dir) + shutil.rmtree(self.work_dir) -class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods + +class ParserTest(ApacheTest): def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts", config_root="debian_apache_2_4/multiple_vhosts/apache2", @@ -72,12 +81,16 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods with mock.patch("certbot_apache.parser.ApacheParser." "update_runtime_variables"): self.parser = ApacheParser( - self.aug, self.config_path, self.vhost_path) + self.aug, self.config_path, self.vhost_path, + configurator=self.config) -def get_apache_configurator( +def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-locals config_path, vhost_path, - config_dir, work_dir, version=(2, 4, 7), conf=None): + config_dir, work_dir, version=(2, 4, 7), + conf=None, + os_info="generic", + conf_vhost_path=None): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -86,8 +99,8 @@ def get_apache_configurator( backups = os.path.join(work_dir, "backups") mock_le_config = mock.MagicMock( apache_server_root=config_path, - apache_vhost_root=vhost_path, - apache_le_vhost_ext=constants.os_constant("le_vhost_ext"), + apache_vhost_root=conf_vhost_path, + apache_le_vhost_ext="-le-ssl.conf", apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, @@ -95,22 +108,37 @@ def get_apache_configurator( in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) - with mock.patch("certbot_apache.configurator.util.run_script"): - with mock.patch("certbot_apache.configurator.util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - with mock.patch("certbot_apache.parser.ApacheParser." - "update_runtime_variables"): - config = configurator.ApacheConfigurator( - config=mock_le_config, - name="apache", - version=version) - # This allows testing scripts to set it a bit more quickly - if conf is not None: - config.conf = conf # pragma: no cover + orig_os_constant = configurator.ApacheConfigurator(mock_le_config, + name="apache", + version=version).constant - config.prepare() + def mock_os_constant(key, vhost_path=vhost_path): + """Mock default vhost path""" + if key == "vhost_root": + return vhost_path + else: + return orig_os_constant(key) + with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons: + mock_cons.side_effect = mock_os_constant + with mock.patch("certbot_apache.configurator.util.run_script"): + with mock.patch("certbot_apache.configurator.util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + with mock.patch("certbot_apache.parser.ApacheParser." + "update_runtime_variables"): + try: + config_class = entrypoint.OVERRIDE_CLASSES[os_info] + except KeyError: + config_class = configurator.ApacheConfigurator + config = config_class(config=mock_le_config, name="apache", + version=version) + # This allows testing scripts to set it a bit more + # quickly + if conf is not None: + config.conf = conf # pragma: no cover + + config.prepare() return config diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b276f49f8..8dc283f2d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -63,7 +63,7 @@ setup( }, entry_points={ 'certbot.plugins': [ - 'apache = certbot_apache.configurator:ApacheConfigurator', + 'apache = certbot_apache.entrypoint:ENTRYPOINT', ], }, test_suite='certbot_apache', diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 2e9e68daf..1d2cfdeca 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -9,8 +9,7 @@ import zope.interface from certbot import configuration from certbot import errors as le_errors from certbot import util as certbot_util -from certbot_apache import configurator -from certbot_apache import constants +from certbot_apache import entrypoint from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces from certbot_compatibility_test import util @@ -56,13 +55,14 @@ class Proxy(configurators_common.Proxy): def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - for k in constants.CLI_DEFAULTS_DEBIAN.keys(): - setattr(self.le_config, "apache_" + k, constants.os_constant(k)) + for k in entrypoint.ENTRYPOINT.OS_DEFAULTS.keys(): + setattr(self.le_config, "apache_" + k, + entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) # An alias self.le_config.apache_handle_modules = self.le_config.apache_handle_mods - self._configurator = configurator.ApacheConfigurator( + self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), name="apache") self._configurator.prepare() diff --git a/certbot/util.py b/certbot/util.py index 30de0c157..b7e60a225 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -342,9 +342,9 @@ def get_os_info_ua(filepath="/etc/os-release"): """ if os.path.isfile(filepath): - os_ua = _get_systemd_os_release_var("PRETTY_NAME", filepath=filepath) + os_ua = get_var_from_file("PRETTY_NAME", filepath=filepath) if not os_ua: - os_ua = _get_systemd_os_release_var("NAME", filepath=filepath) + os_ua = get_var_from_file("NAME", filepath=filepath) if os_ua: return os_ua @@ -361,8 +361,8 @@ def get_systemd_os_info(filepath="/etc/os-release"): :rtype: `tuple` of `str` """ - os_name = _get_systemd_os_release_var("ID", filepath=filepath) - os_version = _get_systemd_os_release_var("VERSION_ID", filepath=filepath) + os_name = get_var_from_file("ID", filepath=filepath) + os_version = get_var_from_file("VERSION_ID", filepath=filepath) return (os_name, os_version) @@ -377,10 +377,10 @@ def get_systemd_os_like(filepath="/etc/os-release"): :rtype: `list` of `str` """ - return _get_systemd_os_release_var("ID_LIKE", filepath).split(" ") + return get_var_from_file("ID_LIKE", filepath).split(" ") -def _get_systemd_os_release_var(varname, filepath="/etc/os-release"): +def get_var_from_file(varname, filepath="/etc/os-release"): """ Get single value from systemd /etc/os-release @@ -405,7 +405,7 @@ def _get_systemd_os_release_var(varname, filepath="/etc/os-release"): def _normalize_string(orig): """ - Helper function for _get_systemd_os_release_var() to remove quotes + Helper function for get_var_from_file() to remove quotes and whitespaces """ return orig.replace('"', '').replace("'", "").strip() From bb70962bb8f7f4bedb5e31c47fad63f30a9eb952 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Dec 2017 14:44:22 -0800 Subject: [PATCH 233/631] Stop using new mock functionality in tests (#5295) * Remove assert_called_once from dns-route53 * Remove assert_called_once from main_test.py * Remove assert_called() usage in dns-digitalocean * Remove assert_called() usage in dns-route53 * Downgrade mock version in certbot-auto --- .../certbot_dns_digitalocean/dns_digitalocean_test.py | 2 +- .../certbot_dns_route53/dns_route53_test.py | 7 ++++--- certbot/tests/main_test.py | 10 +++++----- letsencrypt-auto-source/letsencrypt-auto | 7 ++++--- .../pieces/dependency-requirements.txt | 7 ++++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py index 0fdacf4ad..3b8edce64 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -131,7 +131,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) - correct_record_mock.destroy.assert_called() + self.assertTrue(correct_record_mock.destroy.called) self.assertFalse(first_record_mock.destroy.call_args_list) self.assertFalse(last_record_mock.destroy.call_args_list) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index ff07b6ccd..d5f1b2816 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -31,7 +31,7 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest self.auth._change_txt_record.assert_called_once_with("UPSERT", '_acme-challenge.' + DOMAIN, mock.ANY) - self.auth._wait_for_change.assert_called_once() + self.assertEqual(self.auth._wait_for_change.call_count, 1) def test_perform_no_credentials_error(self): self.auth._change_txt_record = mock.MagicMock(side_effect=NoCredentialsError) @@ -183,7 +183,8 @@ class ClientTest(unittest.TestCase): self.client._change_txt_record("FOO", DOMAIN, "foo") - self.client.r53.change_resource_record_sets.assert_called_once() + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( @@ -192,7 +193,7 @@ class ClientTest(unittest.TestCase): self.client._wait_for_change(1) - self.client.r53.get_change.assert_called() + self.assertTrue(self.client.r53.get_change.called) if __name__ == "__main__": diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..1f690df26 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -356,7 +356,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_for_cert_name.return_value = "/some/reasonable/path" mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -375,7 +375,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_to_lineage.return_value = "example.com" mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -396,7 +396,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_full_archive_dir.return_value = "" mock_match_and_check_overlaps.return_value = "" self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -415,7 +415,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_to_lineage.return_value = config.certname mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.cert_manager.match_and_check_overlaps') @@ -442,7 +442,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): util_mock = mock_get_utility() util_mock.menu.return_value = (display_util.OK, 0) self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 215b684cf..21e47feb8 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1062,9 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 4b3da685c..dec7ae7d0 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -184,6 +184,7 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 From 4db7195e7740fc76c1bf27d8d875ef6bdd70eb9a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Dec 2017 17:09:01 -0800 Subject: [PATCH 234/631] Fix coveralls (#5298) --- tox.cover.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tox.cover.sh b/tox.cover.sh index 3f0a5f72e..2b5a3cf19 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -16,7 +16,7 @@ fi cover () { if [ "$1" = "certbot" ]; then - min=97 + min=98 elif [ "$1" = "acme" ]; then min=100 elif [ "$1" = "certbot_apache" ]; then @@ -24,23 +24,23 @@ cover () { elif [ "$1" = "certbot_dns_cloudflare" ]; then min=98 elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 elif [ "$1" = "certbot_dns_dnsimple" ]; then min=98 elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_dns_luadns" ]; then min=98 elif [ "$1" = "certbot_dns_nsone" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_rfc2136" ]; then min=99 elif [ "$1" = "certbot_dns_route53" ]; then - min=91 + min=92 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then @@ -50,10 +50,12 @@ cover () { exit 1 fi - pytest --cov "$1" --cov-report term-missing \ - --cov-fail-under "$min" --numprocesses auto --pyargs "$1" + pkg_dir=$(echo "$1" | tr _ -) + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } +rm -f .coverage # --cov-append is on, make sure stats are correct for pkg in $pkgs do cover $pkg From 8c4f016b2d4524387ce2ddddf0284118eae455b7 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 22 Nov 2017 13:00:29 -0800 Subject: [PATCH 235/631] In ACMEv2, challenges have "url" instead of "uri". To handle this smoothly, Challenge's uri field becomes private (_uri), and is joined by _url. Serialization and deserialization will preserve whichever one was set. The uri name is taken over by an @property that returns whichever of the two is set. I chose not to enforce that they shouldn't both be present because it would just add unnecessary code and brittleness with no stability benefit. * Make url a virtual field. * Add @property annotation. --- acme/acme/client_test.py | 2 +- acme/acme/messages.py | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4bd762865..3aac9c874 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -181,7 +181,7 @@ class ClientTest(unittest.TestCase): # TODO: split here and separate test self.assertRaises(errors.UnexpectedUpdate, self.client.answer_challenge, - self.challr.body.update(uri='foo'), chall_response) + self.challr.body.update(_uri='foo'), chall_response) def test_answer_challenge_missing_next(self): self.assertRaises( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4b4fa5003..4dee96c58 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -325,13 +325,26 @@ class ChallengeBody(ResourceBody): """ __slots__ = ('chall',) - uri = jose.Field('uri') + # ACMEv1 has a "uri" field in challenges. ACMEv2 has a "url" field. This + # challenge object supports either one. In Client.answer_challenge, + # whichever one is set will be used. + _uri = jose.Field('uri', omitempty=True, default=None) + _url = jose.Field('url', omitempty=True, default=None) status = jose.Field('status', decoder=Status.from_json, omitempty=True, default=STATUS_PENDING) validated = fields.RFC3339Field('validated', omitempty=True) error = jose.Field('error', decoder=Error.from_json, omitempty=True, default=None) + def __init__(self, **kwargs): + new_kwargs = {} + for k, v in kwargs.items(): + if k in ('uri', 'url',): + k = '_' + k + new_kwargs[k] = v + # pylint: disable=star-args + super(ChallengeBody, self).__init__(**new_kwargs) + def to_partial_json(self): jobj = super(ChallengeBody, self).to_partial_json() jobj.update(self.chall.to_partial_json()) @@ -343,6 +356,11 @@ class ChallengeBody(ResourceBody): jobj_fields['chall'] = challenges.Challenge.from_json(jobj) return jobj_fields + @property + def uri(self): + """The URL of this challenge.""" + return self._url or self._uri + def __getattr__(self, name): return getattr(self.chall, name) @@ -358,10 +376,10 @@ class ChallengeResource(Resource): authzr_uri = jose.Field('authzr_uri') @property - def uri(self): # pylint: disable=missing-docstring,no-self-argument - # bug? 'method already defined line None' - # pylint: disable=function-redefined - return self.body.uri # pylint: disable=no-member + def uri(self): + """The URL of the challenge body.""" + # pylint: disable=function-redefined,no-member + return self.body.uri class Authorization(ResourceBody): From 62c1112d10927026501ff7c1d6a830d5e4fa9fee Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Dec 2017 20:50:26 -0800 Subject: [PATCH 236/631] Keep the same behavior with the uri attribute --- acme/acme/client_test.py | 2 +- acme/acme/messages.py | 25 +++++++++++++++++-------- acme/acme/messages_test.py | 3 +++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 3aac9c874..4bd762865 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -181,7 +181,7 @@ class ClientTest(unittest.TestCase): # TODO: split here and separate test self.assertRaises(errors.UnexpectedUpdate, self.client.answer_challenge, - self.challr.body.update(_uri='foo'), chall_response) + self.challr.body.update(uri='foo'), chall_response) def test_answer_challenge_missing_next(self): self.assertRaises( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4dee96c58..2ac29941e 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -326,8 +326,9 @@ class ChallengeBody(ResourceBody): """ __slots__ = ('chall',) # ACMEv1 has a "uri" field in challenges. ACMEv2 has a "url" field. This - # challenge object supports either one. In Client.answer_challenge, - # whichever one is set will be used. + # challenge object supports either one, but should be accessed through the + # name "uri". In Client.answer_challenge, whichever one is set will be + # used. _uri = jose.Field('uri', omitempty=True, default=None) _url = jose.Field('url', omitempty=True, default=None) status = jose.Field('status', decoder=Status.from_json, @@ -337,13 +338,12 @@ class ChallengeBody(ResourceBody): omitempty=True, default=None) def __init__(self, **kwargs): - new_kwargs = {} - for k, v in kwargs.items(): - if k in ('uri', 'url',): - k = '_' + k - new_kwargs[k] = v + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) # pylint: disable=star-args - super(ChallengeBody, self).__init__(**new_kwargs) + super(ChallengeBody, self).__init__(**kwargs) + + def encode(self, name): + return super(ChallengeBody, self).encode(self._internal_name(name)) def to_partial_json(self): jobj = super(ChallengeBody, self).to_partial_json() @@ -364,6 +364,15 @@ class ChallengeBody(ResourceBody): def __getattr__(self, name): return getattr(self.chall, name) + def __iter__(self): + # When iterating over fields, use the external name 'uri' instead of + # the internal '_uri'. + for name in super(ChallengeBody, self).__iter__(): + yield name[1:] if name == '_uri' else name + + def _internal_name(self, name): + return '_' + name if name == 'uri' else name + class ChallengeResource(Resource): """Challenge Resource. diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 631f0ce4d..c9e5c2cf1 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -283,6 +283,9 @@ class ChallengeBodyTest(unittest.TestCase): 'detail': 'Unable to communicate with DNS server', } + def test_encode(self): + self.assertEqual(self.challb.encode('uri'), self.challb.uri) + def test_to_partial_json(self): self.assertEqual(self.jobj_to, self.challb.to_partial_json()) From c9949411cdc5a058d8114a430d98b45c80384650 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Dec 2017 20:04:08 -0800 Subject: [PATCH 237/631] Nginx reversion (#5299) The reason for this PR is many bug fixes in the nginx plugin for changes we haven't released yet are included in #5220 which may not make our next release. If it doesn't, we will (mostly) revert the nginx plugin back to its previous state to avoid releasing these bugs and will revert this PR after the release. * Revert "Nginx IPv6 support (#5178)" This reverts commit 68e37b03c821560e7f316d260f8da97ef3e2087c. * Revert "Fix bug that stopped nginx from finding new server block for redirect (#5198)" This reverts commit e2ab940ac03ffe4f2cf7c478a1597c0b52f14bc4. * Revert "Nginx creates a vhost block if no matching block is found (#5153)" This reverts commit 95a7d4585619d612ff28ac24dac4faefaee59e72. --- certbot-nginx/certbot_nginx/configurator.py | 122 ++----------- certbot-nginx/certbot_nginx/nginxparser.py | 20 +-- certbot-nginx/certbot_nginx/obj.py | 55 ++---- certbot-nginx/certbot_nginx/parser.py | 33 +--- .../certbot_nginx/tests/configurator_test.py | 163 +----------------- .../certbot_nginx/tests/parser_test.py | 56 ++---- .../testdata/etc_nginx/sites-enabled/default | 1 - .../testdata/etc_nginx/sites-enabled/ipv6.com | 5 - .../etc_nginx/sites-enabled/ipv6ssl.com | 5 - .../certbot_nginx/tests/tls_sni_01_test.py | 10 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 32 ++-- certbot/plugins/common.py | 2 +- 12 files changed, 68 insertions(+), 436 deletions(-) delete mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com delete mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 98990664f..fe27dbc4b 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -117,9 +117,6 @@ class NginxConfigurator(common.Installer): # Files to save self.save_notes = "" - # For creating new vhosts if no names match - self.new_vhost = None - # Add number of outstanding challenges self._chall_out = 0 @@ -194,11 +191,9 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, raise_if_no_match=False) - if vhost is None: - vhost = self._vhost_from_duplicated_default(domain) - cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], - ['\n ', 'ssl_certificate_key', ' ', key_path]] + vhost = self.choose_vhost(domain) + cert_directives = [['\n', 'ssl_certificate', ' ', fullchain_path], + ['\n', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) @@ -214,7 +209,7 @@ class NginxConfigurator(common.Installer): ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, raise_if_no_match=True): + def choose_vhost(self, target_name): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -228,8 +223,6 @@ class NginxConfigurator(common.Installer): hostname. Currently we just ignore this. :param str target_name: domain name - :param bool raise_if_no_match: True iff not finding a match is an error; - otherwise, return None :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -240,16 +233,13 @@ class NginxConfigurator(common.Installer): matches = self._get_ranked_matches(target_name) vhost = self._select_best_name_match(matches) if not vhost: - if raise_if_no_match: - # No matches. Raise a misconfiguration error. - raise errors.MisconfigurationError( - ("Cannot find a VirtualHost matching domain %s. " - "In order for Certbot to correctly perform the challenge " - "please add a corresponding server_name directive to your " - "nginx configuration: " - "https://nginx.org/en/docs/http/server_names.html") % (target_name)) - else: - return None + # No matches. Raise a misconfiguration error. + raise errors.MisconfigurationError( + ("Cannot find a VirtualHost matching domain %s. " + "In order for Certbot to correctly perform the challenge " + "please add a corresponding server_name directive to your " + "nginx configuration: " + "https://nginx.org/en/docs/http/server_names.html") % (target_name)) else: # Note: if we are enhancing with ocsp, vhost should already be ssl. if not vhost.ssl: @@ -257,65 +247,6 @@ class NginxConfigurator(common.Installer): return vhost - - def ipv6_info(self, port): - """Returns tuple of booleans (ipv6_active, ipv6only_present) - ipv6_active is true if any server block listens ipv6 address in any port - - ipv6only_present is true if ipv6only=on option exists in any server - block ipv6 listen directive for the specified port. - - :param str port: Port to check ipv6only=on directive for - - :returns: Tuple containing information if IPv6 is enabled in the global - configuration, and existence of ipv6only directive for specified port - :rtype: tuple of type (bool, bool) - """ - vhosts = self.parser.get_vhosts() - ipv6_active = False - ipv6only_present = False - for vh in vhosts: - for addr in vh.addrs: - if addr.ipv6: - ipv6_active = True - if addr.ipv6only and addr.get_port() == port: - ipv6only_present = True - return (ipv6_active, ipv6only_present) - - def _vhost_from_duplicated_default(self, domain): - if self.new_vhost is None: - default_vhost = self._get_default_vhost() - self.new_vhost = self.parser.create_new_vhost_from_default(default_vhost) - if not self.new_vhost.ssl: - self._make_server_ssl(self.new_vhost) - self.new_vhost.names = set() - - self.new_vhost.names.add(domain) - name_block = [['\n ', 'server_name']] - for name in self.new_vhost.names: - name_block[0].append(' ') - name_block[0].append(name) - self.parser.add_server_directives(self.new_vhost, name_block, replace=True) - return self.new_vhost - - def _get_default_vhost(self): - vhost_list = self.parser.get_vhosts() - # if one has default_server set, return that one - default_vhosts = [] - for vhost in vhost_list: - for addr in vhost.addrs: - if addr.default: - default_vhosts.append(vhost) - break - - if len(default_vhosts) == 1: - return default_vhosts[0] - - # TODO: present a list of vhosts for user to choose from - - raise errors.MisconfigurationError("Could not automatically find a matching server" - " block. Set the `server_name` directive to use the Nginx installer.") - def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. The ranking gives preference to SSL vhosts. @@ -474,12 +405,9 @@ class NginxConfigurator(common.Installer): all_names.add(host) elif not common.private_ips_regex.match(host): # If it isn't a private IP, do a reverse DNS lookup + # TODO: IPv6 support try: - if addr.ipv6: - host = addr.get_ipv6_exploded() - socket.inet_pton(socket.AF_INET6, host) - else: - socket.inet_pton(socket.AF_INET, host) + socket.inet_aton(host) all_names.add(socket.gethostbyaddr(host)[0]) except (socket.error, socket.herror, socket.timeout): continue @@ -515,38 +443,16 @@ class NginxConfigurator(common.Installer): :type vhost: :class:`~certbot_nginx.obj.VirtualHost` """ - ipv6info = self.ipv6_info(self.config.tls_sni_01_port) - ipv6_block = [''] - ipv4_block = [''] - # If the vhost was implicitly listening on the default Nginx port, # have it continue to do so. if len(vhost.addrs) == 0: listen_block = [['\n ', 'listen', ' ', self.DEFAULT_LISTEN_PORT]] self.parser.add_server_directives(vhost, listen_block, replace=False) - if vhost.ipv6_enabled(): - ipv6_block = ['\n ', - 'listen', - ' ', - '[::]:{0} ssl'.format(self.config.tls_sni_01_port)] - if not ipv6info[1]: - # ipv6only=on is absent in global config - ipv6_block.append(' ') - ipv6_block.append('ipv6only=on') - - if vhost.ipv4_enabled(): - ipv4_block = ['\n ', - 'listen', - ' ', - '{0} ssl'.format(self.config.tls_sni_01_port)] - - snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = ([ - ipv6_block, - ipv4_block, + ['\n ', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)], ['\n ', 'ssl_certificate', ' ', snakeoil_cert], ['\n ', 'ssl_certificate_key', ' ', snakeoil_key], ['\n ', 'include', ' ', self.mod_ssl_conf], diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 14481e298..20aeeb554 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -7,7 +7,6 @@ from pyparsing import ( Literal, White, Forward, Group, Optional, OneOrMore, QuotedString, Regex, ZeroOrMore, Combine) from pyparsing import stringEnd from pyparsing import restOfLine -import six logger = logging.getLogger(__name__) @@ -72,7 +71,7 @@ class RawNginxDumper(object): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks for b0 in blocks: - if isinstance(b0, six.string_types): + if isinstance(b0, str): yield b0 continue item = copy.deepcopy(b0) @@ -89,7 +88,7 @@ class RawNginxDumper(object): yield '}' else: # not a block - list of strings semicolon = ";" - if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment + if isinstance(item[0], str) and item[0].strip() == '#': # comment semicolon = "" yield "".join(item) + semicolon @@ -146,7 +145,7 @@ def dump(blocks, _file): return _file.write(dumps(blocks)) -spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' +spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -190,15 +189,13 @@ class UnspacedList(list): item, spaced_item = self._coerce(x) slicepos = self._spaced_position(i) if i < len(self) else len(self.spaced) self.spaced.insert(slicepos, spaced_item) - if not spacey(item): - list.insert(self, i, item) + list.insert(self, i, item) self.dirty = True def append(self, x): item, spaced_item = self._coerce(x) self.spaced.append(spaced_item) - if not spacey(item): - list.append(self, item) + list.append(self, item) self.dirty = True def extend(self, x): @@ -229,8 +226,7 @@ class UnspacedList(list): raise NotImplementedError("Slice operations on UnspacedLists not yet implemented") item, spaced_item = self._coerce(value) self.spaced.__setitem__(self._spaced_position(i), spaced_item) - if not spacey(item): - list.__setitem__(self, i, item) + list.__setitem__(self, i, item) self.dirty = True def __delitem__(self, i): @@ -239,8 +235,8 @@ class UnspacedList(list): self.dirty = True def __deepcopy__(self, memo): - new_spaced = copy.deepcopy(self.spaced, memo=memo) - l = UnspacedList(new_spaced) + l = UnspacedList(self[:]) + l.spaced = copy.deepcopy(self.spaced, memo=memo) l.dirty = self.dirty return l diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 5816c5571..849cefe1f 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -34,13 +34,10 @@ class Addr(common.Addr): UNSPECIFIED_IPV4_ADDRESSES = ('', '*', '0.0.0.0') CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] - def __init__(self, host, port, ssl, default, ipv6, ipv6only): - # pylint: disable=too-many-arguments + def __init__(self, host, port, ssl, default): super(Addr, self).__init__((host, port)) self.ssl = ssl self.default = default - self.ipv6 = ipv6 - self.ipv6only = ipv6only self.unspecified_address = host in self.UNSPECIFIED_IPV4_ADDRESSES @classmethod @@ -49,8 +46,6 @@ class Addr(common.Addr): parts = str_addr.split(' ') ssl = False default = False - ipv6 = False - ipv6only = False host = '' port = '' @@ -61,25 +56,15 @@ class Addr(common.Addr): if addr.startswith('unix:'): return None - # IPv6 check - ipv6_match = re.match(r'\[.*\]', addr) - if ipv6_match: - ipv6 = True - # IPv6 handling - host = ipv6_match.group() - # The rest of the addr string will be the port, if any - port = addr[ipv6_match.end()+1:] + tup = addr.partition(':') + if re.match(r'^\d+$', tup[0]): + # This is a bare port, not a hostname. E.g. listen 80 + host = '' + port = tup[0] else: - # IPv4 handling - tup = addr.partition(':') - if re.match(r'^\d+$', tup[0]): - # This is a bare port, not a hostname. E.g. listen 80 - host = '' - port = tup[0] - else: - # This is a host-port tuple. E.g. listen 127.0.0.1:* - host = tup[0] - port = tup[2] + # This is a host-port tuple. E.g. listen 127.0.0.1:* + host = tup[0] + port = tup[2] # The rest of the parts are options; we only care about ssl and default while len(parts) > 0: @@ -88,10 +73,8 @@ class Addr(common.Addr): ssl = True elif nextpart == 'default_server': default = True - elif nextpart == "ipv6only=on": - ipv6only = True - return cls(host, port, ssl, default, ipv6, ipv6only) + return cls(host, port, ssl, default) def to_string(self, include_default=True): """Return string representation of Addr""" @@ -131,6 +114,8 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) + # Nginx plugin currently doesn't support IPv6 but this will + # future-proof it return super(Addr, self).__eq__(other) def __eq__(self, other): @@ -210,24 +195,10 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return True return False - def ipv6_enabled(self): - """Return true if one or more of the listen directives in vhost supports - IPv6""" - for a in self.addrs: - if a.ipv6: - return True - - def ipv4_enabled(self): - """Return true if one or more of the listen directives in vhost are IPv4 - only""" - for a in self.addrs: - if not a.ipv6: - return True - def _find_directive(directives, directive_name): """Find a directive of type directive_name in directives """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: + if not directives or isinstance(directives, str) or len(directives) == 0: return None if directives[0] == directive_name: diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 3eb6264aa..158cb9929 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -6,8 +6,6 @@ import os import pyparsing import re -import six - from certbot import errors from certbot_nginx import obj @@ -314,32 +312,6 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def create_new_vhost_from_default(self, vhost_template): - """Duplicate the default vhost in the configuration files. - - :param :class:`~certbot_nginx.obj.VirtualHost` vhost_template: The vhost - whose information we copy - - :returns: A vhost object for the newly created vhost - :rtype: :class:`~certbot_nginx.obj.VirtualHost` - """ - # TODO: https://github.com/certbot/certbot/issues/5185 - # put it in the same file as the template, at the same level - enclosing_block = self.parsed[vhost_template.filep] - for index in vhost_template.path[:-1]: - enclosing_block = enclosing_block[index] - new_location = vhost_template.path[-1] + 1 - raw_in_parsed = copy.deepcopy(enclosing_block[vhost_template.path[-1]]) - enclosing_block.insert(new_location, raw_in_parsed) - new_vhost = copy.deepcopy(vhost_template) - new_vhost.path[-1] = new_location - for addr in new_vhost.addrs: - addr.default = False - for directive in enclosing_block[new_vhost.path[-1]][1]: - if len(directive) > 0 and directive[0] == 'listen' and 'default_server' in directive: - del directive[directive.index('default_server')] - return new_vhost - def _parse_ssl_options(ssl_options): if ssl_options is not None: try: @@ -472,7 +444,7 @@ def _is_include_directive(entry): """ return (isinstance(entry, list) and len(entry) == 2 and entry[0] == 'include' and - isinstance(entry[1], six.string_types)) + isinstance(entry[1], str)) def _is_ssl_on_directive(entry): """Checks if an nginx parsed entry is an 'ssl on' directive. @@ -589,8 +561,7 @@ def _add_directive(block, directive, replace): directive_name = directive[0] def can_append(loc, dir_name): """ Can we append this directive to the block? """ - return loc is None or (isinstance(dir_name, six.string_types) - and dir_name in REPEATABLE_DIRECTIVES) + return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 996bd238b..f4fe16924 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -45,7 +45,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(10, len(self.config.parser.parsed)) + self.assertEqual(8, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) + "globalssl.com", "globalsslsetssl.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'staple-ocsp'], @@ -131,7 +131,6 @@ class NginxConfiguratorTest(util.NginxTest): server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) foo_conf = set(['*.www.foo.com', '*.www.example.com']) - ipv6_conf = set(['ipv6.com']) results = {'localhost': localhost_conf, 'alias': server_conf, @@ -140,8 +139,7 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': example_conf, 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, - 'www.bar.co.uk': localhost_conf, - 'ipv6.com': ipv6_conf} + 'www.bar.co.uk': localhost_conf} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -150,8 +148,7 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': "etc_nginx/sites-enabled/example.com", 'test.www.example.com': "etc_nginx/foo.conf", 'abc.www.foo.com': "etc_nginx/foo.conf", - 'www.bar.co.uk': "etc_nginx/nginx.conf", - 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + 'www.bar.co.uk': "etc_nginx/nginx.conf"} bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] @@ -162,24 +159,11 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(results[name], vhost.names) self.assertEqual(conf_path[name], path) - # IPv6 specific checks - if name == "ipv6.com": - self.assertTrue(vhost.ipv6_enabled()) - # Make sure that we have SSL enabled also for IPv6 addr - self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) for name in bad_results: self.assertRaises(errors.MisconfigurationError, self.config.choose_vhost, name) - def test_ipv6only(self): - # ipv6_info: (ipv6_active, ipv6only_present) - self.assertEquals((True, False), self.config.ipv6_info("80")) - # Port 443 has ipv6only=on because of ipv6ssl.com vhost - self.assertEquals((True, True), self.config.ipv6_info("443")) - - def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) @@ -574,145 +558,6 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(util.contains_at_depth( generated_conf, ['ssl_stapling_verify', 'on'], 2)) - def test_deploy_no_match_default_set(self): - default_conf = self.config.parser.abs_path('sites-enabled/default') - foo_conf = self.config.parser.abs_path('foo.conf') - del self.config.parser.parsed[foo_conf][2][1][0][1][0] # remove default_server - self.config.version = (1, 3, 1) - - self.config.deploy_cert( - "www.nomatch.com", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - self.config.save() - - self.config.parser.load() - - parsed_default_conf = util.filter_comments(self.config.parser.parsed[default_conf]) - - self.assertEqual([[['server'], - [['listen', 'myhost', 'default_server'], - ['listen', 'otherhost', 'default_server'], - ['server_name', 'www.example.org'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html', 'index.htm']]]]], - [['server'], - [['listen', 'myhost'], - ['listen', 'otherhost'], - ['server_name', 'www.nomatch.com'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html', 'index.htm']]], - ['listen', '5001', 'ssl'], - ['ssl_certificate', 'example/fullchain.pem'], - ['ssl_certificate_key', 'example/key.pem'], - ['include', self.config.mod_ssl_conf], - ['ssl_dhparam', self.config.ssl_dhparams]]]], - parsed_default_conf) - - self.config.deploy_cert( - "nomatch.com", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - self.config.save() - - self.config.parser.load() - - parsed_default_conf = util.filter_comments(self.config.parser.parsed[default_conf]) - - self.assertTrue(util.contains_at_depth(parsed_default_conf, "nomatch.com", 3)) - - def test_deploy_no_match_default_set_multi_level_path(self): - default_conf = self.config.parser.abs_path('sites-enabled/default') - foo_conf = self.config.parser.abs_path('foo.conf') - del self.config.parser.parsed[default_conf][0][1][0] - del self.config.parser.parsed[default_conf][0][1][0] - self.config.version = (1, 3, 1) - - self.config.deploy_cert( - "www.nomatch.com", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - self.config.save() - - self.config.parser.load() - - parsed_foo_conf = util.filter_comments(self.config.parser.parsed[foo_conf]) - - self.assertEqual([['server'], - [['listen', '*:80', 'ssl'], - ['server_name', 'www.nomatch.com'], - ['root', '/home/ubuntu/sites/foo/'], - [['location', '/status'], [[['types'], [['image/jpeg', 'jpg']]]]], - [['location', '~', 'case_sensitive\\.php$'], [['index', 'index.php'], - ['root', '/var/root']]], - [['location', '~*', 'case_insensitive\\.php$'], []], - [['location', '=', 'exact_match\\.php$'], []], - [['location', '^~', 'ignore_regex\\.php$'], []], - ['ssl_certificate', 'example/fullchain.pem'], - ['ssl_certificate_key', 'example/key.pem']]], - parsed_foo_conf[1][1][1]) - - def test_deploy_no_match_no_default_set(self): - default_conf = self.config.parser.abs_path('sites-enabled/default') - foo_conf = self.config.parser.abs_path('foo.conf') - del self.config.parser.parsed[default_conf][0][1][0] - del self.config.parser.parsed[default_conf][0][1][0] - del self.config.parser.parsed[foo_conf][2][1][0][1][0] - self.config.version = (1, 3, 1) - - self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert, - "www.nomatch.com", "example/cert.pem", "example/key.pem", - "example/chain.pem", "example/fullchain.pem") - - def test_deploy_no_match_fail_multiple_defaults(self): - self.config.version = (1, 3, 1) - self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert, - "www.nomatch.com", "example/cert.pem", "example/key.pem", - "example/chain.pem", "example/fullchain.pem") - - def test_deploy_no_match_add_redirect(self): - default_conf = self.config.parser.abs_path('sites-enabled/default') - foo_conf = self.config.parser.abs_path('foo.conf') - del self.config.parser.parsed[foo_conf][2][1][0][1][0] # remove default_server - self.config.version = (1, 3, 1) - - self.config.deploy_cert( - "www.nomatch.com", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - self.config.deploy_cert( - "nomatch.com", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - self.config.enhance("www.nomatch.com", "redirect") - - self.config.save() - - self.config.parser.load() - - expected = [ - ['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ] - - generated_conf = self.config.parser.parsed[default_conf] - self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) - - class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index ca5de7ff6..e655bc3e3 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -50,9 +50,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods 'sites-enabled/example.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', - 'sites-enabled/globalssl.com', - 'sites-enabled/ipv6.com', - 'sites-enabled/ipv6ssl.com']]), + 'sites-enabled/globalssl.com']]), set(nparser.parsed.keys())) self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], nparser.parsed[nparser.abs_path('server.conf')]) @@ -76,7 +74,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(7, len( + self.assertEqual(5, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -112,8 +110,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), - [obj.Addr('4.8.2.6', '57', True, False, - False, False)], + [obj.Addr('4.8.2.6', '57', True, False)], True, True, set(['globalssl.com']), [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] @@ -124,42 +121,34 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost1 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('', '8080', False, False, - False, False)], + [obj.Addr('', '8080', False, False)], False, True, set(['localhost', r'~^(www\.)?(example|bar)\.']), [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('somename', '8080', False, False, - False, False), - obj.Addr('', '8000', False, False, - False, False)], + [obj.Addr('somename', '8080', False, False), + obj.Addr('', '8000', False, False)], False, True, set(['somename', 'another.alias', 'alias']), [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', - False, False, False, False), - obj.Addr('127.0.0.1', '', False, False, - False, False)], + False, False), + obj.Addr('127.0.0.1', '', False, False)], False, True, set(['.example.com', 'example.*']), [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), - [obj.Addr('myhost', '', False, True, - False, False), - obj.Addr('otherhost', '', False, True, - False, False)], + [obj.Addr('myhost', '', False, True)], False, True, set(['www.example.org']), [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), - [obj.Addr('*', '80', True, True, - False, False)], + [obj.Addr('*', '80', True, True)], True, True, set(['*.www.foo.com', '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(12, len(vhosts)) + self.assertEqual(10, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] @@ -406,29 +395,6 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ]) self.assertTrue(server['ssl']) - def test_create_new_vhost_from_default(self): - nparser = parser.NginxParser(self.config_path) - - vhosts = nparser.get_vhosts() - default = [x for x in vhosts if 'default' in x.filep][0] - new_vhost = nparser.create_new_vhost_from_default(default) - nparser.filedump(ext='') - - # check properties of new vhost - self.assertFalse(next(iter(new_vhost.addrs)).default) - self.assertNotEqual(new_vhost.path, default.path) - - # check that things are written to file correctly - new_nparser = parser.NginxParser(self.config_path) - new_vhosts = new_nparser.get_vhosts() - new_defaults = [x for x in new_vhosts if 'default' in x.filep] - self.assertEqual(len(new_defaults), 2) - new_vhost_parsed = new_defaults[1] - self.assertFalse(next(iter(new_vhost_parsed.addrs)).default) - self.assertEqual(next(iter(default.names)), next(iter(new_vhost_parsed.names))) - self.assertEqual(len(default.raw), len(new_vhost_parsed.raw)) - self.assertTrue(next(iter(default.addrs)).super_eq(next(iter(new_vhost_parsed.addrs)))) - if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default index 4f67fa7d1..26f37020c 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default @@ -1,6 +1,5 @@ server { listen myhost default_server; - listen otherhost default_server; server_name www.example.org; location / { diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com deleted file mode 100644 index 7a7744b92..000000000 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com +++ /dev/null @@ -1,5 +0,0 @@ -server { - listen 80; - listen [::]:80; - server_name ipv6.com; -} diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com deleted file mode 100644 index d8f7eff12..000000000 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com +++ /dev/null @@ -1,5 +0,0 @@ -server { - listen 443 ssl; - listen [::]:443 ssl ipv6only=on; - server_name ipv6ssl.com; -} diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 32a5ed7d2..85db584b3 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -66,7 +66,7 @@ class TlsSniPerformTest(util.NginxTest): self.sni.add_chall(self.achalls[1]) mock_choose.return_value = None result = self.sni.perform() - self.assertFalse(result is None) + self.assertTrue(result is None) def test_perform0(self): responses = self.sni.perform() @@ -125,10 +125,10 @@ class TlsSniPerformTest(util.NginxTest): self.sni.add_chall(self.achalls[0]) self.sni.add_chall(self.achalls[2]) - v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False, False, False), - obj.Addr("127.0.0.1", "", False, False, False, False)] - v_addr2 = [obj.Addr("myhost", "", False, True, False, False)] - v_addr2_print = [obj.Addr("myhost", "", False, False, False, False)] + v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False), + obj.Addr("127.0.0.1", "", False, False)] + v_addr2 = [obj.Addr("myhost", "", False, True)] + v_addr2_print = [obj.Addr("myhost", "", False, False)] ll_addr = [v_addr1, v_addr2] self.sni._mod_config(ll_addr) # pylint: disable=protected-access diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 7f597ac4a..d6faa12be 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,32 +51,19 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) - ipv6, ipv6only = self.configurator.ipv6_info( - self.configurator.config.tls_sni_01_port) - for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, raise_if_no_match=False) + vhost = self.configurator.choose_vhost(achall.domain) + if vhost is None: + logger.error( + "No nginx vhost exists with server_name matching: %s. " + "Please specify server_names in the Nginx config.", + achall.domain) + return None - if vhost is not None and vhost.addrs: + if vhost.addrs: addresses.append(list(vhost.addrs)) else: - if ipv6: - # If IPv6 is active in Nginx configuration - ipv6_addr = "[::]:{0} ssl".format( - self.configurator.config.tls_sni_01_port) - if not ipv6only: - # If ipv6only=on is not already present in the config - ipv6_addr = ipv6_addr + " ipv6only=on" - addresses.append([obj.Addr.fromstring(default_addr), - obj.Addr.fromstring(ipv6_addr)]) - logger.info(("Using default addresses %s and %s for " + - "TLSSNI01 authentication."), - default_addr, - ipv6_addr) - else: - addresses.append([obj.Addr.fromstring(default_addr)]) - logger.info("Using default address %s for TLSSNI01 authentication.", - default_addr) + addresses.append([obj.Addr.fromstring(default_addr)]) # Create challenge certs responses = [self._setup_challenge_cert(x) for x in self.achalls] @@ -130,6 +117,7 @@ class NginxTlsSni01(common.TLSSNI01): raise errors.MisconfigurationError( 'Certbot could not find an HTTP block to include ' 'TLS-SNI-01 challenges in %s.' % root) + config = [self._make_server_block(pair[0], pair[1]) for pair in six.moves.zip(self.achalls, ll_addrs)] config = nginxparser.UnspacedList(config) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 420d15679..f605eb751 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -251,7 +251,7 @@ class Addr(object): """Normalized representation of addr/port tuple """ if self.ipv6: - return (self.get_ipv6_exploded(), self.tup[1]) + return (self._normalize_ipv6(self.tup[0]), self.tup[1]) return self.tup def __eq__(self, other): From f1554324da4c68bfe8ba035647e2664edeb561aa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Dec 2017 14:46:55 -0800 Subject: [PATCH 238/631] Release 0.20.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 33 ++++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 6 ++-- letsencrypt-auto | 33 ++++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 14 ++++---- letsencrypt-auto-source/letsencrypt-auto | 26 +++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | 4 +-- .../pieces/certbot-requirements.txt | 24 +++++++------- 22 files changed, 86 insertions(+), 84 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index c28e0c152..c5a85c96b 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 8dc283f2d..838d2fd04 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index 25f2ce889..444bee1b9 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.19.0" +LE_AUTO_VERSION="0.20.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1062,9 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1077,18 +1078,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 166c383b3..d8965f2e4 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 6392e483c..448df1ab8 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 304acf110..5ad92f961 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 489321435..dbb4e9c68 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 67d68ee16..e24a9116c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 88e02304e..0c0bbdeb9 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b40899e80..49c4f8ad9 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 1b72168e8..5c5f10e90 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e9dc2b31d..6b626ad5e 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 79b523aed..aab3bd0ee 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2a14e8ab1..8223226a5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index f3919413d..94beef24b 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.20.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 231a0f5f5..0f7b8f5fd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.20.0.dev0' +__version__ = '0.20.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 6b43fd0a2..abaa95b9b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,7 +107,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.19.0 (certbot; + "". (default: CertbotACMEClient/0.20.0 (certbot; Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags encoded in the user agent are: --duplicate, --force- @@ -121,7 +121,7 @@ optional arguments: (Example: Foo-Wrapper/1.0) (default: None) automation: - Arguments for automating execution & other tweaks + Flags for automating execution & other tweaks --keep-until-expiring, --keep, --reinstall If the requested certificate matches an existing @@ -228,7 +228,7 @@ testing: False) paths: - Arguments changing execution paths & servers + Flags for changing execution paths & servers --cert-path CERT_PATH Path to where certificate is saved (with auth --csr), diff --git a/letsencrypt-auto b/letsencrypt-auto index 25f2ce889..444bee1b9 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.19.0" +LE_AUTO_VERSION="0.20.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1062,9 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1077,18 +1078,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 834358464..eeab78cd6 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJZ1TJAAAoJEE0XyZXNl3XyWjAIAKxR5v0qbSyOEwM1LrSoLqud -V3KkyEUlMq7IPHxoPKXbqUrIi4eZuhpJz+84LtVJe4ZQ6HYP9lPogX+PtmWW7dyO -YerxA2rUVGB9rFZofZYwTuJyvO5Nc0aDyp1FHHPg/5khWWhhhxKpWqqG3zT01+Vf -W8Lvvn7vr7sjTvxBdqHQ3z3hlUY62P2IKui9C5un5ozlSQpDrWh3Thi9r6CxbASL -/r1PQ6EfnNdPAizVrJWe5iUd0Nzj7VMkFwZ02A3OlOUvrHGVb1H6oj0S1lZ8LEpj -awOTys8PVBQ3vW2qbAL3Zk7Lr+CGfVfmoWC9TQEKiSN1woYFrFD39S527vB1onc= -=Meks +iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt +XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf +p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX +OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4 +X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI +/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE= +=Is56 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 21e47feb8..444bee1b9 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0.dev0" +LE_AUTO_VERSION="0.20.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1078,18 +1078,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 708bbbee6..e276aae53 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,2 +1,2 @@ -è¾לHêÉ­mì³ÊÄ+ˆ²Ä~™¦ES«ëM„4ø»ò¡Ù K“íY”jLãŸÁèÚê7øñöZ½åÕ³ÿ°dŸdÝïI.:†ÓdZMOü|’±K¢Öí°¾âm|göÊ(–$bšljÇÐ…’/ñAâ^Ãéÿ©¶`ra®^ª0˜Ôß÷xÜÐå’²ƒwæÈá9”¦ckâNÃù¬Å‘.[ ?ë” -hð¡/Ì8!÷ü\§º’Å!»ÎöØÿ¯U5ñ£9bÉR£Ÿlb±-•«1‰Âà‰±ü(›p>¹ -û¢%Îu2ÁgnêÍ \ No newline at end of file +HtÃÚPdM-b_ 8Gݵ¥œx\¨cf<9n™$-ä€^5¶¤¡ÌÙð—6¯ò¢¹zéOy¯3üäðo-äÃN~“ֹ麛À²Ñn%… ww''}q;åÌ°: + M§4­Ìàí\¬¬@¿)€°-¶ã:ǺzD•Y›Ááþ‘=ð›ìŸ­*†à'žà Date: Wed, 6 Dec 2017 14:52:16 -0800 Subject: [PATCH 239/631] Bump version to 0.21.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index c5a85c96b..d04b84739 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 838d2fd04..3270f2c79 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index d8965f2e4..1faf30643 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 448df1ab8..428271045 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5ad92f961..4a103193f 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index dbb4e9c68..23098d4b6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index e24a9116c..4ed5a06ca 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0c0bbdeb9..8a0b88aab 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 49c4f8ad9..b00bd1ac3 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 5c5f10e90..b8f50254e 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 6b626ad5e..2a388e487 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index aab3bd0ee..78007afb5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8223226a5..7d1eb0bc9 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 94beef24b..2ad7aaf08 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 0f7b8f5fd..cbea701ee 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.20.0' +__version__ = '0.21.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 444bee1b9..8d2e8a6b6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 716f25743ca7df91b0d55ee08058f2271983d9d4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Dec 2017 16:33:55 -0800 Subject: [PATCH 240/631] Update changelog for 0.20.0 --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aaef4af1..92d059b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,38 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.20.0 - 2017-12-06 + +### Added + +* Certbot's ACME library now recognizes URL fields in challenge objects in + preparation for Let's Encrypt's new ACME endpoint. The value is still + accessible in our ACME library through the name "uri". + +### Changed + +* The Apache plugin now parses some distro specific Apache configuration files + on non-Debian systems allowing it to get a clearer picture on the running + Apache configuration. +* Certbot better reports network failures by removing information about + connection retries from the error output. +* An unnecessary question when using Certbot's webroot plugin interactively has + been removed. + +### Fixed + +* Certbot's NGINX plugin no longer sometimes incorrectly reports that it was + unable to deploy a HTTP->HTTPS redirect when requesting Certbot to enable a + redirect for multiple domains. +* An issue running the test shipped with Certbot and some our DNS plugins with + older versions of mock have been resolved. +* On some systems, users reported strangely interleaved output depending on + when stdout and stderr were flushed. This problem was resolved by having + Certbot regularly flush these streams. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/44?closed=1 + ## 0.19.0 - 2017-10-04 ### Added From abed73a8e4877e5166b017d5fe29bb9d9a497cb0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Dec 2017 17:45:20 -0800 Subject: [PATCH 241/631] Revert "Nginx reversion (#5299)" (#5305) This reverts commit c9949411cdc5a058d8114a430d98b45c80384650. --- certbot-nginx/certbot_nginx/configurator.py | 122 +++++++++++-- certbot-nginx/certbot_nginx/nginxparser.py | 20 ++- certbot-nginx/certbot_nginx/obj.py | 55 ++++-- certbot-nginx/certbot_nginx/parser.py | 33 +++- .../certbot_nginx/tests/configurator_test.py | 163 +++++++++++++++++- .../certbot_nginx/tests/parser_test.py | 56 ++++-- .../testdata/etc_nginx/sites-enabled/default | 1 + .../testdata/etc_nginx/sites-enabled/ipv6.com | 5 + .../etc_nginx/sites-enabled/ipv6ssl.com | 5 + .../certbot_nginx/tests/tls_sni_01_test.py | 10 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 34 ++-- certbot/plugins/common.py | 2 +- 12 files changed, 437 insertions(+), 69 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index fe27dbc4b..98990664f 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -117,6 +117,9 @@ class NginxConfigurator(common.Installer): # Files to save self.save_notes = "" + # For creating new vhosts if no names match + self.new_vhost = None + # Add number of outstanding challenges self._chall_out = 0 @@ -191,9 +194,11 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain) - cert_directives = [['\n', 'ssl_certificate', ' ', fullchain_path], - ['\n', 'ssl_certificate_key', ' ', key_path]] + vhost = self.choose_vhost(domain, raise_if_no_match=False) + if vhost is None: + vhost = self._vhost_from_duplicated_default(domain) + cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], + ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) @@ -209,7 +214,7 @@ class NginxConfigurator(common.Installer): ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name): + def choose_vhost(self, target_name, raise_if_no_match=True): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -223,6 +228,8 @@ class NginxConfigurator(common.Installer): hostname. Currently we just ignore this. :param str target_name: domain name + :param bool raise_if_no_match: True iff not finding a match is an error; + otherwise, return None :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -233,13 +240,16 @@ class NginxConfigurator(common.Installer): matches = self._get_ranked_matches(target_name) vhost = self._select_best_name_match(matches) if not vhost: - # No matches. Raise a misconfiguration error. - raise errors.MisconfigurationError( - ("Cannot find a VirtualHost matching domain %s. " - "In order for Certbot to correctly perform the challenge " - "please add a corresponding server_name directive to your " - "nginx configuration: " - "https://nginx.org/en/docs/http/server_names.html") % (target_name)) + if raise_if_no_match: + # No matches. Raise a misconfiguration error. + raise errors.MisconfigurationError( + ("Cannot find a VirtualHost matching domain %s. " + "In order for Certbot to correctly perform the challenge " + "please add a corresponding server_name directive to your " + "nginx configuration: " + "https://nginx.org/en/docs/http/server_names.html") % (target_name)) + else: + return None else: # Note: if we are enhancing with ocsp, vhost should already be ssl. if not vhost.ssl: @@ -247,6 +257,65 @@ class NginxConfigurator(common.Installer): return vhost + + def ipv6_info(self, port): + """Returns tuple of booleans (ipv6_active, ipv6only_present) + ipv6_active is true if any server block listens ipv6 address in any port + + ipv6only_present is true if ipv6only=on option exists in any server + block ipv6 listen directive for the specified port. + + :param str port: Port to check ipv6only=on directive for + + :returns: Tuple containing information if IPv6 is enabled in the global + configuration, and existence of ipv6only directive for specified port + :rtype: tuple of type (bool, bool) + """ + vhosts = self.parser.get_vhosts() + ipv6_active = False + ipv6only_present = False + for vh in vhosts: + for addr in vh.addrs: + if addr.ipv6: + ipv6_active = True + if addr.ipv6only and addr.get_port() == port: + ipv6only_present = True + return (ipv6_active, ipv6only_present) + + def _vhost_from_duplicated_default(self, domain): + if self.new_vhost is None: + default_vhost = self._get_default_vhost() + self.new_vhost = self.parser.create_new_vhost_from_default(default_vhost) + if not self.new_vhost.ssl: + self._make_server_ssl(self.new_vhost) + self.new_vhost.names = set() + + self.new_vhost.names.add(domain) + name_block = [['\n ', 'server_name']] + for name in self.new_vhost.names: + name_block[0].append(' ') + name_block[0].append(name) + self.parser.add_server_directives(self.new_vhost, name_block, replace=True) + return self.new_vhost + + def _get_default_vhost(self): + vhost_list = self.parser.get_vhosts() + # if one has default_server set, return that one + default_vhosts = [] + for vhost in vhost_list: + for addr in vhost.addrs: + if addr.default: + default_vhosts.append(vhost) + break + + if len(default_vhosts) == 1: + return default_vhosts[0] + + # TODO: present a list of vhosts for user to choose from + + raise errors.MisconfigurationError("Could not automatically find a matching server" + " block. Set the `server_name` directive to use the Nginx installer.") + def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. The ranking gives preference to SSL vhosts. @@ -405,9 +474,12 @@ class NginxConfigurator(common.Installer): all_names.add(host) elif not common.private_ips_regex.match(host): # If it isn't a private IP, do a reverse DNS lookup - # TODO: IPv6 support try: - socket.inet_aton(host) + if addr.ipv6: + host = addr.get_ipv6_exploded() + socket.inet_pton(socket.AF_INET6, host) + else: + socket.inet_pton(socket.AF_INET, host) all_names.add(socket.gethostbyaddr(host)[0]) except (socket.error, socket.herror, socket.timeout): continue @@ -443,16 +515,38 @@ class NginxConfigurator(common.Installer): :type vhost: :class:`~certbot_nginx.obj.VirtualHost` """ + ipv6info = self.ipv6_info(self.config.tls_sni_01_port) + ipv6_block = [''] + ipv4_block = [''] + # If the vhost was implicitly listening on the default Nginx port, # have it continue to do so. if len(vhost.addrs) == 0: listen_block = [['\n ', 'listen', ' ', self.DEFAULT_LISTEN_PORT]] self.parser.add_server_directives(vhost, listen_block, replace=False) + if vhost.ipv6_enabled(): + ipv6_block = ['\n ', + 'listen', + ' ', + '[::]:{0} ssl'.format(self.config.tls_sni_01_port)] + if not ipv6info[1]: + # ipv6only=on is absent in global config + ipv6_block.append(' ') + ipv6_block.append('ipv6only=on') + + if vhost.ipv4_enabled(): + ipv4_block = ['\n ', + 'listen', + ' ', + '{0} ssl'.format(self.config.tls_sni_01_port)] + + snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = ([ - ['\n ', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)], + ipv6_block, + ipv4_block, ['\n ', 'ssl_certificate', ' ', snakeoil_cert], ['\n ', 'ssl_certificate_key', ' ', snakeoil_key], ['\n ', 'include', ' ', self.mod_ssl_conf], diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 20aeeb554..14481e298 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -7,6 +7,7 @@ from pyparsing import ( Literal, White, Forward, Group, Optional, OneOrMore, QuotedString, Regex, ZeroOrMore, Combine) from pyparsing import stringEnd from pyparsing import restOfLine +import six logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class RawNginxDumper(object): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks for b0 in blocks: - if isinstance(b0, str): + if isinstance(b0, six.string_types): yield b0 continue item = copy.deepcopy(b0) @@ -88,7 +89,7 @@ class RawNginxDumper(object): yield '}' else: # not a block - list of strings semicolon = ";" - if isinstance(item[0], str) and item[0].strip() == '#': # comment + if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment semicolon = "" yield "".join(item) + semicolon @@ -145,7 +146,7 @@ def dump(blocks, _file): return _file.write(dumps(blocks)) -spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' +spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -189,13 +190,15 @@ class UnspacedList(list): item, spaced_item = self._coerce(x) slicepos = self._spaced_position(i) if i < len(self) else len(self.spaced) self.spaced.insert(slicepos, spaced_item) - list.insert(self, i, item) + if not spacey(item): + list.insert(self, i, item) self.dirty = True def append(self, x): item, spaced_item = self._coerce(x) self.spaced.append(spaced_item) - list.append(self, item) + if not spacey(item): + list.append(self, item) self.dirty = True def extend(self, x): @@ -226,7 +229,8 @@ class UnspacedList(list): raise NotImplementedError("Slice operations on UnspacedLists not yet implemented") item, spaced_item = self._coerce(value) self.spaced.__setitem__(self._spaced_position(i), spaced_item) - list.__setitem__(self, i, item) + if not spacey(item): + list.__setitem__(self, i, item) self.dirty = True def __delitem__(self, i): @@ -235,8 +239,8 @@ class UnspacedList(list): self.dirty = True def __deepcopy__(self, memo): - l = UnspacedList(self[:]) - l.spaced = copy.deepcopy(self.spaced, memo=memo) + new_spaced = copy.deepcopy(self.spaced, memo=memo) + l = UnspacedList(new_spaced) l.dirty = self.dirty return l diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 849cefe1f..5816c5571 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -34,10 +34,13 @@ class Addr(common.Addr): UNSPECIFIED_IPV4_ADDRESSES = ('', '*', '0.0.0.0') CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] - def __init__(self, host, port, ssl, default): + def __init__(self, host, port, ssl, default, ipv6, ipv6only): + # pylint: disable=too-many-arguments super(Addr, self).__init__((host, port)) self.ssl = ssl self.default = default + self.ipv6 = ipv6 + self.ipv6only = ipv6only self.unspecified_address = host in self.UNSPECIFIED_IPV4_ADDRESSES @classmethod @@ -46,6 +49,8 @@ class Addr(common.Addr): parts = str_addr.split(' ') ssl = False default = False + ipv6 = False + ipv6only = False host = '' port = '' @@ -56,15 +61,25 @@ class Addr(common.Addr): if addr.startswith('unix:'): return None - tup = addr.partition(':') - if re.match(r'^\d+$', tup[0]): - # This is a bare port, not a hostname. E.g. listen 80 - host = '' - port = tup[0] + # IPv6 check + ipv6_match = re.match(r'\[.*\]', addr) + if ipv6_match: + ipv6 = True + # IPv6 handling + host = ipv6_match.group() + # The rest of the addr string will be the port, if any + port = addr[ipv6_match.end()+1:] else: - # This is a host-port tuple. E.g. listen 127.0.0.1:* - host = tup[0] - port = tup[2] + # IPv4 handling + tup = addr.partition(':') + if re.match(r'^\d+$', tup[0]): + # This is a bare port, not a hostname. E.g. listen 80 + host = '' + port = tup[0] + else: + # This is a host-port tuple. E.g. listen 127.0.0.1:* + host = tup[0] + port = tup[2] # The rest of the parts are options; we only care about ssl and default while len(parts) > 0: @@ -73,8 +88,10 @@ class Addr(common.Addr): ssl = True elif nextpart == 'default_server': default = True + elif nextpart == "ipv6only=on": + ipv6only = True - return cls(host, port, ssl, default) + return cls(host, port, ssl, default, ipv6, ipv6only) def to_string(self, include_default=True): """Return string representation of Addr""" @@ -114,8 +131,6 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) - # Nginx plugin currently doesn't support IPv6 but this will - # future-proof it return super(Addr, self).__eq__(other) def __eq__(self, other): @@ -195,10 +210,24 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return True return False + def ipv6_enabled(self): + """Return true if one or more of the listen directives in vhost supports + IPv6""" + for a in self.addrs: + if a.ipv6: + return True + + def ipv4_enabled(self): + """Return true if one or more of the listen directives in vhost are IPv4 + only""" + for a in self.addrs: + if not a.ipv6: + return True + def _find_directive(directives, directive_name): """Find a directive of type directive_name in directives """ - if not directives or isinstance(directives, str) or len(directives) == 0: + if not directives or isinstance(directives, six.string_types) or len(directives) == 0: return None if directives[0] == directive_name: diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 158cb9929..3eb6264aa 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -6,6 +6,8 @@ import os import pyparsing import re +import six + from certbot import errors from certbot_nginx import obj @@ -312,6 +314,32 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) + def create_new_vhost_from_default(self, vhost_template): + """Duplicate the default vhost in the configuration files. + + :param :class:`~certbot_nginx.obj.VirtualHost` vhost_template: The vhost + whose information we copy + + :returns: A vhost object for the newly created vhost + :rtype: :class:`~certbot_nginx.obj.VirtualHost` + """ + # TODO: https://github.com/certbot/certbot/issues/5185 + # put it in the same file as the template, at the same level + enclosing_block = self.parsed[vhost_template.filep] + for index in vhost_template.path[:-1]: + enclosing_block = enclosing_block[index] + new_location = vhost_template.path[-1] + 1 + raw_in_parsed = copy.deepcopy(enclosing_block[vhost_template.path[-1]]) + enclosing_block.insert(new_location, raw_in_parsed) + new_vhost = copy.deepcopy(vhost_template) + new_vhost.path[-1] = new_location + for addr in new_vhost.addrs: + addr.default = False + for directive in enclosing_block[new_vhost.path[-1]][1]: + if len(directive) > 0 and directive[0] == 'listen' and 'default_server' in directive: + del directive[directive.index('default_server')] + return new_vhost + def _parse_ssl_options(ssl_options): if ssl_options is not None: try: @@ -444,7 +472,7 @@ def _is_include_directive(entry): """ return (isinstance(entry, list) and len(entry) == 2 and entry[0] == 'include' and - isinstance(entry[1], str)) + isinstance(entry[1], six.string_types)) def _is_ssl_on_directive(entry): """Checks if an nginx parsed entry is an 'ssl on' directive. @@ -561,7 +589,8 @@ def _add_directive(block, directive, replace): directive_name = directive[0] def can_append(loc, dir_name): """ Can we append this directive to the block? """ - return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) + return loc is None or (isinstance(dir_name, six.string_types) + and dir_name in REPEATABLE_DIRECTIVES) err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index f4fe16924..996bd238b 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -45,7 +45,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(8, len(self.config.parser.parsed)) + self.assertEqual(10, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com"])) + "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'staple-ocsp'], @@ -131,6 +131,7 @@ class NginxConfiguratorTest(util.NginxTest): server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) foo_conf = set(['*.www.foo.com', '*.www.example.com']) + ipv6_conf = set(['ipv6.com']) results = {'localhost': localhost_conf, 'alias': server_conf, @@ -139,7 +140,8 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': example_conf, 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, - 'www.bar.co.uk': localhost_conf} + 'www.bar.co.uk': localhost_conf, + 'ipv6.com': ipv6_conf} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -148,7 +150,8 @@ class NginxConfiguratorTest(util.NginxTest): 'www.example.com': "etc_nginx/sites-enabled/example.com", 'test.www.example.com': "etc_nginx/foo.conf", 'abc.www.foo.com': "etc_nginx/foo.conf", - 'www.bar.co.uk': "etc_nginx/nginx.conf"} + 'www.bar.co.uk': "etc_nginx/nginx.conf", + 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] @@ -159,11 +162,24 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(results[name], vhost.names) self.assertEqual(conf_path[name], path) + # IPv6 specific checks + if name == "ipv6.com": + self.assertTrue(vhost.ipv6_enabled()) + # Make sure that we have SSL enabled also for IPv6 addr + self.assertTrue( + any([True for x in vhost.addrs if x.ssl and x.ipv6])) for name in bad_results: self.assertRaises(errors.MisconfigurationError, self.config.choose_vhost, name) + def test_ipv6only(self): + # ipv6_info: (ipv6_active, ipv6only_present) + self.assertEquals((True, False), self.config.ipv6_info("80")) + # Port 443 has ipv6only=on because of ipv6ssl.com vhost + self.assertEquals((True, True), self.config.ipv6_info("443")) + + def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) @@ -558,6 +574,145 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(util.contains_at_depth( generated_conf, ['ssl_stapling_verify', 'on'], 2)) + def test_deploy_no_match_default_set(self): + default_conf = self.config.parser.abs_path('sites-enabled/default') + foo_conf = self.config.parser.abs_path('foo.conf') + del self.config.parser.parsed[foo_conf][2][1][0][1][0] # remove default_server + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "www.nomatch.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.save() + + self.config.parser.load() + + parsed_default_conf = util.filter_comments(self.config.parser.parsed[default_conf]) + + self.assertEqual([[['server'], + [['listen', 'myhost', 'default_server'], + ['listen', 'otherhost', 'default_server'], + ['server_name', 'www.example.org'], + [['location', '/'], + [['root', 'html'], + ['index', 'index.html', 'index.htm']]]]], + [['server'], + [['listen', 'myhost'], + ['listen', 'otherhost'], + ['server_name', 'www.nomatch.com'], + [['location', '/'], + [['root', 'html'], + ['index', 'index.html', 'index.htm']]], + ['listen', '5001', 'ssl'], + ['ssl_certificate', 'example/fullchain.pem'], + ['ssl_certificate_key', 'example/key.pem'], + ['include', self.config.mod_ssl_conf], + ['ssl_dhparam', self.config.ssl_dhparams]]]], + parsed_default_conf) + + self.config.deploy_cert( + "nomatch.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.save() + + self.config.parser.load() + + parsed_default_conf = util.filter_comments(self.config.parser.parsed[default_conf]) + + self.assertTrue(util.contains_at_depth(parsed_default_conf, "nomatch.com", 3)) + + def test_deploy_no_match_default_set_multi_level_path(self): + default_conf = self.config.parser.abs_path('sites-enabled/default') + foo_conf = self.config.parser.abs_path('foo.conf') + del self.config.parser.parsed[default_conf][0][1][0] + del self.config.parser.parsed[default_conf][0][1][0] + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "www.nomatch.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.save() + + self.config.parser.load() + + parsed_foo_conf = util.filter_comments(self.config.parser.parsed[foo_conf]) + + self.assertEqual([['server'], + [['listen', '*:80', 'ssl'], + ['server_name', 'www.nomatch.com'], + ['root', '/home/ubuntu/sites/foo/'], + [['location', '/status'], [[['types'], [['image/jpeg', 'jpg']]]]], + [['location', '~', 'case_sensitive\\.php$'], [['index', 'index.php'], + ['root', '/var/root']]], + [['location', '~*', 'case_insensitive\\.php$'], []], + [['location', '=', 'exact_match\\.php$'], []], + [['location', '^~', 'ignore_regex\\.php$'], []], + ['ssl_certificate', 'example/fullchain.pem'], + ['ssl_certificate_key', 'example/key.pem']]], + parsed_foo_conf[1][1][1]) + + def test_deploy_no_match_no_default_set(self): + default_conf = self.config.parser.abs_path('sites-enabled/default') + foo_conf = self.config.parser.abs_path('foo.conf') + del self.config.parser.parsed[default_conf][0][1][0] + del self.config.parser.parsed[default_conf][0][1][0] + del self.config.parser.parsed[foo_conf][2][1][0][1][0] + self.config.version = (1, 3, 1) + + self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert, + "www.nomatch.com", "example/cert.pem", "example/key.pem", + "example/chain.pem", "example/fullchain.pem") + + def test_deploy_no_match_fail_multiple_defaults(self): + self.config.version = (1, 3, 1) + self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert, + "www.nomatch.com", "example/cert.pem", "example/key.pem", + "example/chain.pem", "example/fullchain.pem") + + def test_deploy_no_match_add_redirect(self): + default_conf = self.config.parser.abs_path('sites-enabled/default') + foo_conf = self.config.parser.abs_path('foo.conf') + del self.config.parser.parsed[foo_conf][2][1][0][1][0] # remove default_server + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "www.nomatch.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + self.config.deploy_cert( + "nomatch.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + self.config.enhance("www.nomatch.com", "redirect") + + self.config.save() + + self.config.parser.load() + + expected = [ + ['if', '($scheme', '!=', '"https")'], + [['return', '301', 'https://$host$request_uri']] + ] + + generated_conf = self.config.parser.parsed[default_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + + class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index e655bc3e3..ca5de7ff6 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -50,7 +50,9 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods 'sites-enabled/example.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', - 'sites-enabled/globalssl.com']]), + 'sites-enabled/globalssl.com', + 'sites-enabled/ipv6.com', + 'sites-enabled/ipv6ssl.com']]), set(nparser.parsed.keys())) self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], nparser.parsed[nparser.abs_path('server.conf')]) @@ -74,7 +76,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(5, len( + self.assertEqual(7, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -110,7 +112,8 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), - [obj.Addr('4.8.2.6', '57', True, False)], + [obj.Addr('4.8.2.6', '57', True, False, + False, False)], True, True, set(['globalssl.com']), [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] @@ -121,34 +124,42 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() vhost1 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('', '8080', False, False)], + [obj.Addr('', '8080', False, False, + False, False)], False, True, set(['localhost', r'~^(www\.)?(example|bar)\.']), [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), - [obj.Addr('somename', '8080', False, False), - obj.Addr('', '8000', False, False)], + [obj.Addr('somename', '8080', False, False, + False, False), + obj.Addr('', '8000', False, False, + False, False)], False, True, set(['somename', 'another.alias', 'alias']), [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', - False, False), - obj.Addr('127.0.0.1', '', False, False)], + False, False, False, False), + obj.Addr('127.0.0.1', '', False, False, + False, False)], False, True, set(['.example.com', 'example.*']), [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), - [obj.Addr('myhost', '', False, True)], + [obj.Addr('myhost', '', False, True, + False, False), + obj.Addr('otherhost', '', False, True, + False, False)], False, True, set(['www.example.org']), [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), - [obj.Addr('*', '80', True, True)], + [obj.Addr('*', '80', True, True, + False, False)], True, True, set(['*.www.foo.com', '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(10, len(vhosts)) + self.assertEqual(12, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] @@ -395,6 +406,29 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ]) self.assertTrue(server['ssl']) + def test_create_new_vhost_from_default(self): + nparser = parser.NginxParser(self.config_path) + + vhosts = nparser.get_vhosts() + default = [x for x in vhosts if 'default' in x.filep][0] + new_vhost = nparser.create_new_vhost_from_default(default) + nparser.filedump(ext='') + + # check properties of new vhost + self.assertFalse(next(iter(new_vhost.addrs)).default) + self.assertNotEqual(new_vhost.path, default.path) + + # check that things are written to file correctly + new_nparser = parser.NginxParser(self.config_path) + new_vhosts = new_nparser.get_vhosts() + new_defaults = [x for x in new_vhosts if 'default' in x.filep] + self.assertEqual(len(new_defaults), 2) + new_vhost_parsed = new_defaults[1] + self.assertFalse(next(iter(new_vhost_parsed.addrs)).default) + self.assertEqual(next(iter(default.names)), next(iter(new_vhost_parsed.names))) + self.assertEqual(len(default.raw), len(new_vhost_parsed.raw)) + self.assertTrue(next(iter(default.addrs)).super_eq(next(iter(new_vhost_parsed.addrs)))) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default index 26f37020c..4f67fa7d1 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default @@ -1,5 +1,6 @@ server { listen myhost default_server; + listen otherhost default_server; server_name www.example.org; location / { diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com new file mode 100644 index 000000000..7a7744b92 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6.com @@ -0,0 +1,5 @@ +server { + listen 80; + listen [::]:80; + server_name ipv6.com; +} diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com new file mode 100644 index 000000000..d8f7eff12 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -0,0 +1,5 @@ +server { + listen 443 ssl; + listen [::]:443 ssl ipv6only=on; + server_name ipv6ssl.com; +} diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 85db584b3..32a5ed7d2 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -66,7 +66,7 @@ class TlsSniPerformTest(util.NginxTest): self.sni.add_chall(self.achalls[1]) mock_choose.return_value = None result = self.sni.perform() - self.assertTrue(result is None) + self.assertFalse(result is None) def test_perform0(self): responses = self.sni.perform() @@ -125,10 +125,10 @@ class TlsSniPerformTest(util.NginxTest): self.sni.add_chall(self.achalls[0]) self.sni.add_chall(self.achalls[2]) - v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False), - obj.Addr("127.0.0.1", "", False, False)] - v_addr2 = [obj.Addr("myhost", "", False, True)] - v_addr2_print = [obj.Addr("myhost", "", False, False)] + v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False, False, False), + obj.Addr("127.0.0.1", "", False, False, False, False)] + v_addr2 = [obj.Addr("myhost", "", False, True, False, False)] + v_addr2_print = [obj.Addr("myhost", "", False, False, False, False)] ll_addr = [v_addr1, v_addr2] self.sni._mod_config(ll_addr) # pylint: disable=protected-access diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index d6faa12be..7f597ac4a 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,19 +51,32 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) - for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain) - if vhost is None: - logger.error( - "No nginx vhost exists with server_name matching: %s. " - "Please specify server_names in the Nginx config.", - achall.domain) - return None + ipv6, ipv6only = self.configurator.ipv6_info( + self.configurator.config.tls_sni_01_port) - if vhost.addrs: + for achall in self.achalls: + vhost = self.configurator.choose_vhost(achall.domain, raise_if_no_match=False) + + if vhost is not None and vhost.addrs: addresses.append(list(vhost.addrs)) else: - addresses.append([obj.Addr.fromstring(default_addr)]) + if ipv6: + # If IPv6 is active in Nginx configuration + ipv6_addr = "[::]:{0} ssl".format( + self.configurator.config.tls_sni_01_port) + if not ipv6only: + # If ipv6only=on is not already present in the config + ipv6_addr = ipv6_addr + " ipv6only=on" + addresses.append([obj.Addr.fromstring(default_addr), + obj.Addr.fromstring(ipv6_addr)]) + logger.info(("Using default addresses %s and %s for " + + "TLSSNI01 authentication."), + default_addr, + ipv6_addr) + else: + addresses.append([obj.Addr.fromstring(default_addr)]) + logger.info("Using default address %s for TLSSNI01 authentication.", + default_addr) # Create challenge certs responses = [self._setup_challenge_cert(x) for x in self.achalls] @@ -117,7 +130,6 @@ class NginxTlsSni01(common.TLSSNI01): raise errors.MisconfigurationError( 'Certbot could not find an HTTP block to include ' 'TLS-SNI-01 challenges in %s.' % root) - config = [self._make_server_block(pair[0], pair[1]) for pair in six.moves.zip(self.achalls, ll_addrs)] config = nginxparser.UnspacedList(config) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index f605eb751..420d15679 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -251,7 +251,7 @@ class Addr(object): """Normalized representation of addr/port tuple """ if self.ipv6: - return (self._normalize_ipv6(self.tup[0]), self.tup[1]) + return (self.get_ipv6_exploded(), self.tup[1]) return self.tup def __eq__(self, other): From 8b5d6879cc3dcdf41ed097cf08ef2ac9dc8a1e36 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 7 Dec 2017 09:48:54 -0800 Subject: [PATCH 242/631] Create a new server block when making server block ssl (#5220) * create_new_vhost_from_default --> duplicate_vhost * add source_path property * set source path for duplicated vhost * change around logic of where making ssl happens * don't add listen 80 to newly created ssl block * cache vhosts list * remove source path * add redirect block if we created a new server block * Remove listen directives when making server block ssl * Reset vhost cache on parser load * flip connected pointer direction for finding newly made server block to match previous redirect search constraints * also test for new redirect block styles * fix contains_list and test redirect blocks * update lint, parser, and obj tests * reset new vhost (fixing previous bug) and move removing default from addrs under if statement * reuse and update newly created ssl server block when appropriate, and update unit tests * append newly created server blocks to file instead of inserting directly after, so we don't have to update other vhosts' paths * add coverage for NO_IF_REDIRECT_COMMENT_BLOCK * add coverage for parser load calls * replace some double quotes with single quotes * replace backslash continuations with parentheses * update docstrings * switch to only creating a new block on redirect enhancement, including removing the get_vhosts cache * update configurator tests * update obj test * switch delete_default default for duplicate_vhost --- certbot-nginx/certbot_nginx/configurator.py | 150 +++++++++--------- certbot-nginx/certbot_nginx/obj.py | 4 +- certbot-nginx/certbot_nginx/parser.py | 116 ++++++++++---- .../certbot_nginx/tests/configurator_test.py | 81 ++++++++-- certbot-nginx/certbot_nginx/tests/obj_test.py | 8 +- .../certbot_nginx/tests/parser_test.py | 10 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 2 +- 7 files changed, 243 insertions(+), 128 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 98990664f..e9d4e36d4 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,43 +23,24 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants -from certbot_nginx import tls_sni_01 +from certbot_nginx import nginxparser from certbot_nginx import parser +from certbot_nginx import tls_sni_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [[ - ['\n ', 'if', ' ', '($scheme', ' ', '!=', ' ', '"https")'], - [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - '\n '] -], ['\n']] - -TEST_REDIRECT_BLOCK = [ - [ - ['if', '($scheme', '!=', '"https")'], - [ - ['return', '301', 'https://$host$request_uri'] - ] - ], - ['#', ' managed by Certbot'] +REDIRECT_BLOCK = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] ] REDIRECT_COMMENT_BLOCK = [ ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' if ($scheme != "https") {'], - ['\n ', '#', " return 301 https://$host$request_uri;"], - ['\n ', '#', " } # managed by Certbot"], + ['\n ', '#', ' return 301 https://$host$request_uri;'], ['\n'] ] -TEST_REDIRECT_COMMENT_BLOCK = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', " return 301 https://$host$request_uri;"], - ['#', " } # managed by Certbot"], -] - @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class NginxConfigurator(common.Installer): @@ -194,9 +175,7 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, raise_if_no_match=False) - if vhost is None: - vhost = self._vhost_from_duplicated_default(domain) + vhost = self.choose_vhost(domain, create_if_no_match=True) cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] @@ -214,7 +193,7 @@ class NginxConfigurator(common.Installer): ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, raise_if_no_match=True): + def choose_vhost(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -228,8 +207,8 @@ class NginxConfigurator(common.Installer): hostname. Currently we just ignore this. :param str target_name: domain name - :param bool raise_if_no_match: True iff not finding a match is an error; - otherwise, return None + :param bool create_if_no_match: If we should create a new vhost from default + when there is no match found :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -240,7 +219,9 @@ class NginxConfigurator(common.Installer): matches = self._get_ranked_matches(target_name) vhost = self._select_best_name_match(matches) if not vhost: - if raise_if_no_match: + if create_if_no_match: + vhost = self._vhost_from_duplicated_default(target_name) + else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( ("Cannot find a VirtualHost matching domain %s. " @@ -248,16 +229,12 @@ class NginxConfigurator(common.Installer): "please add a corresponding server_name directive to your " "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) - else: - return None - else: - # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + # Note: if we are enhancing with ocsp, vhost should already be ssl. + if not vhost.ssl: + self._make_server_ssl(vhost) return vhost - def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) ipv6_active is true if any server block listens ipv6 address in any port @@ -285,18 +262,19 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain): if self.new_vhost is None: default_vhost = self._get_default_vhost() - self.new_vhost = self.parser.create_new_vhost_from_default(default_vhost) - if not self.new_vhost.ssl: - self._make_server_ssl(self.new_vhost) + self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) self.new_vhost.names = set() - self.new_vhost.names.add(domain) + self._add_server_name_to_vhost(self.new_vhost, domain) + return self.new_vhost + + def _add_server_name_to_vhost(self, vhost, domain): + vhost.names.add(domain) name_block = [['\n ', 'server_name']] - for name in self.new_vhost.names: + for name in vhost.names: name_block[0].append(' ') name_block[0].append(name) - self.parser.add_server_directives(self.new_vhost, name_block, replace=True) - return self.new_vhost + self.parser.add_server_directives(vhost, name_block, replace=True) def _get_default_vhost(self): vhost_list = self.parser.get_vhosts() @@ -505,11 +483,7 @@ class NginxConfigurator(common.Installer): def _make_server_ssl(self, vhost): """Make a server SSL. - Make a server SSL based on server_name and filename by adding a - ``listen IConfig.tls_sni_01_port ssl`` directive to the server block. - - .. todo:: Maybe this should create a new block instead of modifying - the existing one? + Make a server SSL by adding new listen and SSL directives. :param vhost: The vhost to add SSL to. :type vhost: :class:`~certbot_nginx.obj.VirtualHost` @@ -529,7 +503,9 @@ class NginxConfigurator(common.Installer): ipv6_block = ['\n ', 'listen', ' ', - '[::]:{0} ssl'.format(self.config.tls_sni_01_port)] + '[::]:{0}'.format(self.config.tls_sni_01_port), + ' ', + 'ssl'] if not ipv6info[1]: # ipv6only=on is absent in global config ipv6_block.append(' ') @@ -539,8 +515,9 @@ class NginxConfigurator(common.Installer): ipv4_block = ['\n ', 'listen', ' ', - '{0} ssl'.format(self.config.tls_sni_01_port)] - + '{0}'.format(self.config.tls_sni_01_port), + ' ', + 'ssl'] snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() @@ -584,10 +561,12 @@ class NginxConfigurator(common.Installer): raise def _has_certbot_redirect(self, vhost): - return vhost.contains_list(TEST_REDIRECT_BLOCK) + test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + return vhost.contains_list(test_redirect_block) def _has_certbot_redirect_comment(self, vhost): - return vhost.contains_list(TEST_REDIRECT_COMMENT_BLOCK) + test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) + return vhost.contains_list(test_redirect_comment_block) def _add_redirect_block(self, vhost, active=True): """Add redirect directive to vhost @@ -603,7 +582,8 @@ class NginxConfigurator(common.Installer): def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. - Add rewrite directive to non https traffic + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. .. note:: This function saves the configuration @@ -616,26 +596,46 @@ class NginxConfigurator(common.Installer): vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. + vhost = self.choose_redirect_vhost(domain, port) if vhost is None: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) + return + + if vhost.ssl: + new_vhost = self.parser.duplicate_vhost(vhost, + only_directives=['listen', 'server_name']) + + def _ssl_match_func(directive): + return 'ssl' in directive + + def _no_ssl_match_func(directive): + return 'ssl' not in directive + + # remove all ssl addresses from the new block + self.parser.remove_server_directives(new_vhost, 'listen', match_func=_ssl_match_func) + + # remove all non-ssl addresses from the existing block + self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + + vhost = new_vhost + + if self._has_certbot_redirect(vhost): + logger.info("Traffic on port %s already redirecting to ssl in %s", + self.DEFAULT_LISTEN_PORT, vhost.filep) + elif vhost.has_redirect(): + if not self._has_certbot_redirect_comment(vhost): + self._add_redirect_block(vhost, active=False) + logger.info("The appropriate server block is already redirecting " + "traffic. To enable redirect anyway, uncomment the " + "redirect lines in %s.", vhost.filep) else: - if self._has_certbot_redirect(vhost): - logger.info("Traffic on port %s already redirecting to ssl in %s", - self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) - else: - # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) - logger.info("Redirecting all traffic on port %s to ssl in %s", - self.DEFAULT_LISTEN_PORT, vhost.filep) + # Redirect plaintextish host to https + self._add_redirect_block(vhost, active=True) + logger.info("Redirecting all traffic on port %s to ssl in %s", + self.DEFAULT_LISTEN_PORT, vhost.filep) def _enable_ocsp_stapling(self, domain, chain_path): """Include OCSP response in TLS handshake @@ -809,6 +809,7 @@ class NginxConfigurator(common.Installer): """ super(NginxConfigurator, self).recovery_routine() + self.new_vhost = None self.parser.load() def revert_challenge_config(self): @@ -818,6 +819,7 @@ class NginxConfigurator(common.Installer): """ self.revert_temporary_config() + self.new_vhost = None self.parser.load() def rollback_checkpoints(self, rollback=1): @@ -830,6 +832,7 @@ class NginxConfigurator(common.Installer): """ super(NginxConfigurator, self).rollback_checkpoints(rollback) + self.new_vhost = None self.parser.load() ########################################################################### @@ -882,6 +885,11 @@ class NginxConfigurator(common.Installer): self.restart() +def _test_block_from_block(block): + test_block = nginxparser.UnspacedList(block) + parser.comment_directive(test_block, 0) + return test_block[:-1] + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 5816c5571..f5ac5c2e3 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -205,7 +205,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in six.moves.range(0, len(self.raw) - len(test)): + for i in six.moves.range(0, len(self.raw) - len(test) + 1): if self.raw[i:i + len(test)] == test: return True return False @@ -220,6 +220,8 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def ipv4_enabled(self): """Return true if one or more of the listen directives in vhost are IPv4 only""" + if self.addrs is None or len(self.addrs) == 0: + return True for a in self.addrs: if not a.ipv6: return True diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 3eb6264aa..9f13bc59f 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -1,5 +1,6 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy +import functools import glob import logging import os @@ -294,6 +295,30 @@ class NginxParser(object): :param bool replace: Whether to only replace existing directives """ + self._modify_server_directives(vhost, + functools.partial(_add_directives, directives, replace)) + + def remove_server_directives(self, vhost, directive_name, match_func=None): + """Remove all directives of type directive_name. + + :param :class:`~certbot_nginx.obj.VirtualHost` vhost: The vhost + to remove directives from + :param string directive_name: The directive type to remove + :param callable match_func: Function of the directive that returns true for directives + to be deleted. + """ + self._modify_server_directives(vhost, + functools.partial(_remove_directives, directive_name, match_func)) + + def _update_vhost_based_on_new_directives(self, vhost, directives_list): + new_server = self._get_included_directives(directives_list) + parsed_server = self.parse_server(new_server) + vhost.addrs = parsed_server['addrs'] + vhost.ssl = parsed_server['ssl'] + vhost.names = parsed_server['names'] + vhost.raw = new_server + + def _modify_server_directives(self, vhost, block_func): filename = vhost.filep try: result = self.parsed[filename] @@ -302,42 +327,52 @@ class NginxParser(object): if not isinstance(result, list) or len(result) != 2: raise errors.MisconfigurationError("Not a server block.") result = result[1] - _add_directives(result, directives, replace) + block_func(result) - # update vhost based on new directives - new_server = self._get_included_directives(result) - parsed_server = self.parse_server(new_server) - vhost.addrs = parsed_server['addrs'] - vhost.ssl = parsed_server['ssl'] - vhost.names = parsed_server['names'] - vhost.raw = new_server + self._update_vhost_based_on_new_directives(vhost, result) except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def create_new_vhost_from_default(self, vhost_template): - """Duplicate the default vhost in the configuration files. + def duplicate_vhost(self, vhost_template, delete_default=False, only_directives=None): + """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx.obj.VirtualHost` vhost_template: The vhost whose information we copy + :param bool delete_default: If we should remove default_server + from listen directives in the block. + :param list only_directives: If it exists, only duplicate the named directives. Only + looks at first level of depth; does not expand includes. :returns: A vhost object for the newly created vhost :rtype: :class:`~certbot_nginx.obj.VirtualHost` """ # TODO: https://github.com/certbot/certbot/issues/5185 # put it in the same file as the template, at the same level + new_vhost = copy.deepcopy(vhost_template) + enclosing_block = self.parsed[vhost_template.filep] for index in vhost_template.path[:-1]: enclosing_block = enclosing_block[index] - new_location = vhost_template.path[-1] + 1 raw_in_parsed = copy.deepcopy(enclosing_block[vhost_template.path[-1]]) - enclosing_block.insert(new_location, raw_in_parsed) - new_vhost = copy.deepcopy(vhost_template) - new_vhost.path[-1] = new_location - for addr in new_vhost.addrs: - addr.default = False - for directive in enclosing_block[new_vhost.path[-1]][1]: - if len(directive) > 0 and directive[0] == 'listen' and 'default_server' in directive: - del directive[directive.index('default_server')] + + if only_directives is not None: + new_directives = nginxparser.UnspacedList([]) + for directive in raw_in_parsed[1]: + if len(directive) > 0 and directive[0] in only_directives: + new_directives.append(directive) + raw_in_parsed[1] = new_directives + + self._update_vhost_based_on_new_directives(new_vhost, new_directives) + + enclosing_block.append(raw_in_parsed) + new_vhost.path[-1] = len(enclosing_block) - 1 + if delete_default: + for addr in new_vhost.addrs: + addr.default = False + for directive in enclosing_block[new_vhost.path[-1]][1]: + if (len(directive) > 0 and directive[0] == 'listen' + and 'default_server' in directive): + del directive[directive.index('default_server')] return new_vhost def _parse_ssl_options(ssl_options): @@ -486,7 +521,7 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(block, directives, replace): +def _add_directives(directives, replace, block): """Adds or replaces directives in a config block. When replace=False, it's an error to try and add a directive that already @@ -498,8 +533,9 @@ def _add_directives(block, directives, replace): ..todo :: Find directives that are in included files. - :param list block: The block to replace in :param list directives: The new directives. + :param bool replace: Described above. + :param list block: The block to replace in """ for directive in directives: @@ -513,8 +549,12 @@ REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] -def _comment_directive(block, location): - """Add a comment to the end of the line at location.""" +def comment_directive(block, location): + """Add a ``#managed by Certbot`` comment to the end of the line at location. + + :param list block: The block containing the directive to be commented + :param int location: The location within ``block`` of the directive to be commented + """ next_entry = block[location + 1] if location + 1 < len(block) else None if isinstance(next_entry, list) and next_entry: if len(next_entry) >= 2 and next_entry[-2] == "#" and COMMENT in next_entry[-1]: @@ -551,6 +591,12 @@ def _comment_out_directive(block, location, include_location): block[location] = new_dir[0] # set the now-single-line-comment directive back in place +def _find_location(block, directive_name, match_func=None): + """Finds the index of the first instance of directive_name in block. + If no line exists, use None.""" + return next((index for index, line in enumerate(block) \ + if line and line[0] == directive_name and (match_func is None or match_func(line))), None) + def _add_directive(block, directive, replace): """Adds or replaces a single directive in a config block. @@ -566,19 +612,12 @@ def _add_directive(block, directive, replace): block.append(directive) return - def find_location(direc): - """ Find the index of a config line where the name of the directive matches - the name of the directive we want to add. If no line exists, use None. - """ - return next((index for index, line in enumerate(block) \ - if line and line[0] == direc[0]), None) - - location = find_location(directive) + location = _find_location(block, directive[0]) if replace: if location is not None: block[location] = directive - _comment_directive(block, location) + comment_directive(block, location) return # Append directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value @@ -602,7 +641,7 @@ def _add_directive(block, directive, replace): included_directives = _parse_ssl_options(directive[1]) for included_directive in included_directives: - included_dir_loc = find_location(included_directive) + included_dir_loc = _find_location(block, included_directive[0]) included_dir_name = included_directive[0] if not is_whitespace_or_comment(included_directive) \ and not can_append(included_dir_loc, included_dir_name): @@ -614,10 +653,19 @@ def _add_directive(block, directive, replace): if can_append(location, directive_name): block.append(directive) - _comment_directive(block, len(block) - 1) + comment_directive(block, len(block) - 1) elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) +def _remove_directives(directive_name, match_func, block): + """Removes directives of name directive_name from a config block if match_func matches. + """ + while True: + location = _find_location(block, directive_name, match_func=match_func) + if location is None: + return + del block[location] + def _apply_global_addr_ssl(addr_to_ssl, parsed_server): """Apply global sslishness information to the parsed server block """ diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 996bd238b..e708b159a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -443,10 +443,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = [ - ['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ] + expected = ['return', '301', 'https://$host$request_uri'] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -462,6 +459,35 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_split_for_redirect(self): + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.deploy_cert( + "example.org", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.enhance("www.example.com", "redirect") + generated_conf = self.config.parser.parsed[example_conf] + self.assertEqual( + [[['server'], [ + ['server_name', '.example.com'], + ['server_name', 'example.*'], [], + ['listen', '5001', 'ssl'], ['#', ' managed by Certbot'], + ['ssl_certificate', 'example/fullchain.pem'], ['#', ' managed by Certbot'], + ['ssl_certificate_key', 'example/key.pem'], ['#', ' managed by Certbot'], + ['include', self.config.mod_ssl_conf], ['#', ' managed by Certbot'], + ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], + [], []]], + [['server'], [ + ['listen', '69.50.225.155:9000'], + ['listen', '127.0.0.1'], + ['server_name', '.example.com'], + ['server_name', 'example.*'], + ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], + [], []]]], + generated_conf) + @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): @@ -494,9 +520,38 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[example_conf] expected = [ ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] + ['#', ' return 301 https://$host$request_uri;'], + ] + for line in expected: + self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) + + @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') + @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') + def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): + # Test that we add a redirect as a comment if there is already a + # redirect-class statement in the block that isn't managed by certbot + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + + self.config.deploy_cert( + "example.org", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + # Has a non-Certbot redirect, and has no existing comment + mock_contains_list.return_value = False + mock_has_redirect.return_value = True + with mock.patch("certbot_nginx.configurator.logger") as mock_logger: + self.config.enhance("www.example.com", "redirect") + self.assertEqual(mock_logger.info.call_args[0][0], + "The appropriate server block is already redirecting " + "traffic. To enable redirect anyway, uncomment the " + "redirect lines in %s.") + generated_conf = self.config.parser.parsed[example_conf] + expected = [ + ['#', ' Redirect non-https traffic to https'], + ['#', ' return 301 https://$host$request_uri;'], ] for line in expected: self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) @@ -704,14 +759,18 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = [ - ['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ] + expected = ['return', '301', 'https://$host$request_uri'] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + @mock.patch('certbot.reverter.logger') + @mock.patch('certbot_nginx.parser.NginxParser.load') + def test_parser_reload_after_config_changes(self, mock_parser_load, unused_mock_logger): + self.config.recovery_routine() + self.config.revert_challenge_config() + self.config.rollback_checkpoints() + self.assertTrue(mock_parser_load.call_count == 3) class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index ba136bb78..92cb0e086 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -171,8 +171,8 @@ class VirtualHostTest(unittest.TestCase): def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import TEST_REDIRECT_BLOCK - test_needle = TEST_REDIRECT_BLOCK + from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block + test_needle = _test_block_from_block(REDIRECT_BLOCK) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], @@ -181,9 +181,7 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ], + ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index ca5de7ff6..e21acb8ea 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -334,9 +334,9 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ["\n", "a", " ", "b", "\n"], ["c", " ", "d"], ["\n", "e", " ", "f"]]) - from certbot_nginx.parser import _comment_directive, COMMENT_BLOCK - _comment_directive(block, 1) - _comment_directive(block, 0) + from certbot_nginx.parser import comment_directive, COMMENT_BLOCK + comment_directive(block, 1) + comment_directive(block, 0) self.assertEqual(block.spaced, [ ["\n", "a", " ", "b", "\n"], COMMENT_BLOCK, @@ -406,12 +406,12 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ]) self.assertTrue(server['ssl']) - def test_create_new_vhost_from_default(self): + def test_duplicate_vhost(self): nparser = parser.NginxParser(self.config_path) vhosts = nparser.get_vhosts() default = [x for x in vhosts if 'default' in x.filep][0] - new_vhost = nparser.create_new_vhost_from_default(default) + new_vhost = nparser.duplicate_vhost(default, delete_default=True) nparser.filedump(ext='') # check properties of new vhost diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 7f597ac4a..eca198bfe 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,7 +55,7 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, raise_if_no_match=False) + vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) if vhost is not None and vhost.addrs: addresses.append(list(vhost.addrs)) From e696766ed102a6999a53bba8cb2881348d870487 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Dec 2017 13:48:44 -0800 Subject: [PATCH 243/631] Expand on changes to the Apache plugin --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d059b53..4acfc0401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * The Apache plugin now parses some distro specific Apache configuration files on non-Debian systems allowing it to get a clearer picture on the running - Apache configuration. + configuration. Internally, these changes were structured so that external + contributors can easily write patches to make the plugin work in new Apache + configurations. * Certbot better reports network failures by removing information about connection retries from the error output. * An unnecessary question when using Certbot's webroot plugin interactively has @@ -25,6 +27,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Certbot's NGINX plugin no longer sometimes incorrectly reports that it was unable to deploy a HTTP->HTTPS redirect when requesting Certbot to enable a redirect for multiple domains. +* Problems where the Apache plugin was failing to find directives and + duplicating existing directives on openSUSE have been resolved. * An issue running the test shipped with Certbot and some our DNS plugins with older versions of mock have been resolved. * On some systems, users reported strangely interleaved output depending on From 5d0888809f6337e36616bf753e24405d6d957491 Mon Sep 17 00:00:00 2001 From: Michael Coleman Date: Fri, 8 Dec 2017 12:53:47 +1300 Subject: [PATCH 244/631] Remove slash from document root path in Webroot example (#5293) It seems the document root path to the `--webroot-path`, `-w` option can't have a trailing slash. Here is an example of a user who followed this example and had their certificate signing request error out. https://superuser.com/questions/1273984/why-does-certbot-letsencrypt-recieve-a-403-forbidden --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 4fd0b5ec8..ab4670052 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -106,7 +106,7 @@ specified ``--webroot-path``. So, for instance, :: - certbot certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net + certbot certonly --webroot -w /var/www/example -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and From 00464283824045e026552b8871677b3bfdbebac8 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Dec 2017 12:45:04 -0800 Subject: [PATCH 245/631] print warnings for 3.3 users (#5283) fix errors --- acme/acme/__init__.py | 9 +++++++++ certbot/main.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index e8a0b16a8..618dda200 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,3 +10,12 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ +import sys +import warnings + +if sys.version_info[:2] == (3, 3): + warnings.warn( + "Python 3.3 support will be dropped in the next release of " + "acme. Please upgrade your Python version.", + PendingDeprecationWarning, + ) #pragma: no cover diff --git a/certbot/main.py b/certbot/main.py index 2d4881d1d..72af7fbba 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1215,6 +1215,9 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise + if sys.version_info[:2] == (3, 3): + logger.warning("Python 3.3 support will be dropped in the next release " + "of Certbot - please upgrade your Python version.") set_displayer(config) From 8bc785ed4656db9542571b2093815265acaad201 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 8 Dec 2017 16:35:59 -0800 Subject: [PATCH 246/631] Make Travis builds faster in master (#5314) * Remove extra le-auto tests from master * Remove dockerfile-dev test from master * Remove intermediate Python 3.x tests from master * Reorder travis jobs for speed --- .travis.yml | 66 +++++++++++++++-------------------------------------- 1 file changed, 19 insertions(+), 47 deletions(-) diff --git a/.travis.yml b/.travis.yml index 359801622..3d41bfa4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,19 +13,32 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" - - python: "2.7" - env: TOXENV=lint - - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV=py27_install BOULDER_INTEGRATION=1 sudo: required services: docker + - python: "2.7" + env: TOXENV=cover FYI="this also tests py27" + - sudo: required + env: TOXENV=nginx_compat + services: docker + before_install: + addons: + - python: "2.7" + env: TOXENV=lint - python: "2.6" env: TOXENV=py26 sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27-oldest + sudo: required + services: docker + - python: "3.3" + env: TOXENV=py33 + sudo: required + services: docker + - python: "3.6" + env: TOXENV=py36 sudo: required services: docker - sudo: required @@ -33,55 +46,14 @@ matrix: services: docker before_install: addons: - - sudo: required - env: TOXENV=nginx_compat - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=le_auto_precise - services: docker - before_install: - addons: - sudo: required env: TOXENV=le_auto_trusty services: docker before_install: addons: - - sudo: required - env: TOXENV=le_auto_wheezy - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=le_auto_centos6 - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=docker_dev - services: docker - before_install: - addons: - python: "2.7" env: TOXENV=apacheconftest sudo: required - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - - python: "3.4" - env: TOXENV=py34 - sudo: required - services: docker - - python: "3.5" - env: TOXENV=py35 - sudo: required - services: docker - - python: "3.6" - env: TOXENV=py36 - sudo: required - services: docker - python: "2.7" env: TOXENV=nginxroundtrip From 2abc94661a16f1a4c5bc18e9c48870f2db458eac Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 11 Dec 2017 20:25:09 +0100 Subject: [PATCH 247/631] Use josepy instead of acme.jose. (#5203) --- acme/acme/challenges.py | 2 +- acme/acme/challenges_test.py | 2 +- acme/acme/client.py | 4 +- acme/acme/client_test.py | 2 +- acme/acme/crypto_util_test.py | 2 +- acme/acme/errors.py | 2 +- acme/acme/fields.py | 3 +- acme/acme/fields_test.py | 3 +- acme/acme/jose/__init__.py | 82 --- acme/acme/jose/b64.py | 61 --- acme/acme/jose/b64_test.py | 77 --- acme/acme/jose/errors.py | 35 -- acme/acme/jose/errors_test.py | 17 - acme/acme/jose/interfaces.py | 216 -------- acme/acme/jose/interfaces_test.py | 114 ---- acme/acme/jose/json_util.py | 485 ------------------ acme/acme/jose/json_util_test.py | 381 -------------- acme/acme/jose/jwa.py | 180 ------- acme/acme/jose/jwa_test.py | 104 ---- acme/acme/jose/jwk.py | 281 ---------- acme/acme/jose/jwk_test.py | 191 ------- acme/acme/jose/jws.py | 433 ---------------- acme/acme/jose/jws_test.py | 239 --------- acme/acme/jose/util.py | 226 -------- acme/acme/jose/util_test.py | 199 ------- acme/acme/jws.py | 6 +- acme/acme/jws_test.py | 3 +- acme/acme/messages.py | 9 +- acme/acme/messages_test.py | 2 +- acme/acme/standalone_test.py | 2 +- acme/acme/test_util.py | 3 +- acme/docs/api/jose.rst | 9 +- acme/docs/api/jose/base64.rst | 5 - acme/docs/api/jose/errors.rst | 5 - acme/docs/api/jose/interfaces.rst | 5 - acme/docs/api/jose/json_util.rst | 5 - acme/docs/api/jose/jwa.rst | 5 - acme/docs/api/jose/jwk.rst | 5 - acme/docs/api/jose/jws.rst | 5 - acme/docs/api/jose/util.rst | 5 - acme/docs/conf.py | 1 + acme/examples/example_client.py | 2 +- acme/setup.py | 7 +- certbot-apache/certbot_apache/tests/util.py | 3 +- .../certbot_compatibility_test/util.py | 3 +- certbot-nginx/certbot_nginx/tests/util.py | 3 +- certbot/account.py | 2 +- certbot/achallenges.py | 3 +- certbot/client.py | 2 +- certbot/crypto_util.py | 4 +- certbot/main.py | 2 +- certbot/plugins/common.py | 2 +- certbot/plugins/common_test.py | 2 +- certbot/plugins/dns_test_common.py | 2 +- certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/plugins/standalone_test.py | 2 +- certbot/plugins/webroot_test.py | 2 +- certbot/tests/account_test.py | 2 +- certbot/tests/acme_util.py | 2 +- certbot/tests/client_test.py | 2 +- certbot/tests/display/ops_test.py | 2 +- certbot/tests/main_test.py | 3 +- certbot/tests/util.py | 3 +- tools/deactivate.py | 2 +- 64 files changed, 53 insertions(+), 3422 deletions(-) delete mode 100644 acme/acme/jose/__init__.py delete mode 100644 acme/acme/jose/b64.py delete mode 100644 acme/acme/jose/b64_test.py delete mode 100644 acme/acme/jose/errors.py delete mode 100644 acme/acme/jose/errors_test.py delete mode 100644 acme/acme/jose/interfaces.py delete mode 100644 acme/acme/jose/interfaces_test.py delete mode 100644 acme/acme/jose/json_util.py delete mode 100644 acme/acme/jose/json_util_test.py delete mode 100644 acme/acme/jose/jwa.py delete mode 100644 acme/acme/jose/jwa_test.py delete mode 100644 acme/acme/jose/jwk.py delete mode 100644 acme/acme/jose/jwk_test.py delete mode 100644 acme/acme/jose/jws.py delete mode 100644 acme/acme/jose/jws_test.py delete mode 100644 acme/acme/jose/util.py delete mode 100644 acme/acme/jose/util_test.py delete mode 100644 acme/docs/api/jose/base64.rst delete mode 100644 acme/docs/api/jose/errors.rst delete mode 100644 acme/docs/api/jose/interfaces.rst delete mode 100644 acme/docs/api/jose/json_util.rst delete mode 100644 acme/docs/api/jose/jwa.rst delete mode 100644 acme/docs/api/jose/jwk.rst delete mode 100644 acme/docs/api/jose/jws.rst delete mode 100644 acme/docs/api/jose/util.rst diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 14641af10..96997297b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -6,13 +6,13 @@ import logging import socket from cryptography.hazmat.primitives import hashes # type: ignore +import josepy as jose import OpenSSL import requests from acme import errors from acme import crypto_util from acme import fields -from acme import jose logger = logging.getLogger(__name__) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 49e790102..834d569aa 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,6 +1,7 @@ """Tests for acme.challenges.""" import unittest +import josepy as jose import mock import OpenSSL import requests @@ -8,7 +9,6 @@ import requests from six.moves.urllib import parse as urllib_parse # pylint: disable=import-error from acme import errors -from acme import jose from acme import test_util CERT = test_util.load_comparable_cert('cert.pem') diff --git a/acme/acme/client.py b/acme/acme/client.py index 2e07d34d7..dc5efbe86 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -10,13 +10,13 @@ import time import six from six.moves import http_client # pylint: disable=import-error +import josepy as jose import OpenSSL import re import requests import sys from acme import errors -from acme import jose from acme import jws from acme import messages @@ -408,7 +408,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param str uri: URI of certificate :returns: tuple of the form - (response, :class:`acme.jose.ComparableX509`) + (response, :class:`josepy.util.ComparableX509`) :rtype: tuple """ diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4bd762865..84620fc99 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -5,12 +5,12 @@ import unittest from six.moves import http_client # pylint: disable=import-error +import josepy as jose import mock import requests from acme import challenges from acme import errors -from acme import jose from acme import jws as acme_jws from acme import messages from acme import messages_test diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index da433c5a2..1d7f83ccf 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,10 +8,10 @@ import unittest import six from six.moves import socketserver #type: ignore # pylint: disable=import-error +import josepy as jose import OpenSSL from acme import errors -from acme import jose from acme import test_util diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 9d991fd75..de5f9d1f4 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -1,5 +1,5 @@ """ACME errors.""" -from acme.jose import errors as jose_errors +from josepy import errors as jose_errors class Error(Exception): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 12d09acf4..d7ec78403 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -1,10 +1,9 @@ """ACME JSON fields.""" import logging +import josepy as jose import pyrfc3339 -from acme import jose - logger = logging.getLogger(__name__) diff --git a/acme/acme/fields_test.py b/acme/acme/fields_test.py index de852b6fa..69dde8b89 100644 --- a/acme/acme/fields_test.py +++ b/acme/acme/fields_test.py @@ -2,10 +2,9 @@ import datetime import unittest +import josepy as jose import pytz -from acme import jose - class FixedTest(unittest.TestCase): """Tests for acme.fields.Fixed.""" diff --git a/acme/acme/jose/__init__.py b/acme/acme/jose/__init__.py deleted file mode 100644 index 9116bc433..000000000 --- a/acme/acme/jose/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Javascript Object Signing and Encryption (jose). - -This package is a Python implementation of the standards developed by -IETF `Javascript Object Signing and Encryption (Active WG)`_, in -particular the following RFCs: - - - `JSON Web Algorithms (JWA)`_ - - `JSON Web Key (JWK)`_ - - `JSON Web Signature (JWS)`_ - - -.. _`Javascript Object Signing and Encryption (Active WG)`: - https://tools.ietf.org/wg/jose/ - -.. _`JSON Web Algorithms (JWA)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/ - -.. _`JSON Web Key (JWK)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/ - -.. _`JSON Web Signature (JWS)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/ - -""" -from acme.jose.b64 import ( - b64decode, - b64encode, -) - -from acme.jose.errors import ( - DeserializationError, - SerializationError, - Error, - UnrecognizedTypeError, -) - -from acme.jose.interfaces import JSONDeSerializable - -from acme.jose.json_util import ( - Field, - JSONObjectWithFields, - TypedJSONObjectWithFields, - decode_b64jose, - decode_cert, - decode_csr, - decode_hex16, - encode_b64jose, - encode_cert, - encode_csr, - encode_hex16, -) - -from acme.jose.jwa import ( - HS256, - HS384, - HS512, - JWASignature, - PS256, - PS384, - PS512, - RS256, - RS384, - RS512, -) - -from acme.jose.jwk import ( - JWK, - JWKRSA, -) - -from acme.jose.jws import ( - Header, - JWS, - Signature, -) - -from acme.jose.util import ( - ComparableX509, - ComparableKey, - ComparableRSAKey, - ImmutableMap, -) diff --git a/acme/acme/jose/b64.py b/acme/acme/jose/b64.py deleted file mode 100644 index cf79aa820..000000000 --- a/acme/acme/jose/b64.py +++ /dev/null @@ -1,61 +0,0 @@ -"""JOSE Base64. - -`JOSE Base64`_ is defined as: - - - URL-safe Base64 - - padding stripped - - -.. _`JOSE Base64`: - https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C - -.. Do NOT try to call this module "base64", as it will "shadow" the - standard library. - -""" -import base64 - -import six - - -def b64encode(data): - """JOSE Base64 encode. - - :param data: Data to be encoded. - :type data: `bytes` - - :returns: JOSE Base64 string. - :rtype: bytes - - :raises TypeError: if `data` is of incorrect type - - """ - if not isinstance(data, six.binary_type): - raise TypeError('argument should be {0}'.format(six.binary_type)) - return base64.urlsafe_b64encode(data).rstrip(b'=') - - -def b64decode(data): - """JOSE Base64 decode. - - :param data: Base64 string to be decoded. If it's unicode, then - only ASCII characters are allowed. - :type data: `bytes` or `unicode` - - :returns: Decoded data. - :rtype: bytes - - :raises TypeError: if input is of incorrect type - :raises ValueError: if input is unicode with non-ASCII characters - - """ - if isinstance(data, six.string_types): - try: - data = data.encode('ascii') - except UnicodeEncodeError: - raise ValueError( - 'unicode argument should contain only ASCII characters') - elif not isinstance(data, six.binary_type): - raise TypeError('argument should be a str or unicode') - - return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4))) diff --git a/acme/acme/jose/b64_test.py b/acme/acme/jose/b64_test.py deleted file mode 100644 index cbabe2251..000000000 --- a/acme/acme/jose/b64_test.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Tests for acme.jose.b64.""" -import unittest - -import six - - -# https://en.wikipedia.org/wiki/Base64#Examples -B64_PADDING_EXAMPLES = { - b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='), - b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='), - b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''), - b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='), - b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='), -} - - -B64_URL_UNSAFE_EXAMPLES = { - six.int2byte(251) + six.int2byte(239): b'--8', - six.int2byte(255) * 2: b'__8', -} - - -class B64EncodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64encode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64encode - return b64encode(data) - - def test_empty(self): - self.assertEqual(self._call(b''), b'') - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_different_paddings(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_unicode_fails_with_type_error(self): - self.assertRaises(TypeError, self._call, u'some unicode') - - -class B64DecodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64decode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64decode - return b64decode(data) - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_without_padding(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_with_padding(self): - for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64 + pad), text) - - def test_unicode_with_ascii(self): - self.assertEqual(self._call(u'YQ'), b'a') - - def test_non_ascii_unicode_fails(self): - self.assertRaises(ValueError, self._call, u'\u0105') - - def test_type_error_no_unicode_or_bytes(self): - self.assertRaises(TypeError, self._call, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/errors.py b/acme/acme/jose/errors.py deleted file mode 100644 index 74c9443e1..000000000 --- a/acme/acme/jose/errors.py +++ /dev/null @@ -1,35 +0,0 @@ -"""JOSE errors.""" - - -class Error(Exception): - """Generic JOSE Error.""" - - -class DeserializationError(Error): - """JSON deserialization error.""" - - def __str__(self): - return "Deserialization error: {0}".format( - super(DeserializationError, self).__str__()) - - -class SerializationError(Error): - """JSON serialization error.""" - - -class UnrecognizedTypeError(DeserializationError): - """Unrecognized type error. - - :ivar str typ: The unrecognized type of the JSON object. - :ivar jobj: Full JSON object. - - """ - - def __init__(self, typ, jobj): - self.typ = typ - self.jobj = jobj - super(UnrecognizedTypeError, self).__init__(str(self)) - - def __str__(self): - return '{0} was not recognized, full message: {1}'.format( - self.typ, self.jobj) diff --git a/acme/acme/jose/errors_test.py b/acme/acme/jose/errors_test.py deleted file mode 100644 index 919980920..000000000 --- a/acme/acme/jose/errors_test.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for acme.jose.errors.""" -import unittest - - -class UnrecognizedTypeErrorTest(unittest.TestCase): - def setUp(self): - from acme.jose.errors import UnrecognizedTypeError - self.error = UnrecognizedTypeError('foo', {'type': 'foo'}) - - def test_str(self): - self.assertEqual( - "foo was not recognized, full message: {'type': 'foo'}", - str(self.error)) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py deleted file mode 100644 index f841848b3..000000000 --- a/acme/acme/jose/interfaces.py +++ /dev/null @@ -1,216 +0,0 @@ -"""JOSE interfaces.""" -import abc -import collections -import json - -import six - -from acme.jose import errors -from acme.jose import util - -# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class -# pylint: disable=too-few-public-methods - - -@six.add_metaclass(abc.ABCMeta) -class JSONDeSerializable(object): - # pylint: disable=too-few-public-methods - """Interface for (de)serializable JSON objects. - - Please recall, that standard Python library implements - :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform - translations based on respective :ref:`conversion tables - ` that look pretty much like the one below (for - complete tables see relevant Python documentation): - - .. _conversion-table: - - ====== ====== - JSON Python - ====== ====== - object dict - ... ... - ====== ====== - - While the above **conversion table** is about translation of JSON - documents to/from the basic Python types only, - :class:`JSONDeSerializable` introduces the following two concepts: - - serialization - Turning an arbitrary Python object into Python object that can - be encoded into a JSON document. **Full serialization** produces - a Python object composed of only basic types as required by the - :ref:`conversion table `. **Partial - serialization** (accomplished by :meth:`to_partial_json`) - produces a Python object that might also be built from other - :class:`JSONDeSerializable` objects. - - deserialization - Turning a decoded Python object (necessarily one of the basic - types as required by the :ref:`conversion table - `) into an arbitrary Python object. - - Serialization produces **serialized object** ("partially serialized - object" or "fully serialized object" for partial and full - serialization respectively) and deserialization produces - **deserialized object**, both usually denoted in the source code as - ``jobj``. - - Wording in the official Python documentation might be confusing - after reading the above, but in the light of those definitions, one - can view :meth:`json.JSONDecoder.decode` as decoder and - deserializer of basic types, :meth:`json.JSONEncoder.default` as - serializer of basic types, :meth:`json.JSONEncoder.encode` as - serializer and encoder of basic types. - - One could extend :mod:`json` to support arbitrary object - (de)serialization either by: - - - overriding :meth:`json.JSONDecoder.decode` and - :meth:`json.JSONEncoder.default` in subclasses - - - or passing ``object_hook`` argument (or ``object_hook_pairs``) - to :func:`json.load`/:func:`json.loads` or ``default`` argument - for :func:`json.dump`/:func:`json.dumps`. - - Interestingly, ``default`` is required to perform only partial - serialization, as :func:`json.dumps` applies ``default`` - recursively. This is the idea behind making :meth:`to_partial_json` - produce only partial serialization, while providing custom - :meth:`json_dumps` that dumps with ``default`` set to - :meth:`json_dump_default`. - - To make further documentation a bit more concrete, please, consider - the following imaginatory implementation example:: - - class Foo(JSONDeSerializable): - def to_partial_json(self): - return 'foo' - - @classmethod - def from_json(cls, jobj): - return Foo() - - class Bar(JSONDeSerializable): - def to_partial_json(self): - return [Foo(), Foo()] - - @classmethod - def from_json(cls, jobj): - return Bar() - - """ - - @abc.abstractmethod - def to_partial_json(self): # pragma: no cover - """Partially serialize. - - Following the example, **partial serialization** means the following:: - - assert isinstance(Bar().to_partial_json()[0], Foo) - assert isinstance(Bar().to_partial_json()[1], Foo) - - # in particular... - assert Bar().to_partial_json() != ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Partially serializable object. - - """ - raise NotImplementedError() - - def to_json(self): - """Fully serialize. - - Again, following the example from before, **full serialization** - means the following:: - - assert Bar().to_json() == ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Fully serialized object. - - """ - def _serialize(obj): - if isinstance(obj, JSONDeSerializable): - return _serialize(obj.to_partial_json()) - if isinstance(obj, six.string_types): # strings are Sequence - return obj - elif isinstance(obj, list): - return [_serialize(subobj) for subobj in obj] - elif isinstance(obj, collections.Sequence): - # default to tuple, otherwise Mapping could get - # unhashable list - return tuple(_serialize(subobj) for subobj in obj) - elif isinstance(obj, collections.Mapping): - return dict((_serialize(key), _serialize(value)) - for key, value in six.iteritems(obj)) - else: - return obj - - return _serialize(self) - - @util.abstractclassmethod - def from_json(cls, jobj): # pylint: disable=unused-argument - """Deserialize a decoded JSON document. - - :param jobj: Python object, composed of only other basic data - types, as decoded from JSON document. Not necessarily - :class:`dict` (as decoded from "JSON object" document). - - :raises acme.jose.errors.DeserializationError: - if decoding was unsuccessful, e.g. in case of unparseable - X509 certificate, or wrong padding in JOSE base64 encoded - string, etc. - - """ - # TypeError: Can't instantiate abstract class with - # abstract methods from_json, to_partial_json - return cls() # pylint: disable=abstract-class-instantiated - - @classmethod - def json_loads(cls, json_string): - """Deserialize from JSON document string.""" - try: - loads = json.loads(json_string) - except ValueError as error: - raise errors.DeserializationError(error) - return cls.from_json(loads) - - def json_dumps(self, **kwargs): - """Dump to JSON string using proper serializer. - - :returns: JSON document string. - :rtype: str - - """ - return json.dumps(self, default=self.json_dump_default, **kwargs) - - def json_dumps_pretty(self): - """Dump the object to pretty JSON document string. - - :rtype: str - - """ - return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) - - @classmethod - def json_dump_default(cls, python_object): - """Serialize Python object. - - This function is meant to be passed as ``default`` to - :func:`json.dump` or :func:`json.dumps`. They call - ``default(python_object)`` only for non-basic Python types, so - this function necessarily raises :class:`TypeError` if - ``python_object`` is not an instance of - :class:`IJSONSerializable`. - - Please read the class docstring for more information. - - """ - if isinstance(python_object, JSONDeSerializable): - return python_object.to_partial_json() - else: # this branch is necessary, cannot just "return" - raise TypeError(repr(python_object) + ' is not JSON serializable') diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py deleted file mode 100644 index cf98ff371..000000000 --- a/acme/acme/jose/interfaces_test.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Tests for acme.jose.interfaces.""" -import unittest - - -class JSONDeSerializableTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes - - def setUp(self): - from acme.jose.interfaces import JSONDeSerializable - - # pylint: disable=missing-docstring,invalid-name - - class Basic(JSONDeSerializable): - def __init__(self, v): - self.v = v - - def to_partial_json(self): - return self.v - - @classmethod - def from_json(cls, jobj): - return cls(jobj) - - class Sequence(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return [self.x, self.y] - - @classmethod - def from_json(cls, jobj): - return cls( - Basic.from_json(jobj[0]), Basic.from_json(jobj[1])) - - class Mapping(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return {self.x: self.y} - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - - self.basic1 = Basic('foo1') - self.basic2 = Basic('foo2') - self.seq = Sequence(self.basic1, self.basic2) - self.mapping = Mapping(self.basic1, self.basic2) - self.nested = Basic([[self.basic1]]) - self.tuple = Basic(('foo',)) - - # pylint: disable=invalid-name - self.Basic = Basic - self.Sequence = Sequence - self.Mapping = Mapping - - def test_to_json_sequence(self): - self.assertEqual(self.seq.to_json(), ['foo1', 'foo2']) - - def test_to_json_mapping(self): - self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'}) - - def test_to_json_other(self): - mock_value = object() - self.assertTrue(self.Basic(mock_value).to_json() is mock_value) - - def test_to_json_nested(self): - self.assertEqual(self.nested.to_json(), [['foo1']]) - - def test_to_json(self): - self.assertEqual(self.tuple.to_json(), (('foo', ))) - - def test_from_json_not_implemented(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx') - - def test_json_loads(self): - seq = self.Sequence.json_loads('["foo1", "foo2"]') - self.assertTrue(isinstance(seq, self.Sequence)) - self.assertTrue(isinstance(seq.x, self.Basic)) - self.assertTrue(isinstance(seq.y, self.Basic)) - self.assertEqual(seq.x.v, 'foo1') - self.assertEqual(seq.y.v, 'foo2') - - def test_json_dumps(self): - self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps()) - - def test_json_dumps_pretty(self): - self.assertEqual(self.seq.json_dumps_pretty(), - '[\n "foo1",\n "foo2"\n]') - - def test_json_dump_default(self): - from acme.jose.interfaces import JSONDeSerializable - - self.assertEqual( - 'foo1', JSONDeSerializable.json_dump_default(self.basic1)) - - jobj = JSONDeSerializable.json_dump_default(self.seq) - self.assertEqual(len(jobj), 2) - self.assertTrue(jobj[0] is self.basic1) - self.assertTrue(jobj[1] is self.basic2) - - def test_json_dump_default_type_error(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises( - TypeError, JSONDeSerializable.json_dump_default, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py deleted file mode 100644 index 4baadda5e..000000000 --- a/acme/acme/jose/json_util.py +++ /dev/null @@ -1,485 +0,0 @@ -"""JSON (de)serialization framework. - -The framework presented here is somewhat based on `Go's "json" package`_ -(especially the ``omitempty`` functionality). - -.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/ - -""" -import abc -import binascii -import logging - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class Field(object): - """JSON object field. - - :class:`Field` is meant to be used together with - :class:`JSONObjectWithFields`. - - ``encoder`` (``decoder``) is a callable that accepts a single - parameter, i.e. a value to be encoded (decoded), and returns the - serialized (deserialized) value. In case of errors it should raise - :class:`~acme.jose.errors.SerializationError` - (:class:`~acme.jose.errors.DeserializationError`). - - Note, that ``decoder`` should perform partial serialization only. - - :ivar str json_name: Name of the field when encoded to JSON. - :ivar default: Default value (used when not present in JSON object). - :ivar bool omitempty: If ``True`` and the field value is empty, then - it will not be included in the serialized JSON object, and - ``default`` will be used for deserialization. Otherwise, if ``False``, - field is considered as required, value will always be included in the - serialized JSON objected, and it must also be present when - deserializing. - - """ - __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc') - - def __init__(self, json_name, default=None, omitempty=False, - decoder=None, encoder=None): - # pylint: disable=too-many-arguments - self.json_name = json_name - self.default = default - self.omitempty = omitempty - - self.fdec = self.default_decoder if decoder is None else decoder - self.fenc = self.default_encoder if encoder is None else encoder - - @classmethod - def _empty(cls, value): - """Is the provided value considered "empty" for this field? - - This is useful for subclasses that might want to override the - definition of being empty, e.g. for some more exotic data types. - - """ - return not isinstance(value, bool) and not value - - def omit(self, value): - """Omit the value in output?""" - return self._empty(value) and self.omitempty - - def _update_params(self, **kwargs): - current = dict(json_name=self.json_name, default=self.default, - omitempty=self.omitempty, - decoder=self.fdec, encoder=self.fenc) - current.update(kwargs) - return type(self)(**current) # pylint: disable=star-args - - def decoder(self, fdec): - """Descriptor to change the decoder on JSON object field.""" - return self._update_params(decoder=fdec) - - def encoder(self, fenc): - """Descriptor to change the encoder on JSON object field.""" - return self._update_params(encoder=fenc) - - def decode(self, value): - """Decode a value, optionally with context JSON object.""" - return self.fdec(value) - - def encode(self, value): - """Encode a value, optionally with context JSON object.""" - return self.fenc(value) - - @classmethod - def default_decoder(cls, value): - """Default decoder. - - Recursively deserialize into immutable types ( - :class:`acme.jose.util.frozendict` instead of - :func:`dict`, :func:`tuple` instead of :func:`list`). - - """ - # bases cases for different types returned by json.loads - if isinstance(value, list): - return tuple(cls.default_decoder(subvalue) for subvalue in value) - elif isinstance(value, dict): - return util.frozendict( - dict((cls.default_decoder(key), cls.default_decoder(value)) - for key, value in six.iteritems(value))) - else: # integer or string - return value - - @classmethod - def default_encoder(cls, value): - """Default (passthrough) encoder.""" - # field.to_partial_json() is no good as encoder has to do partial - # serialization only - return value - - -class JSONObjectWithFieldsMeta(abc.ABCMeta): - """Metaclass for :class:`JSONObjectWithFields` and its subclasses. - - It makes sure that, for any class ``cls`` with ``__metaclass__`` - set to ``JSONObjectWithFieldsMeta``: - - 1. All fields (attributes of type :class:`Field`) in the class - definition are moved to the ``cls._fields`` dictionary, where - keys are field attribute names and values are fields themselves. - - 2. ``cls.__slots__`` is extended by all field attribute names - (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__`` - are stored in ``cls._orig_slots``. - - In a consequence, for a field attribute name ``some_field``, - ``cls.some_field`` will be a slot descriptor and not an instance - of :class:`Field`. For example:: - - some_field = Field('someField', default=()) - - class Foo(object): - __metaclass__ = JSONObjectWithFieldsMeta - __slots__ = ('baz',) - some_field = some_field - - assert Foo.__slots__ == ('some_field', 'baz') - assert Foo._orig_slots == () - assert Foo.some_field is not Field - - assert Foo._fields.keys() == ['some_field'] - assert Foo._fields['some_field'] is some_field - - As an implementation note, this metaclass inherits from - :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate - the metaclass conflict (:class:`ImmutableMap` and - :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`, - use :class:`abc.ABCMeta` as its metaclass). - - """ - - def __new__(mcs, name, bases, dikt): - fields = {} - - for base in bases: - fields.update(getattr(base, '_fields', {})) - # Do not reorder, this class might override fields from base classes! - for key, value in tuple(six.iteritems(dikt)): - # not six.iterkeys() (in-place edit!) - if isinstance(value, Field): - fields[key] = dikt.pop(key) - - dikt['_orig_slots'] = dikt.get('__slots__', ()) - dikt['__slots__'] = tuple( - list(dikt['_orig_slots']) + list(six.iterkeys(fields))) - dikt['_fields'] = fields - - return abc.ABCMeta.__new__(mcs, name, bases, dikt) - - -@six.add_metaclass(JSONObjectWithFieldsMeta) -class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): - # pylint: disable=too-few-public-methods - """JSON object with fields. - - Example:: - - class Foo(JSONObjectWithFields): - bar = Field('Bar') - empty = Field('Empty', omitempty=True) - - @bar.encoder - def bar(value): - return value + 'bar' - - @bar.decoder - def bar(value): - if not value.endswith('bar'): - raise errors.DeserializationError('No bar suffix!') - return value[:-3] - - assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'} - assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz') - assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'}) - == Foo(bar='baz', empty='!')) - assert Foo(bar='baz').bar == 'baz' - - """ - - @classmethod - def _defaults(cls): - """Get default fields values.""" - return dict([(slot, field.default) for slot, field - in six.iteritems(cls._fields)]) - - def __init__(self, **kwargs): - # pylint: disable=star-args - super(JSONObjectWithFields, self).__init__( - **(dict(self._defaults(), **kwargs))) - - def encode(self, name): - """Encode a single field. - - :param str name: Name of the field to be encoded. - - :raises errors.SerializationError: if field cannot be serialized - :raises errors.Error: if field could not be found - - """ - try: - field = self._fields[name] - except KeyError: - raise errors.Error("Field not found: {0}".format(name)) - - return field.encode(getattr(self, name)) - - def fields_to_partial_json(self): - """Serialize fields to JSON.""" - jobj = {} - omitted = set() - for slot, field in six.iteritems(self._fields): - value = getattr(self, slot) - - if field.omit(value): - omitted.add((slot, value)) - else: - try: - jobj[field.json_name] = field.encode(value) - except errors.SerializationError as error: - raise errors.SerializationError( - 'Could not encode {0} ({1}): {2}'.format( - slot, value, error)) - return jobj - - def to_partial_json(self): - return self.fields_to_partial_json() - - @classmethod - def _check_required(cls, jobj): - missing = set() - for _, field in six.iteritems(cls._fields): - if not field.omitempty and field.json_name not in jobj: - missing.add(field.json_name) - - if missing: - raise errors.DeserializationError( - 'The following fields are required: {0}'.format( - ','.join(missing))) - - @classmethod - def fields_from_json(cls, jobj): - """Deserialize fields from JSON.""" - cls._check_required(jobj) - fields = {} - for slot, field in six.iteritems(cls._fields): - if field.json_name not in jobj and field.omitempty: - fields[slot] = field.default - else: - value = jobj[field.json_name] - try: - fields[slot] = field.decode(value) - except errors.DeserializationError as error: - raise errors.DeserializationError( - 'Could not decode {0!r} ({1!r}): {2}'.format( - slot, value, error)) - return fields - - @classmethod - def from_json(cls, jobj): - return cls(**cls.fields_from_json(jobj)) - - -def encode_b64jose(data): - """Encode JOSE Base-64 field. - - :param bytes data: - :rtype: `unicode` - - """ - # b64encode produces ASCII characters only - return b64.b64encode(data).decode('ascii') - - -def decode_b64jose(data, size=None, minimum=False): - """Decode JOSE Base-64 field. - - :param unicode data: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - error_cls = TypeError if six.PY2 else binascii.Error - try: - decoded = b64.b64decode(data.encode()) - except error_cls as error: - raise errors.DeserializationError(error) - - if size is not None and ((not minimum and len(decoded) != size) or - (minimum and len(decoded) < size)): - raise errors.DeserializationError( - "Expected at least or exactly {0} bytes".format(size)) - - return decoded - - -def encode_hex16(value): - """Hexlify. - - :param bytes value: - :rtype: unicode - - """ - return binascii.hexlify(value).decode() - - -def decode_hex16(value, size=None, minimum=False): - """Decode hexlified field. - - :param unicode value: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - value = value.encode() - if size is not None and ((not minimum and len(value) != size * 2) or - (minimum and len(value) < size * 2)): - raise errors.DeserializationError() - error_cls = TypeError if six.PY2 else binascii.Error - try: - return binascii.unhexlify(value) - except error_cls as error: - raise errors.DeserializationError(error) - - -def encode_cert(cert): - """Encode certificate as JOSE Base-64 DER. - - :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) - - -def decode_cert(b64der): - """Decode JOSE Base-64 DER-encoded certificate. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -def encode_csr(csr): - """Encode CSR as JOSE Base-64 DER. - - :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) - - -def decode_csr(b64der): - """Decode JOSE Base-64 DER-encoded CSR. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class TypedJSONObjectWithFields(JSONObjectWithFields): - """JSON object with type.""" - - typ = NotImplemented - """Type of the object. Subclasses must override.""" - - type_field_name = "type" - """Field name used to distinguish different object types. - - Subclasses will probably have to override this. - - """ - - TYPES = NotImplemented - """Types registered for JSON deserialization""" - - @classmethod - def register(cls, type_cls, typ=None): - """Register class for JSON deserialization.""" - typ = type_cls.typ if typ is None else typ - cls.TYPES[typ] = type_cls - return type_cls - - @classmethod - def get_type_cls(cls, jobj): - """Get the registered class for ``jobj``.""" - if cls in six.itervalues(cls.TYPES): - if cls.type_field_name not in jobj: - raise errors.DeserializationError( - "Missing type field ({0})".format(cls.type_field_name)) - # cls is already registered type_cls, force to use it - # so that, e.g Revocation.from_json(jobj) fails if - # jobj["type"] != "revocation". - return cls - - if not isinstance(jobj, dict): - raise errors.DeserializationError( - "{0} is not a dictionary object".format(jobj)) - try: - typ = jobj[cls.type_field_name] - except KeyError: - raise errors.DeserializationError("missing type field") - - try: - return cls.TYPES[typ] - except KeyError: - raise errors.UnrecognizedTypeError(typ, jobj) - - def to_partial_json(self): - """Get JSON serializable object. - - :returns: Serializable JSON object representing ACME typed object. - :meth:`validate` will almost certainly not work, due to reasons - explained in :class:`acme.interfaces.IJSONSerializable`. - :rtype: dict - - """ - jobj = self.fields_to_partial_json() - jobj[self.type_field_name] = self.typ - return jobj - - @classmethod - def from_json(cls, jobj): - """Deserialize ACME object from valid JSON object. - - :raises acme.errors.UnrecognizedTypeError: if type - of the ACME object has not been registered. - - """ - # make sure subclasses don't cause infinite recursive from_json calls - type_cls = cls.get_type_cls(jobj) - return type_cls(**type_cls.fields_from_json(jobj)) diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py deleted file mode 100644 index 25e36211e..000000000 --- a/acme/acme/jose/json_util_test.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Tests for acme.jose.json_util.""" -import itertools -import unittest - -import mock -import six - -from acme import test_util - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -CERT = test_util.load_comparable_cert('cert.pem') -CSR = test_util.load_comparable_csr('csr.pem') - - -class FieldTest(unittest.TestCase): - """Tests for acme.jose.json_util.Field.""" - - def test_no_omit_boolean(self): - from acme.jose.json_util import Field - for default, omitempty, value in itertools.product( - [True, False], [True, False], [True, False]): - self.assertFalse( - Field("foo", default=default, omitempty=omitempty).omit(value)) - - def test_descriptors(self): - mock_value = mock.MagicMock() - - # pylint: disable=missing-docstring - - def decoder(unused_value): - return 'd' - - def encoder(unused_value): - return 'e' - - from acme.jose.json_util import Field - field = Field('foo') - - field = field.encoder(encoder) - self.assertEqual('e', field.encode(mock_value)) - - field = field.decoder(decoder) - self.assertEqual('e', field.encode(mock_value)) - self.assertEqual('d', field.decode(mock_value)) - - def test_default_encoder_is_partial(self): - class MockField(interfaces.JSONDeSerializable): - # pylint: disable=missing-docstring - def to_partial_json(self): - return 'foo' # pragma: no cover - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - mock_field = MockField() - - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_field) is mock_field) - # in particular... - self.assertNotEqual('foo', Field.default_encoder(mock_field)) - - def test_default_encoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_value) is mock_value) - - def test_default_decoder_list_to_tuple(self): - from acme.jose.json_util import Field - self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3])) - - def test_default_decoder_dict_to_frozendict(self): - from acme.jose.json_util import Field - obj = Field.default_decoder({'x': 2}) - self.assertTrue(isinstance(obj, util.frozendict)) - self.assertEqual(obj, util.frozendict(x=2)) - - def test_default_decoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_decoder(mock_value) is mock_value) - - -class JSONObjectWithFieldsMetaTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta.""" - - def setUp(self): - from acme.jose.json_util import Field - from acme.jose.json_util import JSONObjectWithFieldsMeta - self.field = Field('Baz') - self.field2 = Field('Baz2') - # pylint: disable=invalid-name,missing-docstring,too-few-public-methods - # pylint: disable=blacklisted-name - - @six.add_metaclass(JSONObjectWithFieldsMeta) - class A(object): - __slots__ = ('bar',) - baz = self.field - - class B(A): - pass - - class C(A): - baz = self.field2 - - self.a_cls = A - self.b_cls = B - self.c_cls = C - - def test_fields(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field}, self.a_cls._fields) - self.assertEqual({'baz': self.field}, self.b_cls._fields) - - def test_fields_inheritance(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field2}, self.c_cls._fields) - - def test_slots(self): - self.assertEqual(('bar', 'baz'), self.a_cls.__slots__) - self.assertEqual(('baz',), self.b_cls.__slots__) - - def test_orig_slots(self): - # pylint: disable=protected-access,no-member - self.assertEqual(('bar',), self.a_cls._orig_slots) - self.assertEqual((), self.b_cls._orig_slots) - - -class JSONObjectWithFieldsTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFields.""" - # pylint: disable=protected-access - - def setUp(self): - from acme.jose.json_util import JSONObjectWithFields - from acme.jose.json_util import Field - - class MockJSONObjectWithFields(JSONObjectWithFields): - # pylint: disable=invalid-name,missing-docstring,no-self-argument - # pylint: disable=too-few-public-methods - x = Field('x', omitempty=True, - encoder=(lambda x: x * 2), - decoder=(lambda x: x / 2)) - y = Field('y') - z = Field('Z') # on purpose uppercase - - @y.encoder - def y(value): - if value == 500: - raise errors.SerializationError() - return value - - @y.decoder - def y(value): - if value == 500: - raise errors.DeserializationError() - return value - - # pylint: disable=invalid-name - self.MockJSONObjectWithFields = MockJSONObjectWithFields - self.mock = MockJSONObjectWithFields(x=None, y=2, z=3) - - def test_init_defaults(self): - self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3)) - - def test_encode(self): - self.assertEqual(10, self.MockJSONObjectWithFields( - x=5, y=0, z=0).encode("x")) - - def test_encode_wrong_field(self): - self.assertRaises(errors.Error, self.mock.encode, 'foo') - - def test_encode_serialization_error_passthrough(self): - self.assertRaises( - errors.SerializationError, - self.MockJSONObjectWithFields(y=500, z=None).encode, "y") - - def test_fields_to_partial_json_omits_empty(self): - self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3}) - - def test_fields_from_json_fills_default_for_empty(self): - self.assertEqual( - {'x': None, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3})) - - def test_fields_from_json_fails_on_missing(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'Z': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0}) - - def test_fields_to_partial_json_encoder(self): - self.assertEqual( - self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(), - {'x': 2, 'y': 2, 'Z': 3}) - - def test_fields_from_json_decoder(self): - self.assertEqual( - {'x': 2, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json( - {'x': 4, 'y': 2, 'Z': 3})) - - def test_fields_to_partial_json_error_passthrough(self): - self.assertRaises( - errors.SerializationError, self.MockJSONObjectWithFields( - x=1, y=500, z=3).to_partial_json) - - def test_fields_from_json_error_passthrough(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.from_json, - {'x': 4, 'y': 500, 'Z': 3}) - - -class DeEncodersTest(unittest.TestCase): - def setUp(self): - self.b64_cert = ( - u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM' - u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz' - u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF' - u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx' - u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI' - u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW' - u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD' - u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1' - u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE' - u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd' - u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o' - ) - self.b64_csr = ( - u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F' - u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw' - u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb' - u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As' - u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3' - u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG' - u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW' - u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg' - ) - - def test_encode_b64jose(self): - from acme.jose.json_util import encode_b64jose - encoded = encode_b64jose(b'x') - self.assertTrue(isinstance(encoded, six.string_types)) - self.assertEqual(u'eA', encoded) - - def test_decode_b64jose(self): - from acme.jose.json_util import decode_b64jose - decoded = decode_b64jose(u'eA') - self.assertTrue(isinstance(decoded, six.binary_type)) - self.assertEqual(b'x', decoded) - - def test_decode_b64jose_padding_error(self): - from acme.jose.json_util import decode_b64jose - self.assertRaises(errors.DeserializationError, decode_b64jose, u'x') - - def test_decode_b64jose_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3)) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=2) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=4) - - def test_decode_b64jose_minimum_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True)) - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_b64jose, - u'Zm9v', size=4, minimum=True) - - def test_encode_hex16(self): - from acme.jose.json_util import encode_hex16 - encoded = encode_hex16(b'foo') - self.assertEqual(u'666f6f', encoded) - self.assertTrue(isinstance(encoded, six.string_types)) - - def test_decode_hex16(self): - from acme.jose.json_util import decode_hex16 - decoded = decode_hex16(u'666f6f') - self.assertEqual(b'foo', decoded) - self.assertTrue(isinstance(decoded, six.binary_type)) - - def test_decode_hex16_minimum_size(self): - from acme.jose.json_util import decode_hex16 - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True)) - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_hex16, - u'666f6f', size=4, minimum=True) - - def test_decode_hex16_odd_length(self): - from acme.jose.json_util import decode_hex16 - self.assertRaises(errors.DeserializationError, decode_hex16, u'x') - - def test_encode_cert(self): - from acme.jose.json_util import encode_cert - self.assertEqual(self.b64_cert, encode_cert(CERT)) - - def test_decode_cert(self): - from acme.jose.json_util import decode_cert - cert = decode_cert(self.b64_cert) - self.assertTrue(isinstance(cert, util.ComparableX509)) - self.assertEqual(cert, CERT) - self.assertRaises(errors.DeserializationError, decode_cert, u'') - - def test_encode_csr(self): - from acme.jose.json_util import encode_csr - self.assertEqual(self.b64_csr, encode_csr(CSR)) - - def test_decode_csr(self): - from acme.jose.json_util import decode_csr - csr = decode_csr(self.b64_csr) - self.assertTrue(isinstance(csr, util.ComparableX509)) - self.assertEqual(csr, CSR) - self.assertRaises(errors.DeserializationError, decode_csr, u'') - - -class TypedJSONObjectWithFieldsTest(unittest.TestCase): - - def setUp(self): - from acme.jose.json_util import TypedJSONObjectWithFields - - # pylint: disable=missing-docstring,abstract-method - # pylint: disable=too-few-public-methods - - class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields): - TYPES = {} - type_field_name = 'type' - - @MockParentTypedJSONObjectWithFields.register - class MockTypedJSONObjectWithFields( - MockParentTypedJSONObjectWithFields): - typ = 'test' - __slots__ = ('foo',) - - @classmethod - def fields_from_json(cls, jobj): - return {'foo': jobj['foo']} - - def fields_to_partial_json(self): - return {'foo': self.foo} - - self.parent_cls = MockParentTypedJSONObjectWithFields - self.msg = MockTypedJSONObjectWithFields(foo='bar') - - def test_to_partial_json(self): - self.assertEqual(self.msg.to_partial_json(), { - 'type': 'test', - 'foo': 'bar', - }) - - def test_from_json_non_dict_fails(self): - for value in [[], (), 5, "asd"]: # all possible input types - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, value) - - def test_from_json_dict_no_type_fails(self): - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, {}) - - def test_from_json_unknown_type_fails(self): - self.assertRaises(errors.UnrecognizedTypeError, - self.parent_cls.from_json, {'type': 'bar'}) - - def test_from_json_returns_obj(self): - self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json( - {'type': 'test', 'foo': 'bar'})) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py deleted file mode 100644 index 9b682ecab..000000000 --- a/acme/acme/jose/jwa.py +++ /dev/null @@ -1,180 +0,0 @@ -"""JSON Web Algorithm. - -https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40 - -""" -import abc -import collections -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import hmac # type: ignore -from cryptography.hazmat.primitives.asymmetric import padding # type: ignore - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import jwk - - -logger = logging.getLogger(__name__) - - -class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method - # pylint: disable=too-few-public-methods - # for some reason disable=abstract-method has to be on the line - # above... - """JSON Web Algorithm.""" - - -class JWASignature(JWA, collections.Hashable): # type: ignore - """JSON Web Signature Algorithm.""" - SIGNATURES = {} # type: dict - - def __init__(self, name): - self.name = name - - def __eq__(self, other): - if not isinstance(other, JWASignature): - return NotImplemented - return self.name == other.name - - def __hash__(self): - return hash((self.__class__, self.name)) - - def __ne__(self, other): - return not self == other - - @classmethod - def register(cls, signature_cls): - """Register class for JSON deserialization.""" - cls.SIGNATURES[signature_cls.name] = signature_cls - return signature_cls - - def to_partial_json(self): - return self.name - - @classmethod - def from_json(cls, jobj): - return cls.SIGNATURES[jobj] - - @abc.abstractmethod - def sign(self, key, msg): # pragma: no cover - """Sign the ``msg`` using ``key``.""" - raise NotImplementedError() - - @abc.abstractmethod - def verify(self, key, msg, sig): # pragma: no cover - """Verify the ``msg` and ``sig`` using ``key``.""" - raise NotImplementedError() - - def __repr__(self): - return self.name - - -class _JWAHS(JWASignature): - - kty = jwk.JWKOct - - def __init__(self, name, hash_): - super(_JWAHS, self).__init__(name) - self.hash = hash_() - - def sign(self, key, msg): - signer = hmac.HMAC(key, self.hash, backend=default_backend()) - signer.update(msg) - return signer.finalize() - - def verify(self, key, msg, sig): - verifier = hmac.HMAC(key, self.hash, backend=default_backend()) - verifier.update(msg) - try: - verifier.verify(sig) - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARSA(object): - - kty = jwk.JWKRSA - padding = NotImplemented - hash = NotImplemented - - def sign(self, key, msg): - """Sign the ``msg`` using ``key``.""" - try: - signer = key.signer(self.padding, self.hash) - except AttributeError as error: - logger.debug(error, exc_info=True) - raise errors.Error("Public key cannot be used for signing") - except ValueError as error: # digest too large - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - signer.update(msg) - try: - return signer.finalize() - except ValueError as error: - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - - def verify(self, key, msg, sig): - """Verify the ``msg` and ``sig`` using ``key``.""" - verifier = key.verifier(sig, self.padding, self.hash) - verifier.update(msg) - try: - verifier.verify() - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWARS, self).__init__(name) - self.padding = padding.PKCS1v15() - self.hash = hash_() - - -class _JWAPS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWAPS, self).__init__(name) - self.padding = padding.PSS( - mgf=padding.MGF1(hash_()), - salt_length=padding.PSS.MAX_LENGTH) - self.hash = hash_() - - -class _JWAES(JWASignature): # pylint: disable=abstract-class-not-used - - # TODO: implement ES signatures - - def sign(self, key, msg): # pragma: no cover - raise NotImplementedError() - - def verify(self, key, msg, sig): # pragma: no cover - raise NotImplementedError() - - -HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256)) -HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384)) -HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512)) - -RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256)) -RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384)) -RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512)) - -PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256)) -PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384)) -PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512)) - -ES256 = JWASignature.register(_JWAES('ES256')) -ES384 = JWASignature.register(_JWAES('ES384')) -ES512 = JWASignature.register(_JWAES('ES512')) diff --git a/acme/acme/jose/jwa_test.py b/acme/acme/jose/jwa_test.py deleted file mode 100644 index 3328d083a..000000000 --- a/acme/acme/jose/jwa_test.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tests for acme.jose.jwa.""" -import unittest - -from acme import test_util - -from acme.jose import errors - - -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') -RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem') - - -class JWASignatureTest(unittest.TestCase): - """Tests for acme.jose.jwa.JWASignature.""" - - def setUp(self): - from acme.jose.jwa import JWASignature - - class MockSig(JWASignature): - # pylint: disable=missing-docstring,too-few-public-methods - # pylint: disable=abstract-class-not-used - def sign(self, key, msg): - raise NotImplementedError() # pragma: no cover - - def verify(self, key, msg, sig): - raise NotImplementedError() # pragma: no cover - - # pylint: disable=invalid-name - self.Sig1 = MockSig('Sig1') - self.Sig2 = MockSig('Sig2') - - def test_eq(self): - self.assertEqual(self.Sig1, self.Sig1) - - def test_ne(self): - self.assertNotEqual(self.Sig1, self.Sig2) - - def test_ne_other_type(self): - self.assertNotEqual(self.Sig1, 5) - - def test_repr(self): - self.assertEqual('Sig1', repr(self.Sig1)) - self.assertEqual('Sig2', repr(self.Sig2)) - - def test_to_partial_json(self): - self.assertEqual(self.Sig1.to_partial_json(), 'Sig1') - self.assertEqual(self.Sig2.to_partial_json(), 'Sig2') - - def test_from_json(self): - from acme.jose.jwa import JWASignature - from acme.jose.jwa import RS256 - self.assertTrue(JWASignature.from_json('RS256') is RS256) - - -class JWAHSTest(unittest.TestCase): # pylint: disable=too-few-public-methods - - def test_it(self): - from acme.jose.jwa import HS256 - sig = ( - b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4" - b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO" - ) - self.assertEqual(HS256.sign(b'some key', b'foo'), sig) - self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True) - self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False) - - -class JWARSTest(unittest.TestCase): - - def test_sign_no_private_part(self): - from acme.jose.jwa import RS256 - self.assertRaises( - errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo') - - def test_sign_key_too_small(self): - from acme.jose.jwa import RS256 - from acme.jose.jwa import PS256 - self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo') - self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo') - - def test_rs(self): - from acme.jose.jwa import RS256 - sig = ( - b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O' - b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c' - b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99' - b'\xd2\xb9.>}\xfd' - ) - self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig) - self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig)) - self.assertFalse(RS256.verify( - RSA512_KEY.public_key(), b'foo', sig + b'!')) - - def test_ps(self): - from acme.jose.jwa import PS256 - sig = PS256.sign(RSA1024_KEY, b'foo') - self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig)) - self.assertFalse(PS256.verify( - RSA1024_KEY.public_key(), b'foo', sig + b'!')) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py deleted file mode 100644 index 54423f670..000000000 --- a/acme/acme/jose/jwk.py +++ /dev/null @@ -1,281 +0,0 @@ -"""JSON Web Key.""" -import abc -import binascii -import json -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec # type: ignore -from cryptography.hazmat.primitives.asymmetric import rsa - -import six - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class JWK(json_util.TypedJSONObjectWithFields): - # pylint: disable=too-few-public-methods - """JSON Web Key.""" - type_field_name = 'kty' - TYPES = {} # type: dict - cryptography_key_types = () # type: tuple - """Subclasses should override.""" - - required = NotImplemented - """Required members of public key's representation as defined by JWK/JWA.""" - - _thumbprint_json_dumps_params = { - # "no whitespace or line breaks before or after any syntactic - # elements" - 'indent': None, - 'separators': (',', ':'), - # "members ordered lexicographically by the Unicode [UNICODE] - # code points of the member names" - 'sort_keys': True, - } - - def thumbprint(self, hash_function=hashes.SHA256): - """Compute JWK Thumbprint. - - https://tools.ietf.org/html/rfc7638 - - :returns bytes: - - """ - digest = hashes.Hash(hash_function(), backend=default_backend()) - digest.update(json.dumps( - dict((k, v) for k, v in six.iteritems(self.to_json()) - if k in self.required), - **self._thumbprint_json_dumps_params).encode()) - return digest.finalize() - - @abc.abstractmethod - def public_key(self): # pragma: no cover - """Generate JWK with public key. - - For symmetric cryptosystems, this would return ``self``. - - """ - raise NotImplementedError() - - @classmethod - def _load_cryptography_key(cls, data, password=None, backend=None): - backend = default_backend() if backend is None else backend - exceptions = {} - - # private key? - for loader in (serialization.load_pem_private_key, - serialization.load_der_private_key): - try: - return loader(data, password, backend) - except (ValueError, TypeError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # public key? - for loader in (serialization.load_pem_public_key, - serialization.load_der_public_key): - try: - return loader(data, backend) - except (ValueError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # no luck - raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) - - @classmethod - def load(cls, data, password=None, backend=None): - """Load serialized key as JWK. - - :param str data: Public or private key serialized as PEM or DER. - :param str password: Optional password. - :param backend: A `.PEMSerializationBackend` and - `.DERSerializationBackend` provider. - - :raises errors.Error: if unable to deserialize, or unsupported - JWK algorithm - - :returns: JWK of an appropriate type. - :rtype: `JWK` - - """ - try: - key = cls._load_cryptography_key(data, password, backend) - except errors.Error as error: - logger.debug('Loading symmetric key, asymmetric failed: %s', error) - return JWKOct(key=data) - - if cls.typ is not NotImplemented and not isinstance( - key, cls.cryptography_key_types): - raise errors.Error('Unable to deserialize {0} into {1}'.format( - key.__class__, cls.__class__)) - for jwk_cls in six.itervalues(cls.TYPES): - if isinstance(key, jwk_cls.cryptography_key_types): - return jwk_cls(key=key) - raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) - - -@JWK.register -class JWKES(JWK): # pragma: no cover - # pylint: disable=abstract-class-not-used - """ES JWK. - - .. warning:: This is not yet implemented! - - """ - typ = 'ES' - cryptography_key_types = ( - ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) - required = ('crv', JWK.type_field_name, 'x', 'y') - - def fields_to_partial_json(self): - raise NotImplementedError() - - @classmethod - def fields_from_json(cls, jobj): - raise NotImplementedError() - - def public_key(self): - raise NotImplementedError() - - -@JWK.register -class JWKOct(JWK): - """Symmetric JWK.""" - typ = 'oct' - __slots__ = ('key',) - required = ('k', JWK.type_field_name) - - def fields_to_partial_json(self): - # TODO: An "alg" member SHOULD also be present to identify the - # algorithm intended to be used with the key, unless the - # application uses another means or convention to determine - # the algorithm used. - return {'k': json_util.encode_b64jose(self.key)} - - @classmethod - def fields_from_json(cls, jobj): - return cls(key=json_util.decode_b64jose(jobj['k'])) - - def public_key(self): - return self - - -@JWK.register -class JWKRSA(JWK): - """RSA JWK. - - :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey` - or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped - in `.ComparableRSAKey` - - """ - typ = 'RSA' - cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) - __slots__ = ('key',) - required = ('e', JWK.type_field_name, 'n') - - def __init__(self, *args, **kwargs): - if 'key' in kwargs and not isinstance( - kwargs['key'], util.ComparableRSAKey): - kwargs['key'] = util.ComparableRSAKey(kwargs['key']) - super(JWKRSA, self).__init__(*args, **kwargs) - - @classmethod - def _encode_param(cls, data): - """Encode Base64urlUInt. - - :type data: long - :rtype: unicode - - """ - def _leading_zeros(arg): - if len(arg) % 2: - return '0' + arg - return arg - - return json_util.encode_b64jose(binascii.unhexlify( - _leading_zeros(hex(data)[2:].rstrip('L')))) - - @classmethod - def _decode_param(cls, data): - """Decode Base64urlUInt.""" - try: - return int(binascii.hexlify(json_util.decode_b64jose(data)), 16) - except ValueError: # invalid literal for long() with base 16 - raise errors.DeserializationError() - - def public_key(self): - return type(self)(key=self.key.public_key()) - - @classmethod - def fields_from_json(cls, jobj): - # pylint: disable=invalid-name - n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e')) - public_numbers = rsa.RSAPublicNumbers(e=e, n=n) - if 'd' not in jobj: # public key - key = public_numbers.public_key(default_backend()) - else: # private key - d = cls._decode_param(jobj['d']) - if ('p' in jobj or 'q' in jobj or 'dp' in jobj or - 'dq' in jobj or 'qi' in jobj or 'oth' in jobj): - # "If the producer includes any of the other private - # key parameters, then all of the others MUST be - # present, with the exception of "oth", which MUST - # only be present when more than two prime factors - # were used." - p, q, dp, dq, qi, = all_params = tuple( - jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) - if tuple(param for param in all_params if param is None): - raise errors.Error( - 'Some private parameters are missing: {0}'.format( - all_params)) - p, q, dp, dq, qi = tuple( - cls._decode_param(x) for x in all_params) - - # TODO: check for oth - else: - # cryptography>=0.8 - p, q = rsa.rsa_recover_prime_factors(n, e, d) - dp = rsa.rsa_crt_dmp1(d, p) - dq = rsa.rsa_crt_dmq1(d, q) - qi = rsa.rsa_crt_iqmp(p, q) - - key = rsa.RSAPrivateNumbers( - p, q, d, dp, dq, qi, public_numbers).private_key( - default_backend()) - - return cls(key=key) - - def fields_to_partial_json(self): - # pylint: disable=protected-access - if isinstance(self.key._wrapped, rsa.RSAPublicKey): - numbers = self.key.public_numbers() - params = { - 'n': numbers.n, - 'e': numbers.e, - } - else: # rsa.RSAPrivateKey - private = self.key.private_numbers() - public = self.key.public_key().public_numbers() - params = { - 'n': public.n, - 'e': public.e, - 'd': private.d, - 'p': private.p, - 'q': private.q, - 'dp': private.dmp1, - 'dq': private.dmq1, - 'qi': private.iqmp, - } - return dict((key, self._encode_param(value)) - for key, value in six.iteritems(params)) diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py deleted file mode 100644 index eea5793bf..000000000 --- a/acme/acme/jose/jwk_test.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Tests for acme.jose.jwk.""" -import binascii -import unittest - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -DSA_PEM = test_util.load_vector('dsa512_key.pem') -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') - - -class JWKTest(unittest.TestCase): - """Tests for acme.jose.jwk.JWK.""" - - def test_load(self): - from acme.jose.jwk import JWK - self.assertRaises(errors.Error, JWK.load, DSA_PEM) - - def test_load_subclass_wrong_type(self): - from acme.jose.jwk import JWKRSA - self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) - - -class JWKTestBaseMixin(object): - """Mixin test for JWK subclass tests.""" - - thumbprint = NotImplemented - - def test_thumbprint_private(self): - self.assertEqual(self.thumbprint, self.jwk.thumbprint()) - - def test_thumbprint_public(self): - self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) - - -class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKOct.""" - - thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>" - b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea" - b"\x8e(\x8a\xb2i\x1c") - - def setUp(self): - from acme.jose.jwk import JWKOct - self.jwk = JWKOct(key=b'foo') - self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')} - - def test_to_partial_json(self): - self.assertEqual(self.jwk.to_partial_json(), self.jobj) - - def test_from_json(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.from_json(self.jobj)) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWKOct - hash(JWKOct.from_json(self.jobj)) - - def test_load(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.load(b'foo')) - - def test_public_key(self): - self.assertTrue(self.jwk.public_key() is self.jwk) - - -class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKRSA.""" - # pylint: disable=too-many-instance-attributes - - thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c' - b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y') - - def setUp(self): - from acme.jose.jwk import JWKRSA - self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) - self.jwk256json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk', - } - # pylint: disable=protected-access - self.jwk256_not_comparable = JWKRSA( - key=RSA256_KEY.public_key()._wrapped) - self.jwk512 = JWKRSA(key=RSA512_KEY.public_key()) - self.jwk512json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5' - '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q', - } - self.private = JWKRSA(key=RSA256_KEY) - self.private_json_small = self.jwk256json.copy() - self.private_json_small['d'] = ( - 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE') - self.private_json = self.jwk256json.copy() - self.private_json.update({ - 'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE', - 'p': 'zUVNZn4lLLBD1R6NE8TKNQ', - 'q': 'wcfKfc7kl5jfqXArCRSURQ', - 'dp': 'CWJFq43QvT5Bm5iN8n1okQ', - 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', - 'qi': 'oi45cEkbVoJjAbnQpFY87Q', - }) - self.jwk = self.private - - def test_init_auto_comparable(self): - self.assertTrue(isinstance( - self.jwk256_not_comparable.key, util.ComparableRSAKey)) - self.assertEqual(self.jwk256, self.jwk256_not_comparable) - - def test_encode_param_zero(self): - from acme.jose.jwk import JWKRSA - # pylint: disable=protected-access - # TODO: move encode/decode _param to separate class - self.assertEqual('AA', JWKRSA._encode_param(0)) - - def test_equals(self): - self.assertEqual(self.jwk256, self.jwk256) - self.assertEqual(self.jwk512, self.jwk512) - - def test_not_equals(self): - self.assertNotEqual(self.jwk256, self.jwk512) - self.assertNotEqual(self.jwk512, self.jwk256) - - def test_load(self): - from acme.jose.jwk import JWKRSA - self.assertEqual(self.private, JWKRSA.load( - test_util.load_vector('rsa256_key.pem'))) - - def test_public_key(self): - self.assertEqual(self.jwk256, self.private.public_key()) - - def test_to_partial_json(self): - self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json) - self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json) - self.assertEqual(self.private.to_partial_json(), self.private_json) - - def test_from_json(self): - from acme.jose.jwk import JWK - self.assertEqual( - self.jwk256, JWK.from_json(self.jwk256json)) - self.assertEqual( - self.jwk512, JWK.from_json(self.jwk512json)) - self.assertEqual(self.private, JWK.from_json(self.private_json)) - - def test_from_json_private_small(self): - from acme.jose.jwk import JWK - self.assertEqual(self.private, JWK.from_json(self.private_json_small)) - - def test_from_json_missing_one_additional(self): - from acme.jose.jwk import JWK - del self.private_json['q'] - self.assertRaises(errors.Error, JWK.from_json, self.private_json) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWK - hash(JWK.from_json(self.jwk256json)) - - def test_from_json_non_schema_errors(self): - # valid against schema, but still failing - from acme.jose.jwk import JWK - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': ''}) - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': '1'}) - - def test_thumbprint_go_jose(self): - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384 - from acme.jose.jwk import JWKRSA - key = JWKRSA.json_loads("""{ - "kty": "RSA", - "kid": "bilbo.baggins@hobbiton.example", - "use": "sig", - "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw", - "e": "AQAB" -}""") - self.assertEqual( - binascii.hexlify(key.thumbprint()), - b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932") - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py deleted file mode 100644 index 5f446e4b1..000000000 --- a/acme/acme/jose/jws.py +++ /dev/null @@ -1,433 +0,0 @@ -"""JOSE Web Signature.""" -import argparse -import base64 -import sys - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk -from acme.jose import util - - -class MediaType(object): - """MediaType field encoder/decoder.""" - - PREFIX = 'application/' - """MIME Media Type and Content Type prefix.""" - - @classmethod - def decode(cls, value): - """Decoder.""" - # 4.1.10 - if '/' not in value: - if ';' in value: - raise errors.DeserializationError('Unexpected semi-colon') - return cls.PREFIX + value - return value - - @classmethod - def encode(cls, value): - """Encoder.""" - # 4.1.10 - if ';' not in value: - assert value.startswith(cls.PREFIX) - return value[len(cls.PREFIX):] - return value - - -class Header(json_util.JSONObjectWithFields): - """JOSE Header. - - .. warning:: This class supports **only** Registered Header - Parameter Names (as defined in section 4.1 of the - protocol). If you need Public Header Parameter Names (4.2) - or Private Header Parameter Names (4.3), you must subclass - and override :meth:`from_json` and :meth:`to_partial_json` - appropriately. - - .. warning:: This class does not support any extensions through - the "crit" (Critical) Header Parameter (4.1.11) and as a - conforming implementation, :meth:`from_json` treats its - occurrence as an error. Please subclass if you seek for - a different behaviour. - - :ivar x5tS256: "x5t#S256" - :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`. - :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`. - - """ - alg = json_util.Field( - 'alg', decoder=jwa.JWASignature.from_json, omitempty=True) - jku = json_util.Field('jku', omitempty=True) - jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True) - kid = json_util.Field('kid', omitempty=True) - x5u = json_util.Field('x5u', omitempty=True) - x5c = json_util.Field('x5c', omitempty=True, default=()) - x5t = json_util.Field( - 'x5t', decoder=json_util.decode_b64jose, omitempty=True) - x5tS256 = json_util.Field( - 'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True) - typ = json_util.Field('typ', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - cty = json_util.Field('cty', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - crit = json_util.Field('crit', omitempty=True, default=()) - - def not_omitted(self): - """Fields that would not be omitted in the JSON object.""" - return dict((name, getattr(self, name)) - for name, field in six.iteritems(self._fields) - if not field.omit(getattr(self, name))) - - def __add__(self, other): - if not isinstance(other, type(self)): - raise TypeError('Header cannot be added to: {0}'.format( - type(other))) - - not_omitted_self = self.not_omitted() - not_omitted_other = other.not_omitted() - - if set(not_omitted_self).intersection(not_omitted_other): - raise TypeError('Addition of overlapping headers not defined') - - not_omitted_self.update(not_omitted_other) - return type(self)(**not_omitted_self) # pylint: disable=star-args - - def find_key(self): - """Find key based on header. - - .. todo:: Supports only "jwk" header parameter lookup. - - :returns: (Public) key found in the header. - :rtype: .JWK - - :raises acme.jose.errors.Error: if key could not be found - - """ - if self.jwk is None: - raise errors.Error('No key found') - return self.jwk - - @crit.decoder - def crit(unused_value): - # pylint: disable=missing-docstring,no-self-argument,no-self-use - raise errors.DeserializationError( - '"crit" is not supported, please subclass') - - # x5c does NOT use JOSE Base64 (4.1.6) - - @x5c.encoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - return [base64.b64encode(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] - - @x5c.decoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - try: - return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - base64.b64decode(cert))) for cert in value) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class Signature(json_util.JSONObjectWithFields): - """JWS Signature. - - :ivar combined: Combined Header (protected and unprotected, - :class:`Header`). - :ivar unicode protected: JWS protected header (Jose Base-64 decoded). - :ivar header: JWS Unprotected Header (:class:`Header`). - :ivar str signature: The signature. - - """ - header_cls = Header - - __slots__ = ('combined',) - protected = json_util.Field('protected', omitempty=True, default='') - header = json_util.Field( - 'header', omitempty=True, default=header_cls(), - decoder=header_cls.from_json) - signature = json_util.Field( - 'signature', decoder=json_util.decode_b64jose, - encoder=json_util.encode_b64jose) - - @protected.encoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - # wrong type guess (Signature, not bytes) | pylint: disable=no-member - return json_util.encode_b64jose(value.encode('utf-8')) - - @protected.decoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - return json_util.decode_b64jose(value).decode('utf-8') - - def __init__(self, **kwargs): - if 'combined' not in kwargs: - kwargs = self._with_combined(kwargs) - super(Signature, self).__init__(**kwargs) - assert self.combined.alg is not None - - @classmethod - def _with_combined(cls, kwargs): - assert 'combined' not in kwargs - header = kwargs.get('header', cls._fields['header'].default) - protected = kwargs.get('protected', cls._fields['protected'].default) - - if protected: - combined = header + cls.header_cls.json_loads(protected) - else: - combined = header - - kwargs['combined'] = combined - return kwargs - - @classmethod - def _msg(cls, protected, payload): - return (b64.b64encode(protected.encode('utf-8')) + b'.' + - b64.b64encode(payload)) - - def verify(self, payload, key=None): - """Verify. - - :param JWK key: Key used for verification. - - """ - key = self.combined.find_key() if key is None else key - return self.combined.alg.verify( - key=key.key, sig=self.signature, - msg=self._msg(self.protected, payload)) - - @classmethod - def sign(cls, payload, key, alg, include_jwk=True, - protect=frozenset(), **kwargs): - """Sign. - - :param JWK key: Key for signature. - - """ - assert isinstance(key, alg.kty) - - header_params = kwargs - header_params['alg'] = alg - if include_jwk: - header_params['jwk'] = key.public_key() - - assert set(header_params).issubset(cls.header_cls._fields) - assert protect.issubset(cls.header_cls._fields) - - protected_params = {} - for header in protect: - if header in header_params: - protected_params[header] = header_params.pop(header) - if protected_params: - # pylint: disable=star-args - protected = cls.header_cls(**protected_params).json_dumps() - else: - protected = '' - - header = cls.header_cls(**header_params) # pylint: disable=star-args - signature = alg.sign(key.key, cls._msg(protected, payload)) - - return cls(protected=protected, header=header, signature=signature) - - def fields_to_partial_json(self): - fields = super(Signature, self).fields_to_partial_json() - if not fields['header'].not_omitted(): - del fields['header'] - return fields - - @classmethod - def fields_from_json(cls, jobj): - fields = super(Signature, cls).fields_from_json(jobj) - fields_with_combined = cls._with_combined(fields) - if 'alg' not in fields_with_combined['combined'].not_omitted(): - raise errors.DeserializationError('alg not present') - return fields_with_combined - - -class JWS(json_util.JSONObjectWithFields): - """JSON Web Signature. - - :ivar str payload: JWS Payload. - :ivar str signature: JWS Signatures. - - """ - __slots__ = ('payload', 'signatures') - - signature_cls = Signature - - def verify(self, key=None): - """Verify.""" - return all(sig.verify(self.payload, key) for sig in self.signatures) - - @classmethod - def sign(cls, payload, **kwargs): - """Sign.""" - return cls(payload=payload, signatures=( - cls.signature_cls.sign(payload=payload, **kwargs),)) - - @property - def signature(self): - """Get a singleton signature. - - :rtype: `signature_cls` - - """ - assert len(self.signatures) == 1 - return self.signatures[0] - - def to_compact(self): - """Compact serialization. - - :rtype: bytes - - """ - assert len(self.signatures) == 1 - - assert 'alg' not in self.signature.header.not_omitted() - # ... it must be in protected - - return ( - b64.b64encode(self.signature.protected.encode('utf-8')) + - b'.' + - b64.b64encode(self.payload) + - b'.' + - b64.b64encode(self.signature.signature)) - - @classmethod - def from_compact(cls, compact): - """Compact deserialization. - - :param bytes compact: - - """ - try: - protected, payload, signature = compact.split(b'.') - except ValueError: - raise errors.DeserializationError( - 'Compact JWS serialization should comprise of exactly' - ' 3 dot-separated components') - - sig = cls.signature_cls( - protected=b64.b64decode(protected).decode('utf-8'), - signature=b64.b64decode(signature)) - return cls(payload=b64.b64decode(payload), signatures=(sig,)) - - def to_partial_json(self, flat=True): # pylint: disable=arguments-differ - assert self.signatures - payload = json_util.encode_b64jose(self.payload) - - if flat and len(self.signatures) == 1: - ret = self.signatures[0].to_partial_json() - ret['payload'] = payload - return ret - else: - return { - 'payload': payload, - 'signatures': self.signatures, - } - - @classmethod - def from_json(cls, jobj): - if 'signature' in jobj and 'signatures' in jobj: - raise errors.DeserializationError('Flat mixed with non-flat') - elif 'signature' in jobj: # flat - return cls(payload=json_util.decode_b64jose(jobj.pop('payload')), - signatures=(cls.signature_cls.from_json(jobj),)) - else: - return cls(payload=json_util.decode_b64jose(jobj['payload']), - signatures=tuple(cls.signature_cls.from_json(sig) - for sig in jobj['signatures'])) - - -class CLI(object): - """JWS CLI.""" - - @classmethod - def sign(cls, args): - """Sign.""" - key = args.alg.kty.load(args.key.read()) - args.key.close() - if args.protect is None: - args.protect = [] - if args.compact: - args.protect.append('alg') - - sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg, - protect=set(args.protect)) - - if args.compact: - six.print_(sig.to_compact().decode('utf-8')) - else: # JSON - six.print_(sig.json_dumps_pretty()) - - @classmethod - def verify(cls, args): - """Verify.""" - if args.compact: - sig = JWS.from_compact(sys.stdin.read().encode()) - else: # JSON - try: - sig = JWS.json_loads(sys.stdin.read()) - except errors.Error as error: - six.print_(error) - return -1 - - if args.key is not None: - assert args.kty is not None - key = args.kty.load(args.key.read()).public_key() - args.key.close() - else: - key = None - - sys.stdout.write(sig.payload) - return not sig.verify(key=key) - - @classmethod - def _alg_type(cls, arg): - return jwa.JWASignature.from_json(arg) - - @classmethod - def _header_type(cls, arg): - assert arg in Signature.header_cls._fields - return arg - - @classmethod - def _kty_type(cls, arg): - assert arg in jwk.JWK.TYPES - return jwk.JWK.TYPES[arg] - - @classmethod - def run(cls, args=sys.argv[1:]): - """Parse arguments and sign/verify.""" - parser = argparse.ArgumentParser() - parser.add_argument('--compact', action='store_true') - - subparsers = parser.add_subparsers() - parser_sign = subparsers.add_parser('sign') - parser_sign.set_defaults(func=cls.sign) - parser_sign.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=True) - parser_sign.add_argument( - '-a', '--alg', type=cls._alg_type, default=jwa.RS256) - parser_sign.add_argument( - '-p', '--protect', action='append', type=cls._header_type) - - parser_verify = subparsers.add_parser('verify') - parser_verify.set_defaults(func=cls.verify) - parser_verify.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=False) - parser_verify.add_argument( - '--kty', type=cls._kty_type, required=False) - - parsed = parser.parse_args(args) - return parsed.func(parsed) - - -if __name__ == '__main__': - exit(CLI.run()) # pragma: no cover diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py deleted file mode 100644 index ec91f6a1b..000000000 --- a/acme/acme/jose/jws_test.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Tests for acme.jose.jws.""" -import base64 -import unittest - -import mock -import OpenSSL - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk - - -CERT = test_util.load_comparable_cert('cert.pem') -KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) - - -class MediaTypeTest(unittest.TestCase): - """Tests for acme.jose.jws.MediaType.""" - - def test_decode(self): - from acme.jose.jws import MediaType - self.assertEqual('application/app', MediaType.decode('application/app')) - self.assertEqual('application/app', MediaType.decode('app')) - self.assertRaises( - errors.DeserializationError, MediaType.decode, 'app;foo') - - def test_encode(self): - from acme.jose.jws import MediaType - self.assertEqual('app', MediaType.encode('application/app')) - self.assertEqual('application/app;foo', - MediaType.encode('application/app;foo')) - - -class HeaderTest(unittest.TestCase): - """Tests for acme.jose.jws.Header.""" - - def setUp(self): - from acme.jose.jws import Header - self.header1 = Header(jwk='foo') - self.header2 = Header(jwk='bar') - self.crit = Header(crit=('a', 'b')) - self.empty = Header() - - def test_add_non_empty(self): - from acme.jose.jws import Header - self.assertEqual(Header(jwk='foo', crit=('a', 'b')), - self.header1 + self.crit) - - def test_add_empty(self): - self.assertEqual(self.header1, self.header1 + self.empty) - self.assertEqual(self.header1, self.empty + self.header1) - - def test_add_overlapping_error(self): - self.assertRaises(TypeError, self.header1.__add__, self.header2) - - def test_add_wrong_type_error(self): - self.assertRaises(TypeError, self.header1.__add__, 'xxx') - - def test_crit_decode_always_errors(self): - from acme.jose.jws import Header - self.assertRaises(errors.DeserializationError, Header.from_json, - {'crit': ['a', 'b']}) - - def test_x5c_decoding(self): - from acme.jose.jws import Header - header = Header(x5c=(CERT, CERT)) - jobj = header.to_partial_json() - cert_asn1 = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) - cert_b64 = base64.b64encode(cert_asn1) - self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) - self.assertEqual(header, Header.from_json(jobj)) - jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) - self.assertRaises(errors.DeserializationError, Header.from_json, jobj) - - def test_find_key(self): - self.assertEqual('foo', self.header1.find_key()) - self.assertEqual('bar', self.header2.find_key()) - self.assertRaises(errors.Error, self.crit.find_key) - - -class SignatureTest(unittest.TestCase): - """Tests for acme.jose.jws.Signature.""" - - def test_from_json(self): - from acme.jose.jws import Header - from acme.jose.jws import Signature - self.assertEqual( - Signature(signature=b'foo', header=Header(alg=jwa.RS256)), - Signature.from_json( - {'signature': 'Zm9v', 'header': {'alg': 'RS256'}})) - - def test_from_json_no_alg_error(self): - from acme.jose.jws import Signature - self.assertRaises(errors.DeserializationError, - Signature.from_json, {'signature': 'foo'}) - - -class JWSTest(unittest.TestCase): - """Tests for acme.jose.jws.JWS.""" - - def setUp(self): - self.privkey = KEY - self.pubkey = self.privkey.public_key() - - from acme.jose.jws import JWS - self.unprotected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256) - self.protected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['jwk', 'alg'])) - self.mixed = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['alg'])) - - def test_pubkey_jwk(self): - self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.protected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey) - - def test_sign_unprotected(self): - self.assertTrue(self.unprotected.verify()) - - def test_sign_protected(self): - self.assertTrue(self.protected.verify()) - - def test_sign_mixed(self): - self.assertTrue(self.mixed.verify()) - - def test_compact_lost_unprotected(self): - compact = self.mixed.to_compact() - self.assertEqual( - b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb' - b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA', - compact) - - from acme.jose.jws import JWS - mixed = JWS.from_compact(compact) - - self.assertNotEqual(self.mixed, mixed) - self.assertEqual( - set(['alg']), set(mixed.signature.combined.not_omitted())) - - def test_from_compact_missing_components(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.') - - def test_json_omitempty(self): - protected_jobj = self.protected.to_partial_json(flat=True) - unprotected_jobj = self.unprotected.to_partial_json(flat=True) - - self.assertTrue('protected' not in unprotected_jobj) - self.assertTrue('header' not in protected_jobj) - - unprotected_jobj['header'] = unprotected_jobj['header'].to_json() - - from acme.jose.jws import JWS - self.assertEqual(JWS.from_json(protected_jobj), self.protected) - self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected) - - def test_json_flat(self): - jobj_to = { - 'signature': json_util.encode_b64jose( - self.mixed.signature.signature), - 'payload': json_util.encode_b64jose(b'foo'), - 'header': self.mixed.signature.header, - 'protected': json_util.encode_b64jose( - self.mixed.signature.protected.encode('utf-8')), - } - jobj_from = jobj_to.copy() - jobj_from['header'] = jobj_from['header'].to_json() - - self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_json_not_flat(self): - jobj_to = { - 'signatures': (self.mixed.signature,), - 'payload': json_util.encode_b64jose(b'foo'), - } - jobj_from = jobj_to.copy() - jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()] - - self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_from_json_mixed_flat(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_json, - {'signatures': (), 'signature': 'foo'}) - - def test_from_json_hashable(self): - from acme.jose.jws import JWS - hash(JWS.from_json(self.mixed.to_json())) - - -class CLITest(unittest.TestCase): - - def setUp(self): - self.key_path = test_util.vector_path('rsa512_key.pem') - - def test_unverified(self): - from acme.jose.jws import CLI - with mock.patch('sys.stdin') as sin: - sin.read.return_value = '{"payload": "foo", "signature": "xxx"}' - with mock.patch('sys.stdout'): - self.assertEqual(-1, CLI.run(['verify'])) - - def test_json(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['sign', '-k', self.key_path, '-a', 'RS256', - '-p', 'jwk']) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run(['verify'])) - - def test_compact(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['--compact', 'sign', '-k', self.key_path]) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run([ - '--compact', 'verify', '--kty', 'RSA', - '-k', self.key_path])) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py deleted file mode 100644 index 26b7e0c5a..000000000 --- a/acme/acme/jose/util.py +++ /dev/null @@ -1,226 +0,0 @@ -"""JOSE utilities.""" -import collections - -from cryptography.hazmat.primitives.asymmetric import rsa -import OpenSSL -import six - - -class abstractclassmethod(classmethod): - # pylint: disable=invalid-name,too-few-public-methods - """Descriptor for an abstract classmethod. - - It augments the :mod:`abc` framework with an abstract - classmethod. This is implemented as :class:`abc.abstractclassmethod` - in the standard Python library starting with version 3.2. - - This particular implementation, allegedly based on Python 3.3 source - code, is stolen from - http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod. - - """ - __isabstractmethod__ = True - - def __init__(self, target): - target.__isabstractmethod__ = True - super(abstractclassmethod, self).__init__(target) - - -class ComparableX509(object): # pylint: disable=too-few-public-methods - """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. - - :ivar wrapped: Wrapped certificate or certificate request. - :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. - - """ - def __init__(self, wrapped): - assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( - wrapped, OpenSSL.crypto.X509Req) - self.wrapped = wrapped - - def __getattr__(self, name): - return getattr(self.wrapped, name) - - def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): - """Dumps the object into a buffer with the specified encoding. - - :param int filetype: The desired encoding. Should be one of - `OpenSSL.crypto.FILETYPE_ASN1`, - `OpenSSL.crypto.FILETYPE_PEM`, or - `OpenSSL.crypto.FILETYPE_TEXT`. - - :returns: Encoded X509 object. - :rtype: str - - """ - if isinstance(self.wrapped, OpenSSL.crypto.X509): - func = OpenSSL.crypto.dump_certificate - else: # assert in __init__ makes sure this is X509Req - func = OpenSSL.crypto.dump_certificate_request - return func(filetype, self.wrapped) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return NotImplemented - # pylint: disable=protected-access - return self._dump() == other._dump() - - def __hash__(self): - return hash((self.__class__, self._dump())) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) - - -class ComparableKey(object): # pylint: disable=too-few-public-methods - """Comparable wrapper for `cryptography` keys. - - See https://github.com/pyca/cryptography/issues/2122. - - """ - __hash__ = NotImplemented - - def __init__(self, wrapped): - self._wrapped = wrapped - - def __getattr__(self, name): - return getattr(self._wrapped, name) - - def __eq__(self, other): - # pylint: disable=protected-access - if (not isinstance(other, self.__class__) or - self._wrapped.__class__ is not other._wrapped.__class__): - return NotImplemented - elif hasattr(self._wrapped, 'private_numbers'): - return self.private_numbers() == other.private_numbers() - elif hasattr(self._wrapped, 'public_numbers'): - return self.public_numbers() == other.public_numbers() - else: - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) - - def public_key(self): - """Get wrapped public key.""" - return self.__class__(self._wrapped.public_key()) - - -class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods - """Wrapper for `cryptography` RSA keys. - - Wraps around: - - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey` - - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey` - - """ - - def __hash__(self): - # public_numbers() hasn't got stable hash! - # https://github.com/pyca/cryptography/issues/2143 - if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization): - priv = self.private_numbers() - pub = priv.public_numbers - return hash((self.__class__, priv.p, priv.q, priv.dmp1, - priv.dmq1, priv.iqmp, pub.n, pub.e)) - elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization): - pub = self.public_numbers() - return hash((self.__class__, pub.n, pub.e)) - - -class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=too-few-public-methods - """Immutable key to value mapping with attribute access.""" - - __slots__ = () - """Must be overridden in subclasses.""" - - def __init__(self, **kwargs): - if set(kwargs) != set(self.__slots__): - raise TypeError( - '__init__() takes exactly the following arguments: {0} ' - '({1} given)'.format(', '.join(self.__slots__), - ', '.join(kwargs) if kwargs else 'none')) - for slot in self.__slots__: - object.__setattr__(self, slot, kwargs.pop(slot)) - - def update(self, **kwargs): - """Return updated map.""" - items = dict(self) - items.update(kwargs) - return type(self)(**items) # pylint: disable=star-args - - def __getitem__(self, key): - try: - return getattr(self, key) - except AttributeError: - raise KeyError(key) - - def __iter__(self): - return iter(self.__slots__) - - def __len__(self): - return len(self.__slots__) - - def __hash__(self): - return hash(tuple(getattr(self, slot) for slot in self.__slots__)) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return '{0}({1})'.format(self.__class__.__name__, ', '.join( - '{0}={1!r}'.format(key, value) - for key, value in six.iteritems(self))) - - -class frozendict(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=invalid-name,too-few-public-methods - """Frozen dictionary.""" - __slots__ = ('_items', '_keys') - - def __init__(self, *args, **kwargs): - if kwargs and not args: - items = dict(kwargs) - elif len(args) == 1 and isinstance(args[0], collections.Mapping): - items = args[0] - else: - raise TypeError() - # TODO: support generators/iterators - - object.__setattr__(self, '_items', items) - object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items)))) - - def __getitem__(self, key): - return self._items[key] - - def __iter__(self): - return iter(self._keys) - - def __len__(self): - return len(self._items) - - def _sorted_items(self): - return tuple((key, self[key]) for key in self._keys) - - def __hash__(self): - return hash(self._sorted_items()) - - def __getattr__(self, name): - try: - return self._items[name] - except KeyError: - raise AttributeError(name) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format( - key, value) for key, value in self._sorted_items())) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py deleted file mode 100644 index 0038a6cc1..000000000 --- a/acme/acme/jose/util_test.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Tests for acme.jose.util.""" -import functools -import unittest - -import six - -from acme import test_util - - -class ComparableX509Test(unittest.TestCase): - """Tests for acme.jose.util.ComparableX509.""" - - def setUp(self): - # test_util.load_comparable_{csr,cert} return ComparableX509 - self.req1 = test_util.load_comparable_csr('csr.pem') - self.req2 = test_util.load_comparable_csr('csr.pem') - self.req_other = test_util.load_comparable_csr('csr-san.pem') - - self.cert1 = test_util.load_comparable_cert('cert.pem') - self.cert2 = test_util.load_comparable_cert('cert.pem') - self.cert_other = test_util.load_comparable_cert('cert-san.pem') - - def test_getattr_proxy(self): - self.assertTrue(self.cert1.has_expired()) - - def test_eq(self): - self.assertEqual(self.req1, self.req2) - self.assertEqual(self.cert1, self.cert2) - - def test_ne(self): - self.assertNotEqual(self.req1, self.req_other) - self.assertNotEqual(self.cert1, self.cert_other) - - def test_ne_wrong_types(self): - self.assertNotEqual(self.req1, 5) - self.assertNotEqual(self.cert1, 5) - - def test_hash(self): - self.assertEqual(hash(self.req1), hash(self.req2)) - self.assertNotEqual(hash(self.req1), hash(self.req_other)) - - self.assertEqual(hash(self.cert1), hash(self.cert2)) - self.assertNotEqual(hash(self.cert1), hash(self.cert_other)) - - def test_repr(self): - for x509 in self.req1, self.cert1: - self.assertEqual(repr(x509), - ''.format(x509.wrapped)) - - -class ComparableRSAKeyTest(unittest.TestCase): - """Tests for acme.jose.util.ComparableRSAKey.""" - - def setUp(self): - # test_utl.load_rsa_private_key return ComparableRSAKey - self.key = test_util.load_rsa_private_key('rsa256_key.pem') - self.key_same = test_util.load_rsa_private_key('rsa256_key.pem') - self.key2 = test_util.load_rsa_private_key('rsa512_key.pem') - - def test_getattr_proxy(self): - self.assertEqual(256, self.key.key_size) - - def test_eq(self): - self.assertEqual(self.key, self.key_same) - - def test_ne(self): - self.assertNotEqual(self.key, self.key2) - - def test_ne_different_types(self): - self.assertNotEqual(self.key, 5) - - def test_ne_not_wrapped(self): - # pylint: disable=protected-access - self.assertNotEqual(self.key, self.key_same._wrapped) - - def test_ne_no_serialization(self): - from acme.jose.util import ComparableRSAKey - self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5)) - - def test_hash(self): - self.assertTrue(isinstance(hash(self.key), int)) - self.assertEqual(hash(self.key), hash(self.key_same)) - self.assertNotEqual(hash(self.key), hash(self.key2)) - - def test_repr(self): - self.assertTrue(repr(self.key).startswith( - '=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', + # formerly known as acme.jose: + 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13', @@ -74,10 +76,5 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, - entry_points={ - 'console_scripts': [ - 'jws = acme.jose.jws:CLI.run', - ], - }, test_suite='acme', ) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 2405110c5..ca667465c 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -5,11 +5,10 @@ import sys import unittest import augeas +import josepy as jose import mock import zope.component -from acme import jose - from certbot.display import util as display_util from certbot.plugins import common diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index af951aa6a..4155944bd 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -6,7 +6,8 @@ import re import shutil import tarfile -from acme import jose +import josepy as jose + from acme import test_util from certbot import constants diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 6e1b0d8ff..7b32d8e82 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -5,11 +5,10 @@ import pkg_resources import tempfile import unittest +import josepy as jose import mock import zope.component -from acme import jose - from certbot import configuration from certbot.tests import util as test_util diff --git a/certbot/account.py b/certbot/account.py index 389f96791..41e980097 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,13 +7,13 @@ import shutil import socket from cryptography.hazmat.primitives import serialization +import josepy as jose import pyrfc3339 import pytz import six import zope.component from acme import fields as acme_fields -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/achallenges.py b/certbot/achallenges.py index f39bb4cec..6535a6b63 100644 --- a/certbot/achallenges.py +++ b/certbot/achallenges.py @@ -19,8 +19,9 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging +import josepy as jose + from acme import challenges -from acme import jose logger = logging.getLogger(__name__) diff --git a/certbot/client.py b/certbot/client.py index ed70fda71..b735421f5 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,13 +5,13 @@ import platform from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa +import josepy as jose import OpenSSL import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors -from acme import jose from acme import messages import certbot diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 112ef7c85..3ae16529d 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,9 +14,9 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 +import josepy as jose from acme import crypto_util as acme_crypto_util -from acme import jose from certbot import errors from certbot import interfaces @@ -368,7 +368,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in - `acme.jose.ComparableX509`). + :class:`josepy.util.ComparableX509`). """ # XXX: returns empty string when no chain is available, which diff --git a/certbot/main.py b/certbot/main.py index 72af7fbba..1c6432fd9 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -6,9 +6,9 @@ import os import sys import configobj +import josepy as jose import zope.component -from acme import jose from acme import errors as acme_errors import certbot diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 420d15679..002d2f225 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -9,7 +9,7 @@ import OpenSSL import pkg_resources import zope.interface -from acme.jose import util as jose_util +from josepy import util as jose_util from certbot import constants from certbot import crypto_util diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 8ce68bbb5..1a1ca7dcb 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -5,11 +5,11 @@ import shutil import tempfile import unittest +import josepy as jose import mock import OpenSSL from acme import challenges -from acme import jose from certbot import achallenges from certbot import crypto_util diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py index d8cd29404..54b656b20 100644 --- a/certbot/plugins/dns_test_common.py +++ b/certbot/plugins/dns_test_common.py @@ -3,10 +3,10 @@ import os import configobj +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot.tests import acme_util diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py index f9c5735e8..a221cf1bf 100644 --- a/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" +import josepy as jose import mock -from acme import jose from requests.exceptions import HTTPError, RequestException from certbot import errors diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 1ae731e42..5227bc59e 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -3,11 +3,11 @@ import argparse import socket import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 92160bdfa..36e2ffba6 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -10,11 +10,11 @@ import stat import tempfile import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7245ad6a1..8ebda56af 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -6,10 +6,10 @@ import shutil import stat import unittest +import josepy as jose import mock import pytz -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index f0549666a..53a2f214a 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -1,10 +1,10 @@ """ACME utilities for testing.""" import datetime +import josepy as jose import six from acme import challenges -from acme import jose from acme import messages from certbot import auth_handler diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 09c4a50ca..204f46323 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,11 +4,11 @@ import shutil import tempfile import unittest +import josepy as jose import OpenSSL import mock from acme import errors as acme_errors -from acme import jose from certbot import account from certbot import errors diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index cb0fb32e3..57d82f839 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,10 @@ import os import sys import unittest +import josepy as jose import mock import zope.component -from acme import jose from acme import messages from certbot import account diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 1f690df26..04b71dcc7 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,11 +11,10 @@ import unittest import datetime import pytz +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import account from certbot import cli from certbot import constants diff --git a/certbot/tests/util.py b/certbot/tests/util.py index c43b44522..ddd4a1aec 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -14,11 +14,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import mock import OpenSSL +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import constants from certbot import interfaces from certbot import storage diff --git a/tools/deactivate.py b/tools/deactivate.py index 5facc8436..d43b84552 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -18,10 +18,10 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization +import josepy as jose from acme import client as acme_client from acme import errors as acme_errors -from acme import jose from acme import messages DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') From 0e92d4ea98e44bfc9f1797269c6998195dea5f8a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 11 Dec 2017 21:50:56 +0200 Subject: [PATCH 248/631] Parse variables without whitespace separator correctly in CentOS family of distributions (#5318) --- certbot-apache/certbot_apache/apache_util.py | 4 ++++ certbot-apache/certbot_apache/tests/centos_test.py | 2 ++ .../tests/testdata/centos7_apache/apache/sysconfig/httpd | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index b4a24f137..f03c9da87 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -93,4 +93,8 @@ def parse_define_file(filepath, varname): if v == "-D" and len(a_opts) >= i+2: var_parts = a_opts[i+1].partition("=") return_vars[var_parts[0]] = var_parts[2] + elif len(v) > 2 and v.startswith("-D"): + # Found var with no whitespace separator + var_parts = v[2:].partition("=") + return_vars[var_parts[0]] = var_parts[2] return return_vars diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 7ca47a4d5..d7a2a2fd9 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -118,6 +118,8 @@ class MultipleVhostsTestCentOS(util.ApacheTest): self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) self.assertTrue("mock_value" in self.config.parser.variables.keys()) self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) + self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) + self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd index 0bf6b176c..4bcb300c2 100644 --- a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -14,7 +14,7 @@ # To pass additional options (for instance, -D definitions) to the # httpd binary at startup, set OPTIONS here. # -OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE -DMOCK_NOSEP -DNOSEP_TWO=NOSEP_VAL" # # This setting ensures the httpd process is started in the "C" locale From 1b6005cc61f8b977af1bc5513994b4815280dd74 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Dec 2017 18:15:42 -0800 Subject: [PATCH 249/631] Pin josepy in letsencrypt-auto (#5321) * pin josepy in le-auto * Put pinned versions in sorted order --- letsencrypt-auto-source/letsencrypt-auto | 11 +++++++---- .../pieces/dependency-requirements.txt | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8d2e8a6b6..93e3e7b83 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -983,9 +983,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1069,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index dec7ae7d0..0e2cec984 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -105,9 +105,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -184,7 +191,3 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 From a1aea021e7a587ea9396b2ebbfcfaec10411ab86 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 18 Dec 2017 12:31:36 -0800 Subject: [PATCH 250/631] Pin dependencies in oldest tests (#5316) * Add tools/merge_requirements.py * Revert "Fix oldest tests by pinning Google DNS deps (#5000)" This reverts commit f68fba2be2fc342dd72deaaf048ab79e5a8fc2be. * Add tools/oldest_constraints.txt * Remove oldest constraints from tox.ini * Rename dev constraints file * Update tools/pip_install.sh * Update install_and_test.sh * Fix pip_install.sh * Don't cat when you can cp * Add ng-httpsclient to dev constraints for oldest tests * Bump tested setuptools version * Update dev_constraints comment * Better document oldest dependencies * test against oldest versions we say we require * Update dev constraints * Properly handle empty lines * Update constraints gen in pip_install * Remove duplicated zope.component * Reduce pyasn1-modules dependency * Remove blank line * pin back google-api-python-client * pin back uritemplate * pin josepy for oldest tests * Undo changes to install_and_test.sh * Update install_and_test.sh description * use split instead of partition --- ...ip_constraints.txt => dev_constraints.txt} | 25 ++++---- tools/install_and_test.sh | 5 +- tools/merge_requirements.py | 61 +++++++++++++++++++ tools/oldest_constraints.txt | 51 ++++++++++++++++ tools/pip_install.sh | 31 ++++++---- tox.ini | 36 +---------- 6 files changed, 151 insertions(+), 58 deletions(-) rename tools/{pip_constraints.txt => dev_constraints.txt} (71%) create mode 100755 tools/merge_requirements.py create mode 100644 tools/oldest_constraints.txt diff --git a/tools/pip_constraints.txt b/tools/dev_constraints.txt similarity index 71% rename from tools/pip_constraints.txt rename to tools/dev_constraints.txt index cacec37d6..afc362ff8 100644 --- a/tools/pip_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,16 +1,15 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. We should avoid listing packages in -# both places because if both files are used as constraints for the same pip -# invocation, some constraints may be ignored due to pip's lack of dependency -# resolution. +# letsencrypt-auto's requirements file. alabaster==0.7.10 apipkg==1.4 +asn1crypto==0.22.0 astroid==1.3.5 +attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.8.1 +cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -19,7 +18,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.6.4 +google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -27,20 +26,22 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 +josepy==1.0.1 +logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -oauth2client==4.1.2 +ndg-httpsclient==0.3.2 +oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.3.7 -pyasn1-modules==0.1.5 +pyasn1==0.1.9 +pyasn1-modules==0.0.10 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -48,7 +49,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.12 +python-digitalocean==1.11 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -65,6 +66,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==3.0.0 +uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index d57f0974e..25b6d548a 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,8 +2,9 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.sh for more information on the versions pinned +# to. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py new file mode 100755 index 000000000..c8fb95351 --- /dev/null +++ b/tools/merge_requirements.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""Merges multiple Python requirements files into one file. + +Requirements files specified later take precedence over earlier ones. Only +simple SomeProject==1.2.3 format is currently supported. + +""" + +from __future__ import print_function + +import sys + + +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: mapping from a project to its pinned version + :rtype: dict + + """ + d = {} + with open(file_path) as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + project, version = line.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(line)) + d[project] = version + return d + + +def print_requirements(requirements): + """Prints requirements to stdout. + + :param dict requirements: mapping from a project to its pinned version + + """ + print('\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items()))) + + +def merge_requirements_files(*files): + """Merges multiple requirements files together and prints the result. + + Requirement files specified later in the list take precedence over earlier + files. + + :param tuple files: paths to requirements files + + """ + d = {} + for f in files: + d.update(read_file(f)) + print_requirements(d) + + +if __name__ == '__main__': + merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt new file mode 100644 index 000000000..de2b83ad8 --- /dev/null +++ b/tools/oldest_constraints.txt @@ -0,0 +1,51 @@ +# This file contains the oldest versions of our dependencies we say we require +# in our packages or versions we need to support to maintain compatibility with +# the versions included in the various Linux distros where we are packaged. + +# CentOS/RHEL 7 EPEL constraints +cffi==1.6.0 +chardet==2.2.1 +configobj==4.7.2 +ipaddress==1.0.16 +mock==1.0.1 +ndg-httpsclient==0.3.2 +ply==3.4 +pyasn1==0.1.9 +pycparser==2.14 +pyOpenSSL==0.13.1 +pyparsing==1.5.6 +pyRFC3339==1.0 +python-augeas==0.5.0 +six==1.9.0 +# setuptools 0.9.8 is the actual version packaged, but some other dependencies +# in this file require setuptools>=1.0 and there are no relevant changes for us +# between these versions. +setuptools==1.0.0 +urllib3==1.10.2 +zope.component==4.1.0 +zope.event==4.0.3 +zope.interface==4.0.5 + +# Debian Jessie Backports constraints +PyICU==1.8 +colorama==0.3.2 +enum34==1.0.3 +html5lib==0.999 +idna==2.0 +pbr==1.8.0 +pytz==2012rc0 + +# Our setup.py constraints +cloudflare==1.5.1 +cryptography==1.2.0 +google-api-python-client==1.5 +oauth2client==2.0 +parsedatetime==1.3 +pyparsing==1.5.5 +python-digitalocean==1.11 +requests[security]==2.4.1 + +# Ubuntu Xenial constraints +ConfigArgParse==0.10.0 +funcsigs==0.4 +zope.hookable==4.0.4 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index fafd58e54..d2aae4a43 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,17 +1,26 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions +#!/bin/bash -e +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt and +# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's +# requirements file and tools/dev_constraints.txt is used. The other file +# always takes precedence over tools/dev_constraints.txt. # get the root of the Certbot repo -my_path=$("$(dirname $0)/readlink.py" $0) -repo_root=$(dirname $(dirname $my_path)) -requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" -certbot_auto_constraints=$(mktemp) -trap "rm -f $certbot_auto_constraints" EXIT -# extract pinned requirements without hashes -sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints -dev_constraints="$(dirname $my_path)/pip_constraints.txt" +tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) +dev_constraints="$tools_dir/dev_constraints.txt" +merge_reqs="$tools_dir/merge_requirements.py" +test_constraints=$(mktemp) +trap "rm -f $test_constraints" EXIT + +if [ "$CERTBOT_OLDEST" = 1 ]; then + cp "$tools_dir/oldest_constraints.txt" "$test_constraints" +else + repo_root=$(dirname "$tools_dir") + certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" + sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" +fi set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" +pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" diff --git a/tox.ini b/tox.ini index bb421daa5..6ebf681ed 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,8 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# before the script moves on to the next package. All dependencies are pinned +# to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -82,36 +81,7 @@ commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_NO_PIN=1 -deps = - PyOpenSSL==0.13 - cffi==1.5.2 - configargparse==0.10.0 - configargparse==0.10.0 - configobj==4.7.2 - cryptography==1.2.3 - enum34==0.9.23 - google-api-python-client==1.5 - idna==2.0 - ipaddress==1.0.16 - mock==1.0.1 - ndg-httpsclient==0.3.2 - oauth2client==2.0 - parsedatetime==1.4 - pyasn1-modules==0.0.5 - pyasn1==0.1.9 - pyparsing==1.5.6 - pyrfc3339==1.0 - pytest==3.2.5 - python-augeas==0.4.1 - pytz==2012c - requests[security]==2.6.0 - setuptools==0.9.8 - six==1.9.0 - urllib3==1.10 - zope.component==4.0.2 - zope.event==4.0.1 - zope.interface==4.0.5 + CERTBOT_OLDEST=1 [testenv:py27_install] basepython = python2.7 From d6b11fea722ab71584a2bd50cb731a5f67b0e375 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 Dec 2017 16:16:45 -0800 Subject: [PATCH 251/631] More pip dependency resolution workarounds (#5339) * remove pyopenssl and six deps * remove outdated tox.ini dep requirement --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ee108c514..ce505a62e 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,9 @@ readme = read_file(os.path.join(here, 'README.rst')) changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] -# Please update tox.ini when modifying dependency version requirements -# This package relies on requests, however, it isn't specified here to avoid -# masking the more specific request requirements in acme. See -# https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL, requests, and six, however, it isn't +# specified here to avoid masking the more specific request requirements in +# acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but @@ -44,13 +43,11 @@ install_requires = [ 'cryptography>=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT - 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', - 'six', 'zope.component', 'zope.interface', ] From ed2168aaa8c8a7e1bef449e60167b53d501d173a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 21 Dec 2017 16:55:21 -0800 Subject: [PATCH 252/631] Fix auto_tests on systems with new bootstrappers (#5348) --- letsencrypt-auto-source/tests/auto_test.py | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 2fa03105d..156466c82 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -30,6 +30,10 @@ sys.path.insert(0, dirname(tests_dir())) from build import build as build_le_auto +BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' +"""Name of the file where certbot-auto saves its bootstrap version.""" + + class RequestHandler(BaseHTTPRequestHandler): """An HTTPS request handler which is quiet and serves a specific folder.""" @@ -296,17 +300,31 @@ class AutoTests(TestCase): def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" - with temp_paths() as (le_auto_path, venv_dir): - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} + with serving(resources) as base_url: + pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') + with temp_paths() as (le_auto_path, venv_dir): + install_le_auto(self.NEW_LE_AUTO, le_auto_path) + + # Create venv saving the correct bootstrap script version + out, err = run_le_auto(le_auto_path, venv_dir, base_url, + PIP_FIND_LINKS=pip_find_links) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) + with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: + bootstrap_version = f.read() + + # Create a new venv with an old letsencrypt version + with temp_paths() as (le_auto_path, venv_dir): venv_bin = join(venv_dir, 'bin') makedirs(venv_bin) set_le_script_version(venv_dir, '0.0.1') + with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: + f.write(bootstrap_version) install_le_auto(self.NEW_LE_AUTO, le_auto_path) - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) From 5388842e5b3868e29caf545fb771a23e7fce4143 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Jan 2018 17:49:22 -0800 Subject: [PATCH 253/631] Fix pytest on macOS in Travis (#5360) * Add tools/pytest.sh * pass TRAVIS through in tox.ini * Use tools/pytest.sh to run pytest * Add quiet to pytest.ini * ignore pytest cache --- .gitignore | 3 +++ pytest.ini | 2 ++ tools/install_and_test.sh | 2 +- tools/pytest.sh | 15 +++++++++++++++ tox.cover.sh | 3 ++- tox.ini | 9 +++++++++ 6 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100755 tools/pytest.sh diff --git a/.gitignore b/.gitignore index b63e40d1c..e018cf938 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ tests/letstest/*.pem tests/letstest/venv/ .venv + +# pytest cache +/.cache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..b64550cb7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --quiet diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 25b6d548a..0d39e0594 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -23,5 +23,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - pytest --numprocesses auto --quiet --pyargs $pkg + "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tools/pytest.sh b/tools/pytest.sh new file mode 100755 index 000000000..8e3619d5d --- /dev/null +++ b/tools/pytest.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Runs pytest with the provided arguments, adding --numprocesses to the command +# line. This argument is set to "auto" if the environmnent variable TRAVIS is +# not set, otherwise, it is set to 2. This works around +# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis +# environnment provides two cores. See +# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. + +if ${TRAVIS:-false}; then + NUMPROCESSES="2" +else + NUMPROCESSES="auto" +fi + +pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index 2b5a3cf19..bc0e5a8bf 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,7 +51,8 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest="$(dirname $0)/tools/pytest.sh" + "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 6ebf681ed..20f5cda32 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,7 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 +passenv = TRAVIS [testenv] commands = @@ -69,12 +70,16 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 +passenv = + {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 +passenv = + {[testenv]passenv} [testenv:py27-oldest] commands = @@ -82,6 +87,8 @@ commands = setenv = {[testenv]setenv} CERTBOT_OLDEST=1 +passenv = + {[testenv]passenv} [testenv:py27_install] basepython = python2.7 @@ -93,6 +100,8 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh +passenv = + {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187) From a7d00ee21b454115fc0ce831b13f7902d4b62c37 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Jan 2018 13:59:29 -0800 Subject: [PATCH 254/631] print as a string (#5359) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e9d4e36d4..8af474c5e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -182,7 +182,7 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, vhost.names) + vhost.filep, ", ".join(vhost.names)) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, From a3a66cd25d8340e982481e7adf4a521c09f0f35e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 00:36:16 +0200 Subject: [PATCH 255/631] Use apache2ctl modules for Gentoo systems. (#5349) * Do not call Apache binary for module reset in cleanup() * Use apache2ctl modules for Gentoo --- .../certbot_apache/override_gentoo.py | 8 +++ .../certbot_apache/tests/gentoo_test.py | 49 +++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index d4d4e96b9..92f1d4a20 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,6 +49,7 @@ class GentooParser(parser.ApacheParser): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() + self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -56,3 +57,10 @@ class GentooParser(parser.ApacheParser): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] + + def update_modules(self): + """Get loaded modules from httpd process, and add them to DOM""" + mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") + for mod in matches: + self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index 0f2b96818..cfbaffac7 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,6 +2,8 @@ import os import unittest +import mock + from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -46,9 +48,10 @@ class MultipleVhostsTestGentoo(util.ApacheTest): config_root=config_root, vhost_root=vhost_root) - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -78,9 +81,47 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - self.config.parser.update_runtime_variables() + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) + @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") + def test_no_binary_configdump(self, mock_subprocess): + """Make sure we don't call binary dumps other than modules from Apache + as this is not supported in Gentoo currently""" + + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertFalse(mock_subprocess.called) + + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertTrue(mock_subprocess.called) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_opportunistic_httpd_runtime_parsing(self, mock_get): + mod_val = ( + 'Loaded Modules:\n' + ' mock_module (static)\n' + ' another_module (static)\n' + ) + def mock_get_cfg(command): + """Mock httpd process stdout""" + if command == ['apache2ctl', 'modules']: + return mod_val + mock_get.side_effect = mock_get_cfg + self.config.parser.modules = set() + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("gentoo", "123") + self.config.parser.update_runtime_variables() + + self.assertEquals(mock_get.call_count, 1) + self.assertEquals(len(self.config.parser.modules), 4) + self.assertTrue("mod_another.c" in self.config.parser.modules) + if __name__ == "__main__": unittest.main() # pragma: no cover From a1713c0b79b99108ae3a1233cb3e3dc3bef2908a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 21:08:38 +0200 Subject: [PATCH 256/631] Broader git ignore for pytest cache files (#5361) Make gitignore take pytest cache directories in to account, even if they reside in subdirectories. If pytest is run for a certain module, ie. `pytest certbot-apache` the cache directory is created under `certbot-apache` directory. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e018cf938..a01d2e1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ tests/letstest/venv/ .venv # pytest cache -/.cache +.cache From 18f6deada8dca329c1a797bcf5b88dbdcbd18cf7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Jan 2018 19:27:00 -0800 Subject: [PATCH 257/631] Fix letsencrypt-auto name and long forms of -n (#5375) --- letsencrypt-auto-source/letsencrypt-auto | 8 +++++--- letsencrypt-auto-source/letsencrypt-auto.template | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 93e3e7b83..46cb51822 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 4eef10c80..861ef0a6e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi From 8585cdd86190c5da753cae1a691e66e586fd2bde Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 8 Jan 2018 13:57:04 -0800 Subject: [PATCH 258/631] Deprecate Python2.6 by using Python3 on CentOS/RHEL 6 (#5329) * If there's no python or there's only python2.6 on red hat systems, install python3 * Always check for python2.6 * address style, documentation, nits * factor out all initialization code * fix up python version return value when no python installed * add no python error and exit * document DeterminePythonVersion parameters * build letsencrypt-auto * close brace * build leauto * fix syntax errors * set USE_PYTHON_3 for all cases * rip out NOCRASH * replace NOCRASH, update LE_PYTHON set logic * use built-in venv for py3 * switch to LE_PYTHON not affecting bootstrap selection and not overwriting LE_PYTHON * python3ify fetch.py * get fetch.py working with python2 and 3 * don't verify server certificates in fetch.py HttpsGetter * Use SSLContext and an environment variable so that our tests continue to never verify server certificates. * typo * build * remove commented out code * address review comments * add documentation for YES_FLAG and QUIET_FLAG * Add tests to centos6 Dockerfile to make sure we install python3 if and only if appropriate to do so. --- letsencrypt-auto-source/Dockerfile.centos6 | 3 +- letsencrypt-auto-source/letsencrypt-auto | 205 +++++++++++++----- .../letsencrypt-auto.template | 67 ++++-- .../pieces/bootstrappers/rpm_common.sh | 75 +------ .../pieces/bootstrappers/rpm_common_base.sh | 78 +++++++ .../pieces/bootstrappers/rpm_python3.sh | 23 ++ letsencrypt-auto-source/pieces/fetch.py | 34 ++- letsencrypt-auto-source/tests/auto_test.py | 2 + .../tests/centos6_tests.sh | 65 ++++++ 9 files changed, 405 insertions(+), 147 deletions(-) create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh create mode 100644 letsencrypt-auto-source/tests/centos6_tests.sh diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 8c1a4b353..47eb48f50 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -33,4 +33,5 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh +CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 46cb51822..f1361d8ea 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,15 +246,29 @@ DeprecationBootstrap() { fi } - +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + if [ -n "$USE_PYTHON_3" ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi export LE_PYTHON @@ -386,23 +400,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -410,15 +420,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -430,11 +440,17 @@ BootstrapRpmCommon() { /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -446,10 +462,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -457,9 +502,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -470,8 +514,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -479,16 +522,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -717,11 +775,24 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + export LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -860,10 +931,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1358,17 +1437,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1390,8 +1474,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1413,7 +1500,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1421,13 +1508,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in iter(metadata['releases'].keys()) if re.match('^[0-9.]+$', r))) @@ -1444,7 +1531,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1459,6 +1546,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def create_CERT_NONE_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 861ef0a6e..f4c1b202f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,15 +246,29 @@ DeprecationBootstrap() { fi } - +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + if [ -n "$USE_PYTHON_3" ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi export LE_PYTHON @@ -267,7 +281,9 @@ DeterminePythonVersion() { } {{ bootstrappers/deb_common.sh }} +{{ bootstrappers/rpm_common_base.sh }} {{ bootstrappers/rpm_common.sh }} +{{ bootstrappers/rpm_python3.sh }} {{ bootstrappers/suse_common.sh }} {{ bootstrappers/arch_common.sh }} {{ bootstrappers/gentoo_common.sh }} @@ -298,11 +314,24 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + export LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -441,10 +470,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 5b120a9e6..80d55a393 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -7,61 +7,13 @@ BootstrapRpmCommon() { # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) + # - CentOS 6 - if type dnf 2>/dev/null - then - tool=dnf - elif type yum 2>/dev/null - then - tool=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! $tool list *virtualenv >/dev/null 2>&1; then - echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then - error "Enable the EPEL repository and try running Certbot again." - exit 1 - fi - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Enabling the EPEL repository in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." - sleep 1s - fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then - error "Could not enable EPEL. Aborting bootstrap!" - exit 1 - fi - fi - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " + InitializeRPMCommonBase # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then - pkgs="$pkgs - python + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -69,9 +21,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -82,8 +33,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -91,14 +41,5 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi + BootstrapRpmCommonBase "$python_pkgs" } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh new file mode 100644 index 000000000..d7a9f3133 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh @@ -0,0 +1,78 @@ +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. + +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { + if type dnf 2>/dev/null + then + TOOL=dnf + elif type yum 2>/dev/null + then + TOOL=yum + + else + error "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + if [ "$QUIET" = 1 ]; then + QUIET_FLAG='--quiet' + fi + + if ! $TOOL list *virtualenv >/dev/null 2>&1; then + echo "To use Certbot, packages from the EPEL repository need to be installed." + if ! $TOOL list epel-release >/dev/null 2>&1; then + error "Enable the EPEL repository and try running Certbot again." + exit 1 + fi + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Enabling the EPEL repository in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + sleep 1s + fi + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then + error "Could not enable EPEL. Aborting bootstrap!" + exit 1 + fi + fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice + + pkgs=" + gcc + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh new file mode 100644 index 000000000..b011a7235 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh @@ -0,0 +1,23 @@ +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" + exit 1 + fi + + BootstrapRpmCommonBase "$python_pkgs" +} diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index 8f34351c9..ae72a299b 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -11,17 +11,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -43,8 +48,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -66,7 +74,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -74,13 +82,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in iter(metadata['releases'].keys()) if re.match('^[0-9.]+$', r))) @@ -97,7 +105,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -112,6 +120,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def create_CERT_NONE_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 156466c82..d187452a1 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -202,6 +202,7 @@ LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB -----END PUBLIC KEY-----""", + NO_CERT_VERIFY='1', **kwargs) env.update(d) return out_and_err( @@ -349,6 +350,7 @@ class AutoTests(TestCase): self.assertTrue("Couldn't verify signature of downloaded " "certbot-auto." in exc.output) else: + print(out) self.fail('Signature check on certbot-auto erroneously passed.') def test_pip_failure(self): diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh new file mode 100644 index 000000000..e3ebbaec5 --- /dev/null +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Start by making sure your system is up-to-date: +yum update > /dev/null +yum install -y centos-release-scl > /dev/null +yum install -y python27 > /dev/null 2> /dev/null + +# we're going to modify env variables, so do this in a subshell +( +source /opt/rh/python27/enable + +# ensure python 3 isn't installed +python3 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "Python3 is already installed." + exit 1 +fi + +# ensure python2.7 is available +python2.7 --version 2> /dev/null +RESULT=$? +if [ $RESULT -ne 0 ]; then + error "Python3 is not available." + exit 1 +fi + +# bootstrap, but don't install python 3. +certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null + +# ensure python 3 isn't installed +python3 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "letsencrypt-auto installed Python3 even though Python2.7 is present." + exit 1 +fi + +echo "" +echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." +) + +# ensure python2.7 isn't available +python2.7 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "Python2.7 is still available." + exit 1 +fi + +# bootstrap, this time installing python3 +certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null + +# ensure python 3 is installed +python3 --version > /dev/null +RESULT=$? +if [ $RESULT -ne 0 ]; then + error "letsencrypt-auto failed to install Python3 when only Python2.6 is present." + exit 1 +fi + +echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." +echo "" + +# test using python3 +pytest -v -s certbot/letsencrypt-auto-source/tests From 24ddc65cd4765e82067ab5fd963ffe81348e1f02 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Jan 2018 17:02:20 -0800 Subject: [PATCH 259/631] Allow non-interactive revocation without deleting certificates (#5386) * Add --delete-after-revoke flags * Use delete_after_revoke value * Add delete_after_revoke unit tests * Add integration tests for delete-after-revoke. --- certbot/cli.py | 12 ++++++++++ certbot/constants.py | 1 + certbot/main.py | 8 ++++--- certbot/tests/cli_test.py | 14 +++++++++++ certbot/tests/main_test.py | 46 ++++++++++++++++++++++++++++-------- tests/boulder-integration.sh | 9 +++++-- 6 files changed, 75 insertions(+), 15 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 622462278..f0fa7eb7e 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,6 +1220,18 @@ def _create_subparsers(helpful): key=constants.REVOCATION_REASONS.get)), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") + helpful.add("revoke", + "--delete-after-revoke", action="store_true", + default=flag_default("delete_after_revoke"), + help="Delete certificates after revoking them.") + helpful.add("revoke", + "--no-delete-after-revoke", action="store_false", + dest="delete_after_revoke", + default=flag_default("delete_after_revoke"), + help="Do not delete certificates after revoking them. This " + "option should be used with caution because the 'renew' " + "subcommand will attempt to renew undeleted revoked " + "certificates.") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), diff --git a/certbot/constants.py b/certbot/constants.py index 0ac82dafe..a6878824b 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -71,6 +71,7 @@ CLI_DEFAULTS = dict( user_agent_comment=None, csr=None, reason=0, + delete_after_revoke=None, rollback_checkpoints=1, init=False, prepare=False, diff --git a/certbot/main.py b/certbot/main.py index 1c6432fd9..e25e030aa 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -536,9 +536,11 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b display = zope.component.getUtility(interfaces.IDisplay) reporter_util = zope.component.getUtility(interfaces.IReporter) - msg = ("Would you like to delete the cert(s) you just revoked?") - attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", - force_interactive=True, default=True) + attempt_deletion = config.delete_after_revoke + if attempt_deletion is None: + msg = ("Would you like to delete the cert(s) you just revoked?") + attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", + force_interactive=True, default=True) if not attempt_deletion: reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 2fce412e2..c5935d722 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -164,6 +164,8 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue("--cert-path" in out) self.assertTrue("--key-path" in out) self.assertTrue("--reason" in out) + self.assertTrue("--delete-after-revoke" in out) + self.assertTrue("--no-delete-after-revoke" in out) out = self._help_output(['-h', 'config_changes']) self.assertTrue("--cert-path" not in out) @@ -412,6 +414,18 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_no_directory_hooks_unset(self): self.assertTrue(self.parse([]).directory_hooks) + def test_delete_after_revoke(self): + namespace = self.parse(["--delete-after-revoke"]) + self.assertTrue(namespace.delete_after_revoke) + + def test_delete_after_revoke_default(self): + namespace = self.parse([]) + self.assertEqual(namespace.delete_after_revoke, None) + + def test_no_delete_after_revoke(self): + namespace = self.parse(["--no-delete-after-revoke"]) + self.assertFalse(namespace.delete_after_revoke) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 04b71dcc7..b1d58542f 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -298,25 +298,29 @@ class RevokeTest(test_util.TempDirTestCase): self._call() self.assertFalse(mock_delete.called) -class DeleteIfAppropriateTest(unittest.TestCase): +class DeleteIfAppropriateTest(test_util.ConfigTestCase): """Tests for certbot.main._delete_if_appropriate """ - def setUp(self): - self.config = mock.Mock() - self.config.namespace = mock.Mock() - self.config.namespace.noninteractive_mode = False - def _call(self, mock_config): from certbot.main import _delete_if_appropriate _delete_if_appropriate(mock_config) - @mock.patch('certbot.cert_manager.delete') + def _test_delete_opt_out_common(self, mock_get_utility): + with mock.patch('certbot.cert_manager.delete') as mock_delete: + self._call(self.config) + mock_delete.assert_not_called() + self.assertTrue(mock_get_utility().add_message.called) + @test_util.patch_get_utility() - def test_delete_opt_out(self, mock_get_utility, mock_delete): + def test_delete_flag_opt_out(self, mock_get_utility): + self.config.delete_after_revoke = False + self._test_delete_opt_out_common(mock_get_utility) + + @test_util.patch_get_utility() + def test_delete_prompt_opt_out(self, mock_get_utility): util_mock = mock_get_utility() util_mock.yesno.return_value = False - self._call(self.config) - mock_delete.assert_not_called() + self._test_delete_opt_out_common(mock_get_utility) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -397,6 +401,28 @@ class DeleteIfAppropriateTest(unittest.TestCase): self._call(config) self.assertEqual(mock_delete.call_count, 1) + # pylint: disable=too-many-arguments + @mock.patch('certbot.storage.renewal_file_for_certname') + @mock.patch('certbot.cert_manager.match_and_check_overlaps') + @mock.patch('certbot.storage.full_archive_path') + @mock.patch('certbot.cert_manager.cert_path_to_lineage') + @mock.patch('certbot.cert_manager.delete') + @test_util.patch_get_utility() + def test_opt_in_deletion(self, mock_get_utility, mock_delete, + mock_cert_path_to_lineage, mock_full_archive_dir, + mock_match_and_check_overlaps, mock_renewal_file_for_certname): + # pylint: disable = unused-argument + config = self.config + config.namespace.delete_after_revoke = True + config.cert_path = "/some/reasonable/path" + config.certname = "" + mock_cert_path_to_lineage.return_value = "example.com" + mock_full_archive_dir.return_value = "" + mock_match_and_check_overlaps.return_value = "" + self._call(config) + self.assertEqual(mock_delete.call_count, 1) + self.assertFalse(mock_get_utility().yesno.called) + # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 1e0b7754b..e1aad4336 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -345,9 +345,14 @@ common auth --must-staple --domains "must-staple.le.wtf" openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' # revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke # revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke +if [ ! -d "$root/conf/live/le1.wtf" ]; then + echo "cert deleted when --no-delete-after-revoke was used!" + exit 1 +fi +common delete --cert-name le1.wtf # revoke by cert key common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ --key-path "$root/conf/live/le2.wtf/privkey.pem" From e02adec26b9018a3c7fae40fdc13086cd336b05c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Jan 2018 17:38:03 -0800 Subject: [PATCH 260/631] Have letsencrypt-auto do a real upgrade in leauto-upgrades option 2 (#5390) * Make leauto_upgrades do a real upgrade * Cleanup vars and output * Sleep until the server is ready * add simple_http_server.py * Use a randomly assigned port * s/realpath/readlink * wait for server before getting port * s/localhost/all interfaces --- .../letstest/scripts/test_leauto_upgrades.sh | 47 +++++++++++++++++-- tools/simple_http_server.py | 26 ++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100755 tools/simple_http_server.py diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index cb659786e..a83cbd826 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -15,19 +15,56 @@ if ! command -v git ; then exit 1 fi fi -BRANCH=`git rev-parse --abbrev-ref HEAD` # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's # the first version that pins package versions, properly supports # --no-self-upgrade, and works with newer versions of pip. -git checkout -f v0.5.0 +git checkout -f v0.5.0 letsencrypt-auto if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then echo initial installation appeared to fail exit 1 fi -git checkout -f "$BRANCH" -EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then +# Now that python and openssl have been installed, we can set up a fake server +# to provide a new version of letsencrypt-auto. First, we start the server and +# directory to be served. +MY_TEMP_DIR=$(mktemp -d) +PORT_FILE="$MY_TEMP_DIR/port" +SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py) +cd "$MY_TEMP_DIR" +"$SERVER_PATH" 0 > $PORT_FILE & +SERVER_PID=$! +trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT +cd ~- + +# Then, we set up the files to be served. +FAKE_VERSION_NUM="99.99.99" +echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" +LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" +NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" +mkdir "$LE_AUTO_SOURCE_DIR" +cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" +SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" +openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" + +# Next, we wait for the server to start and get the port number. +sleep 5s +SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") + +# Finally, we set the necessary certbot-auto environment variables. +export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" +export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" +export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg +tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G +hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT +uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl +LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 +Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 +iQIDAQAB +-----END PUBLIC KEY----- +" + +if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then echo upgrade appeared to fail exit 1 fi diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py new file mode 100755 index 000000000..26bf231b7 --- /dev/null +++ b/tools/simple_http_server.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" +from BaseHTTPServer import HTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler +import sys + +def serve_forever(port=0): + """Spins up an HTTP server on all interfaces and the given port. + + A message is printed to stdout specifying the address and port being used + by the server. + + :param int port: port number to use. + + """ + server = HTTPServer(('', port), SimpleHTTPRequestHandler) + print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address) + sys.stdout.flush() + server.serve_forever() + + +if __name__ == '__main__': + kwargs = {} + if len(sys.argv) > 1: + kwargs['port'] = int(sys.argv[1]) + serve_forever(**kwargs) From d557475bb6f921b7bdb0459d77a162d673044dce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 07:46:21 -0800 Subject: [PATCH 261/631] update Apache ciphersuites (#5383) --- certbot-apache/certbot_apache/centos-options-ssl-apache.conf | 2 +- certbot-apache/certbot_apache/constants.py | 2 ++ certbot-apache/certbot_apache/options-ssl-apache.conf | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf index 17ae1be76..56c946a4e 100644 --- a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS SSLHonorCipherOrder on SSLOptions +StrictRequire diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index a13ca04a6..fd6a9eb11 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -16,6 +16,8 @@ ALL_SSL_OPTIONS_HASHES = [ '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27', 'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88', 'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b', + '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791', + 'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/options-ssl-apache.conf b/certbot-apache/certbot_apache/options-ssl-apache.conf index 950a02a8b..8113ee81e 100644 --- a/certbot-apache/certbot_apache/options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS SSLHonorCipherOrder on SSLCompression off From 62ffcf53738760f901d8ad3b31bff3855d1648c6 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 9 Jan 2018 17:48:05 +0200 Subject: [PATCH 262/631] Fix macOS builds for Python2.7 in Travis (#5378) * Add OSX Python2 tests * Make sure python2 is originating from homebrew on macOS * Upgrade the already installed python2 instead of trying to reinstall --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d41bfa4b..35666d8e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ cache: - $HOME/.cache/pip before_install: - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)' + - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' From 288c4d956cf59015590bc24e045138335b46a40d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 9 Jan 2018 18:28:52 +0200 Subject: [PATCH 263/631] Automatically install updates in test script (#5394) --- letsencrypt-auto-source/tests/centos6_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index e3ebbaec5..23b1e16e8 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash # Start by making sure your system is up-to-date: -yum update > /dev/null +yum update -y > /dev/null yum install -y centos-release-scl > /dev/null yum install -y python27 > /dev/null 2> /dev/null From 887a6bcfce73f7b3c556720ccbbc470bf8855606 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 15:40:26 -0800 Subject: [PATCH 264/631] Handle need to rebootstrap before fetch.py (#5389) * Fix #5387 * Add test for #5387 * remove LE_PYTHON * Use environment variable to reduce line length --- letsencrypt-auto-source/letsencrypt-auto | 20 +++++++++++++------ .../letsencrypt-auto.template | 20 +++++++++++++------ .../tests/centos6_tests.sh | 12 +++++++++-- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f1361d8ea..5f46e3a31 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,10 +246,14 @@ DeprecationBootstrap() { fi } +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. if [ -n "$USE_PYTHON_3" ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. @@ -273,10 +277,12 @@ DeterminePythonVersion() { export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -1575,8 +1581,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f4c1b202f..7c3cbac08 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,10 +246,14 @@ DeprecationBootstrap() { fi } +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. if [ -n "$USE_PYTHON_3" ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. @@ -273,10 +277,12 @@ DeterminePythonVersion() { export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -586,8 +592,10 @@ else {{ fetch.py }} UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index 23b1e16e8..a0e96edf8 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -4,6 +4,8 @@ yum update -y > /dev/null yum install -y centos-release-scl > /dev/null yum install -y python27 > /dev/null 2> /dev/null +LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" + # we're going to modify env variables, so do this in a subshell ( source /opt/rh/python27/enable @@ -25,7 +27,7 @@ if [ $RESULT -ne 0 ]; then fi # bootstrap, but don't install python 3. -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null +"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null # ensure python 3 isn't installed python3 --version 2> /dev/null @@ -47,8 +49,14 @@ if [ $RESULT -eq 0 ]; then exit 1 fi +# Skip self upgrade due to Python 3 not being available. +if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then + echo "Python upgrade failure warning not printed!" + exit 1 +fi + # bootstrap, this time installing python3 -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null +"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null # ensure python 3 is installed python3 --version > /dev/null From f5a02714cd8b610db52ac04e62685bba9c081a47 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 9 Jan 2018 16:11:04 -0800 Subject: [PATCH 265/631] Add deprecation warning for Python 2.6 (#5391) * Add deprecation warning for Python 2.6 * Allow disabling Python 2.6 warning --- acme/acme/__init__.py | 13 +++++++------ certbot/main.py | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 618dda200..5850fa955 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -13,9 +13,10 @@ supported version: `draft-ietf-acme-01`_. import sys import warnings -if sys.version_info[:2] == (3, 3): - warnings.warn( - "Python 3.3 support will be dropped in the next release of " - "acme. Please upgrade your Python version.", - PendingDeprecationWarning, - ) #pragma: no cover +for (major, minor) in [(2, 6), (3, 3)]: + if sys.version_info[:2] == (major, minor): + warnings.warn( + "Python {0}.{1} support will be dropped in the next release of " + "acme. Please upgrade your Python version.".format(major, minor), + DeprecationWarning, + ) #pragma: no cover diff --git a/certbot/main.py b/certbot/main.py index e25e030aa..32dd69256 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,6 +4,7 @@ import functools import logging.handlers import os import sys +import warnings import configobj import josepy as jose @@ -1217,9 +1218,17 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - if sys.version_info[:2] == (3, 3): - logger.warning("Python 3.3 support will be dropped in the next release " - "of Certbot - please upgrade your Python version.") + deprecation_fmt = ( + "Python %s.%s support will be dropped in the next " + "release of Certbot - please upgrade your Python version.") + # We use the warnings system for Python 2.6 and logging for Python 3 + # because DeprecationWarnings are only reported by default in Python <= 2.6 + # and warnings can be disabled by the user. + if sys.version_info[:2] == (2, 6): + warning = deprecation_fmt % sys.version_info[:2] + warnings.warn(warning, DeprecationWarning) + elif sys.version_info[:2] == (3, 3): + logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) From 6eb459354fc28433ac2eabc3be62c809df66a4a1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 9 Jan 2018 16:48:16 -0800 Subject: [PATCH 266/631] Address erikrose's comments on #5329 (#5400) --- letsencrypt-auto-source/letsencrypt-auto | 13 ++++++++----- letsencrypt-auto-source/letsencrypt-auto.template | 5 ++++- .../pieces/bootstrappers/rpm_common_base.sh | 2 +- letsencrypt-auto-source/pieces/fetch.py | 6 +++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 5f46e3a31..712ef6813 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -254,7 +254,7 @@ DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python # # If no Python is found, PYVER is set to 0. - if [ -n "$USE_PYTHON_3" ]; then + if [ "$USE_PYTHON_3" = 1 ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. $EXISTS "$LE_PYTHON" > /dev/null && break @@ -443,7 +443,7 @@ InitializeRPMCommonBase() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then @@ -781,6 +781,9 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" @@ -1482,7 +1485,7 @@ class HttpsGetter(object): # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) else: self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: @@ -1520,7 +1523,7 @@ def latest_stable_version(get): # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1552,7 +1555,7 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): +def cert_none_context(): """Create a SSLContext object to not check hostname.""" # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 7c3cbac08..b06ac9c80 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -254,7 +254,7 @@ DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python # # If no Python is found, PYVER is set to 0. - if [ -n "$USE_PYTHON_3" ]; then + if [ "$USE_PYTHON_3" = 1 ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. $EXISTS "$LE_PYTHON" > /dev/null && break @@ -320,6 +320,9 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh index d7a9f3133..326ad8b3f 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh @@ -35,7 +35,7 @@ InitializeRPMCommonBase() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index ae72a299b..1515fe353 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -50,7 +50,7 @@ class HttpsGetter(object): # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) else: self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: @@ -88,7 +88,7 @@ def latest_stable_version(get): # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -120,7 +120,7 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): +def cert_none_context(): """Create a SSLContext object to not check hostname.""" # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) From 00634394f2e077ec5e621acd869e370f8619cd08 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 21:16:44 -0800 Subject: [PATCH 267/631] Only respect LE_PYTHON inside USE_PYTHON_3 if we know a user must have set it version 2 (#5402) * stop exporting LE_PYTHON * unset LE_PYTHON sometimes --- letsencrypt-auto-source/letsencrypt-auto | 8 ++++++-- letsencrypt-auto-source/letsencrypt-auto.template | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 712ef6813..0ad089acd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -274,7 +274,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -801,7 +800,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -906,6 +905,10 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" if [ -d "$VENV_PATH" ]; then # If the selected Bootstrap function isn't a noop and it differs from the @@ -1412,6 +1415,7 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b06ac9c80..5b8b1c164 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -274,7 +274,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -340,7 +339,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -445,6 +444,10 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" if [ -d "$VENV_PATH" ]; then # If the selected Bootstrap function isn't a noop and it differs from the @@ -571,6 +574,7 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then From 3acf5d1ef909b2e775588a7e045bb00728e2de83 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 12:10:21 -0800 Subject: [PATCH 268/631] Fix rebootstraping with old venvs (#5392) * Fix rebootstrapping before venv move * add regression test * dedupe test * Cleanup case when two venvs exist. * Add clarifying comment * Add double venv test to leauto_upgrades * Fix logic with the help of coffee * redirect stderr * pass VENV_PATH through sudo * redirect stderr --- letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++++-- .../letsencrypt-auto.template | 24 ++++++++++-- .../letstest/scripts/test_leauto_upgrades.sh | 37 ++++++++++++++++++- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ad089acd..b86517bb6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -897,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -910,13 +914,21 @@ if [ "$1" = "--le-auto-phase2" ]; then fi INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -926,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -1418,7 +1434,7 @@ else export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 5b8b1c164..96e5c2db0 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -436,7 +436,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -449,13 +453,21 @@ if [ "$1" = "--le-auto-phase2" ]; then fi INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -465,6 +477,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -577,7 +593,7 @@ else export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index a83cbd826..51472f2e6 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -64,10 +64,45 @@ iQIDAQAB -----END PUBLIC KEY----- " -if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then +if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + if command -v python3; then + echo "Didn't expect Python 3 to be installed!" + exit 1 + fi + cp letsencrypt-auto cb-auto + if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then + echo "Certbot shouldn't have updated to a new version!" + exit 1 + fi + if [ -d "/opt/eff.org" ]; then + echo "New directory shouldn't have been created!" + exit 1 + fi + # Create a 2nd venv at the new path to ensure we properly handle this case + export VENV_PATH="/opt/eff.org/certbot/venv" + if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then + echo second installation appeared to fail + exit 1 + fi + unset VENV_PATH + EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) + if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then + echo "Certbot didn't upgrade as expected!" + exit 1 + fi + if ! command -v python3; then + echo "Python3 wasn't properly installed" + exit 1 + fi + if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then + echo "Python3 wasn't used in venv!" + exit 1 + fi +elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then echo upgrade appeared to fail exit 1 fi + echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then From 39472f88dead13782f2d84e65135cf11b96258bb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 13:26:31 -0800 Subject: [PATCH 269/631] reduce ipdb version (#5408) --- tools/dev_constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index afc362ff8..47440241d 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -21,7 +21,7 @@ futures==3.1.1 google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 -ipdb==0.10.3 +ipdb==0.10.2 ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 From 9e952081014b9545ce6ddb8b6ecc86a51bf94131 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 18:34:45 -0800 Subject: [PATCH 270/631] Factor out common challengeperformer logic (#5413) --- .../certbot_apache/tests/tls_sni_01_test.py | 4 +- .../certbot_nginx/tests/tls_sni_01_test.py | 2 +- certbot/plugins/common.py | 42 ++++++++++++---- certbot/plugins/common_test.py | 48 ++++++++++++------- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 6c37c2ecc..42fb3021b 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -16,8 +16,8 @@ from six.moves import xrange # pylint: disable=redefined-builtin, import-error class TlsSniPerformTest(util.ApacheTest): """Test the ApacheTlsSni01 challenge.""" - auth_key = common_test.TLSSNI01Test.auth_key - achalls = common_test.TLSSNI01Test.achalls + auth_key = common_test.AUTH_KEY + achalls = common_test.ACHALLS def setUp(self): # pylint: disable=arguments-differ super(TlsSniPerformTest, self).setUp() diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 32a5ed7d2..61ee293fa 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -20,7 +20,7 @@ from certbot_nginx.tests import util class TlsSniPerformTest(util.NginxTest): """Test the NginxTlsSni01 challenge.""" - account_key = common_test.TLSSNI01Test.auth_key + account_key = common_test.AUTH_KEY achalls = [ achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 002d2f225..c281534ca 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -315,23 +315,28 @@ class Addr(object): return result -class TLSSNI01(object): - """Abstract base for TLS-SNI-01 challenge performers""" +class ChallengePerformer(object): + """Abstract base for challenge performers. + + :ivar configurator: Authenticator and installer plugin + :ivar achalls: Annotated challenges + :vartype achalls: `list` of `.KeyAuthorizationAnnotatedChallenge` + :ivar indices: Holds the indices of challenges from a larger array + so the user of the class doesn't have to. + :vartype indices: `list` of `int` + + """ def __init__(self, configurator): self.configurator = configurator self.achalls = [] self.indices = [] - self.challenge_conf = os.path.join( - configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") - # self.completed = 0 def add_chall(self, achall, idx=None): - """Add challenge to TLSSNI01 object to perform at once. + """Store challenge to be performed when perform() is called. :param .KeyAuthorizationAnnotatedChallenge achall: Annotated - TLSSNI01 challenge. - + challenge. :param int idx: index to challenge in a larger array """ @@ -339,6 +344,27 @@ class TLSSNI01(object): if idx is not None: self.indices.append(idx) + def perform(self): + """Perform all added challenges. + + :returns: challenge respones + :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` + + + """ + raise NotImplementedError() + + +class TLSSNI01(ChallengePerformer): + # pylint: disable=abstract-method + """Abstract base for TLS-SNI-01 challenge performers""" + + def __init__(self, configurator): + super(TLSSNI01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") + # self.completed = 0 + def get_cert_path(self, achall): """Returns standardized name for challenge certificate. diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 1a1ca7dcb..103a12499 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -18,6 +18,17 @@ from certbot import errors from certbot.tests import acme_util from certbot.tests import util as test_util +AUTH_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) +ACHALLS = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token1'), "pending"), + domain="encryption-example.demo", account_key=AUTH_KEY), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token2'), "pending"), + domain="certbot.demo", account_key=AUTH_KEY), +] class NamespaceFunctionsTest(unittest.TestCase): """Tests for certbot.plugins.common.*_namespace functions.""" @@ -261,21 +272,27 @@ class AddrTest(unittest.TestCase): self.assertEqual(set_c, set_d) +class ChallengePerformerTest(unittest.TestCase): + """Tests for certbot.plugins.common.ChallengePerformer.""" + + def setUp(self): + configurator = mock.MagicMock() + + from certbot.plugins.common import ChallengePerformer + self.performer = ChallengePerformer(configurator) + + def test_add_chall(self): + self.performer.add_chall(ACHALLS[0], 0) + self.assertEqual(1, len(self.performer.achalls)) + self.assertEqual([0], self.performer.indices) + + def test_perform(self): + self.assertRaises(NotImplementedError, self.performer.perform) + + class TLSSNI01Test(unittest.TestCase): """Tests for certbot.plugins.common.TLSSNI01.""" - auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) - achalls = [ - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token1'), "pending"), - domain="encryption-example.demo", account_key=auth_key), - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token2'), "pending"), - domain="certbot.demo", account_key=auth_key), - ] - def setUp(self): self.tempdir = tempfile.mkdtemp() configurator = mock.MagicMock() @@ -288,11 +305,6 @@ class TLSSNI01Test(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def test_add_chall(self): - self.sni.add_chall(self.achalls[0], 0) - self.assertEqual(1, len(self.sni.achalls)) - self.assertEqual([0], self.sni.indices) - def test_setup_challenge_cert(self): # This is a helper function that can be used for handling # open context managers more elegantly. It avoids dealing with @@ -325,7 +337,7 @@ class TLSSNI01Test(unittest.TestCase): OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) def test_get_z_domain(self): - achall = self.achalls[0] + achall = ACHALLS[0] self.assertEqual(self.sni.get_z_domain(achall), achall.response(achall.account_key).z_domain.decode("utf-8")) From 2ba334a18202926e4e6b4f166a18457861bba031 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 20:14:56 -0800 Subject: [PATCH 271/631] Add basic HTTP01 support to Apache * Add a simple version of HTTP01 * remove cert from chall name * make directory work on 2.2 * cleanup challenges when finished * import shutil * fixup perform and cleanup tests * Add tests for http_01.py --- certbot-apache/certbot_apache/configurator.py | 36 ++++-- certbot-apache/certbot_apache/http_01.py | 94 +++++++++++++++ .../certbot_apache/tests/configurator_test.py | 70 +++++++---- .../certbot_apache/tests/http_01_test.py | 112 ++++++++++++++++++ .../certbot_apache/tests/tls_sni_01_test.py | 2 +- 5 files changed, 278 insertions(+), 36 deletions(-) create mode 100644 certbot-apache/certbot_apache/http_01.py create mode 100644 certbot-apache/certbot_apache/tests/http_01_test.py diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 5a33346ea..60441f30c 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -24,9 +24,10 @@ from certbot_apache import apache_util from certbot_apache import augeas_configurator from certbot_apache import constants from certbot_apache import display_ops -from certbot_apache import tls_sni_01 +from certbot_apache import http_01 from certbot_apache import obj from certbot_apache import parser +from certbot_apache import tls_sni_01 from collections import defaultdict @@ -163,6 +164,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} + # This will be set during the perform function + self.http_doer = None + @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" @@ -1855,7 +1859,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] def perform(self, achalls): """Perform the configuration related challenge. @@ -1867,16 +1871,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self._chall_out.update(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.ApacheTlsSni01(self) + self.http_doer = http_01.ApacheHttp01(self) + sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + self.http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() - if sni_response: + http_response = self.http_doer.perform() + sni_response = sni_doer.perform() + if http_response or sni_response: # Must reload in order to activate the challenges. # Handled here because we may be able to load up other challenge # types @@ -1886,14 +1895,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # of identifying when the new configuration is being used. time.sleep(3) - # Go through all of the challenges and assign them to the proper - # place in the responses return value. All responses must be in the - # same order as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + self._update_responses(responses, http_response, self.http_doer) + self._update_responses(responses, sni_response, sni_doer) return responses + def _update_responses(self, responses, chall_response, chall_doer): + # Go through all of the challenges and assign them to the proper + # place in the responses return value. All responses must be in the + # same order as the original challenges. + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp + def cleanup(self, achalls): """Revert all challenges.""" self._chall_out.difference_update(achalls) @@ -1903,6 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.revert_challenge_config() self.restart() self.parser.reset_modules() + self.http_doer.cleanup() def install_ssl_options_conf(self, options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py new file mode 100644 index 000000000..410e8e9a6 --- /dev/null +++ b/certbot-apache/certbot_apache/http_01.py @@ -0,0 +1,94 @@ +"""A class that performs HTTP-01 challenges for Apache""" +import logging +import os +import shutil +import tempfile + +from certbot.plugins import common + +logger = logging.getLogger(__name__) + +class ApacheHttp01(common.TLSSNI01): + """Class that performs HTPP-01 challenges within the Apache configurator.""" + + CONFIG_TEMPLATE24 = """\ +Alias /.well-known/acme-challenge {0} + + + Require all granted + + +""" + + CONFIG_TEMPLATE22 = """\ +Alias /.well-known/acme-challenge {0} + + + Order allow,deny + Allow from all + + +""" + + def __init__(self, *args, **kwargs): + super(ApacheHttp01, self).__init__(*args, **kwargs) + self.challenge_conf = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge.conf") + self.challenge_dir = None + + def perform(self): + """Perform all HTTP-01 challenges.""" + if not self.achalls: + return [] + # Save any changes to the configuration as a precaution + # About to make temporary changes to the config + self.configurator.save("Changes before challenge setup", True) + + responses = self._set_up_challenges() + self._mod_config() + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def cleanup(self): + """Cleanup the challenge directory.""" + shutil.rmtree(self.challenge_dir, ignore_errors=True) + self.challenge_dir = None + + def _mod_config(self): + self.configurator.parser.add_include( + self.configurator.parser.loc["default"], self.challenge_conf) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf) + + if self.configurator.version < (2, 4): + config_template = self.CONFIG_TEMPLATE22 + else: + config_template = self.CONFIG_TEMPLATE24 + config_text = config_template.format(self.challenge_dir) + + logger.debug("writing a config file with text:\n %s", config_text) + with open(self.challenge_conf, "w") as new_conf: + new_conf.write(config_text) + + def _set_up_challenges(self): + self.challenge_dir = tempfile.mkdtemp() + os.chmod(self.challenge_dir, 0o755) + + responses = [] + for achall in self.achalls: + responses.append(self._set_up_challenge(achall)) + + return responses + + def _set_up_challenge(self, achall): + response, validation = achall.response_and_validation() + + name = os.path.join(self.challenge_dir, achall.chall.encode("token")) + with open(name, 'wb') as f: + f.write(validation.encode()) + os.chmod(name, 0o644) + + return response diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 4f85e1e3f..1620013c8 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -676,23 +676,33 @@ class MultipleVhostsTest(util.ApacheTest): self.config._add_name_vhost_if_necessary(self.vh_truth[0]) self.assertEqual(self.config.add_name_vhost.call_count, 2) + @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform") @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_perform(self, mock_restart, mock_perform): + def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - account_key, achall1, achall2 = self.get_achalls() + account_key, achalls = self.get_key_and_achalls() - expected = [ - achall1.response(account_key), - achall2.response(account_key), - ] + all_expected = [] + http_expected = [] + tls_expected = [] + for achall in achalls: + response = achall.response(account_key) + if isinstance(achall.chall, challenges.HTTP01): + http_expected.append(response) + else: + tls_expected.append(response) + all_expected.append(response) - mock_perform.return_value = expected - responses = self.config.perform([achall1, achall2]) + mock_http_perform.return_value = http_expected + mock_tls_perform.return_value = tls_expected - self.assertEqual(mock_perform.call_count, 1) - self.assertEqual(responses, expected) + responses = self.config.perform(achalls) + + self.assertEqual(mock_http_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(responses, all_expected) self.assertEqual(mock_restart.call_count, 1) @@ -700,30 +710,38 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() + self.config.http_doer = mock.MagicMock() - self.config._chall_out.add(achall1) # pylint: disable=protected-access - self.config._chall_out.add(achall2) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall1]) - self.assertFalse(mock_restart.called) - - self.config.cleanup([achall2]) - self.assertTrue(mock_restart.called) + for i, achall in enumerate(achalls): + self.config.cleanup([achall]) + if i == len(achalls) - 1: + self.assertTrue(mock_restart.called) + self.assertTrue(self.config.http_doer.cleanup.called) + else: + self.assertFalse(mock_restart.called) + self.assertFalse(self.config.http_doer.cleanup.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup_no_errors(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() + self.config.http_doer = mock.MagicMock() - self.config._chall_out.add(achall1) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall2]) + self.config.cleanup([achalls[-1]]) self.assertFalse(mock_restart.called) + self.assertFalse(self.config.http_doer.cleanup.called) - self.config.cleanup([achall1, achall2]) + self.config.cleanup(achalls) self.assertTrue(mock_restart.called) + self.assertTrue(self.config.http_doer.cleanup.called) @mock.patch("certbot.util.run_script") def test_get_version(self, mock_script): @@ -1151,7 +1169,7 @@ class MultipleVhostsTest(util.ApacheTest): not_rewriterule = "NotRewriteRule ^ ..." self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule)) - def get_achalls(self): + def get_key_and_achalls(self): """Return testing achallenges.""" account_key = self.rsa512jwk achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -1166,8 +1184,12 @@ class MultipleVhostsTest(util.ApacheTest): token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), "pending"), domain="certbot.demo", account_key=account_key) + achall3 = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=(b'x' * 16)), "pending"), + domain="example.org", account_key=account_key) - return account_key, achall1, achall2 + return account_key, (achall1, achall2, achall3) def test_make_addrs_sni_ready(self): self.config.version = (2, 2) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py new file mode 100644 index 000000000..11121fae2 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -0,0 +1,112 @@ +"""Test for certbot_apache.http_01.""" +import os +import unittest + +from acme import challenges + +from certbot import achallenges + +from certbot.tests import acme_util + +from certbot_apache.tests import util + + +NUM_ACHALLS = 3 + + +class ApacheHttp01TestMeta(type): + """Generates parmeterized tests for testing perform.""" + def __new__(mcs, name, bases, class_dict): + + def _gen_test(num_achalls, minor_version): + def _test(self): + achalls = self.achalls[:num_achalls] + self.config.version = (2, minor_version) + self.common_perform_test(achalls) + return _test + + for i in range(1, NUM_ACHALLS + 1): + for j in (2, 4): + test_name = "test_perform_{0}_{1}".format(i, j) + class_dict[test_name] = _gen_test(i, j) + return type.__new__(mcs, name, bases, class_dict) + + +class ApacheHttp01Test(util.ApacheTest): + """Test for certbot_apache.http_01.ApacheHttp01.""" + + __metaclass__ = ApacheHttp01TestMeta + + def setUp(self, *args, **kwargs): + super(ApacheHttp01Test, self).setUp(*args, **kwargs) + self.maxDiff = None + + self.account_key = self.rsa512jwk + self.achalls = [] + for i in range(NUM_ACHALLS): + self.achalls.append( + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((chr(ord('a') + i) * 16))), + "pending"), + domain="example{0}.com".format(i), + account_key=self.account_key)) + + from certbot_apache.http_01 import ApacheHttp01 + self.http = ApacheHttp01(self.config) + + def test_empty_perform(self): + self.assertFalse(self.http.perform()) + + def common_perform_test(self, achalls): + """Tests perform with the given achalls.""" + for achall in achalls: + self.http.add_chall(achall) + + expected_response = [ + achall.response(self.account_key) for achall in achalls] + self.assertEqual(self.http.perform(), expected_response) + + self.assertTrue(os.path.isdir(self.http.challenge_dir)) + self._has_min_permissions(self.http.challenge_dir, 0o755) + self._test_challenge_conf() + + for achall in achalls: + self._test_challenge_file(achall) + + challenge_dir = self.http.challenge_dir + self.http.cleanup() + self.assertFalse(os.path.exists(challenge_dir)) + + def _test_challenge_conf(self): + self.assertEqual( + len(self.config.parser.find_dir( + "Include", self.http.challenge_conf)), 1) + + with open(self.http.challenge_conf) as f: + conf_contents = f.read() + + alias_fmt = "Alias /.well-known/acme-challenge {0}" + alias = alias_fmt.format(self.http.challenge_dir) + self.assertTrue(alias in conf_contents) + if self.config.version < (2, 4): + self.assertTrue("Allow from all" in conf_contents) + else: + self.assertTrue("Require all granted" in conf_contents) + + def _test_challenge_file(self, achall): + name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) + validation = achall.validation(self.account_key) + + self._has_min_permissions(name, 0o644) + with open(name, 'rb') as f: + self.assertEqual(f.read(), validation.encode()) + + def _has_min_permissions(self, path, min_mode): + """Tests the given file has at least the permissions in mode.""" + st_mode = os.stat(path).st_mode + self.assertEqual(st_mode, st_mode | min_mode) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 42fb3021b..8cea97f04 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -1,6 +1,6 @@ """Test for certbot_apache.tls_sni_01.""" -import unittest import shutil +import unittest import mock From fa97877cfb058a8a6eda0e86b079876230bcfb7d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 11 Jan 2018 14:46:48 +0200 Subject: [PATCH 272/631] Make sure that Apache is listening on port 80 and has mod_alias * Ensure that mod_alias is enabled * Make sure we listen to port http01_port --- certbot-apache/certbot_apache/configurator.py | 62 +++++++++++++++---- certbot-apache/certbot_apache/http_01.py | 11 ++++ .../certbot_apache/tests/configurator_test.py | 38 +++++++++++- .../certbot_apache/tests/http_01_test.py | 11 ++++ certbot-apache/certbot_apache/tests/util.py | 1 + 5 files changed, 111 insertions(+), 12 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 60441f30c..3b91617e5 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -740,28 +740,40 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ - # If nonstandard port, add service definition for matching - if port != "443": + self.prepare_https_modules(temp) + self.ensure_listen(port, https=True) + + def ensure_listen(self, port, https=False): + """Make sure that Apache is listening on the port. Checks if the + Listen statement for the port already exists, and adds it to the + configuration if necessary. + + :param str port: Port number to check and add Listen for if not in + place already + :param bool https: If the port will be used for HTTPS + + """ + + # If HTTPS requested for nonstandard port, add service definition + if https and port != "443": port_service = "%s %s" % (port, "https") else: port_service = port - self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] - # In case no Listens are set (which really is a broken apache config) - if not listens: - listens = ["80"] - # Listen already in place if self._has_port_already(listens, port): return listen_dirs = set(listens) + if not listens: + listen_dirs.add(port_service) + for listen in listens: # For any listen statement, check if the machine also listens on # Port 443. If not, add such a listen statement. @@ -776,11 +788,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if "%s:%s" % (ip, port_service) not in listen_dirs and ( "%s:%s" % (ip, port_service) not in listen_dirs): listen_dirs.add("%s:%s" % (ip, port_service)) - self._add_listens(listen_dirs, listens, port) + if https: + self._add_listens_https(listen_dirs, listens, port) + else: + self._add_listens_http(listen_dirs, listens, port) - def _add_listens(self, listens, listens_orig, port): - """Helper method for prepare_server_https to figure out which new - listen statements need adding + def _add_listens_http(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTP on port + + :param set listens: Set of all needed Listen statements + :param list listens_orig: List of existing listen statements + :param string port: Port number we're adding + """ + + new_listens = listens.difference(listens_orig) + + if port in new_listens: + # We have wildcard, skip the rest + self.parser.add_dir(parser.get_aug_path(self.parser.loc["listen"]), + "Listen", port) + self.save_notes += "Added Listen %s directive to %s\n" % ( + port, self.parser.loc["listen"]) + else: + for listen in new_listens: + self.parser.add_dir(parser.get_aug_path( + self.parser.loc["listen"]), "Listen", listen.split(" ")) + self.save_notes += ("Added Listen %s directive to " + "%s\n") % (listen, + self.parser.loc["listen"]) + + def _add_listens_https(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTPS on port :param set listens: Set of all needed Listen statements :param list listens_orig: List of existing listen statements diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 410e8e9a6..9c25b1f17 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -45,6 +45,10 @@ Alias /.well-known/acme-challenge {0} # About to make temporary changes to the config self.configurator.save("Changes before challenge setup", True) + self.configurator.ensure_listen(str( + self.configurator.config.http01_port)) + self.prepare_http01_modules() + responses = self._set_up_challenges() self._mod_config() # Save reversible changes @@ -57,6 +61,13 @@ Alias /.well-known/acme-challenge {0} shutil.rmtree(self.challenge_dir, ignore_errors=True) self.challenge_dir = None + def prepare_http01_modules(self): + """Make sure that we have the needed modules available for http01""" + + if self.configurator.conf("handle-modules"): + if "alias_module" not in self.configurator.parser.modules: + self.configurator.enable_mod("alias", temp=True) + def _mod_config(self): self.configurator.parser.add_include( self.configurator.parser.loc["default"], self.challenge_conf) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 1620013c8..ea3867061 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -424,6 +424,43 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:80")) + def test_add_listen_80(self): + mock_find = mock.Mock() + mock_add_dir = mock.Mock() + mock_find.return_value = [] + self.config.parser.find_dir = mock_find + self.config.parser.add_dir = mock_add_dir + self.config.ensure_listen("80") + self.assertTrue(mock_add_dir.called) + self.assertTrue(mock_find.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + self.assertEqual(mock_add_dir.call_args[0][2], "80") + + def test_add_listen_80_named(self): + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2", "test3"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + mock_add_dir = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir = mock_add_dir + + self.config.ensure_listen("80") + self.assertEqual(mock_add_dir.call_count, 0) + + # Reset return lists and inputs + mock_add_dir.reset_mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + + # Test + self.config.ensure_listen("8080") + self.assertEqual(mock_add_dir.call_count, 3) + self.assertTrue(mock_add_dir.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + def test_prepare_server_https(self): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -435,7 +472,6 @@ class MultipleVhostsTest(util.ApacheTest): # This will test the Add listen self.config.parser.find_dir = mock_find self.config.parser.add_dir_to_ifmodssl = mock_add_dir - self.config.prepare_server_https("443") # Changing the order these modules are enabled breaks the reverter self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 11121fae2..f52b9d0cc 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -1,4 +1,5 @@ """Test for certbot_apache.http_01.""" +import mock import os import unittest @@ -53,11 +54,21 @@ class ApacheHttp01Test(util.ApacheTest): account_key=self.account_key)) from certbot_apache.http_01 import ApacheHttp01 + self.config.parser.modules.add("mod_alias.c") + self.config.parser.modules.add("alias_module") self.http = ApacheHttp01(self.config) def test_empty_perform(self): self.assertFalse(self.http.perform()) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_add_alias_module(self, mock_enmod): + self.config.parser.modules.remove("alias_module") + self.config.parser.modules.remove("mod_alias.c") + self.http.prepare_http01_modules() + self.assertTrue(mock_enmod.called) + self.assertEqual(mock_enmod.call_args[0][0], "alias") + def common_perform_test(self, achalls): """Tests perform with the given achalls.""" for achall in achalls: diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index ca667465c..8f5162629 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -103,6 +103,7 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, + http01_port="80", temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) From f0f5defb6ff9fcc54db4bbc977950610b22cd155 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 Jan 2018 09:27:30 -0800 Subject: [PATCH 273/631] Address minor concerns with Apache HTTP-01 * enable other modules * change port type * remove maxDiff from test class * update port comment * add -f to a2dismod --- certbot-apache/certbot_apache/configurator.py | 2 +- certbot-apache/certbot_apache/http_01.py | 10 ++++- .../certbot_apache/override_debian.py | 2 +- .../certbot_apache/tests/http_01_test.py | 39 ++++++++++++++++--- certbot-apache/certbot_apache/tests/util.py | 2 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 3b91617e5..8d6995211 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -776,7 +776,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for listen in listens: # For any listen statement, check if the machine also listens on - # Port 443. If not, add such a listen statement. + # the given port. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces if port not in listen_dirs and port_service not in listen_dirs: diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 9c25b1f17..804644975 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -65,8 +65,14 @@ Alias /.well-known/acme-challenge {0} """Make sure that we have the needed modules available for http01""" if self.configurator.conf("handle-modules"): - if "alias_module" not in self.configurator.parser.modules: - self.configurator.enable_mod("alias", temp=True) + needed_modules = ["alias"] + if self.configurator.version < (2, 4): + needed_modules.append("authz_host") + else: + needed_modules.append("authz_core") + for mod in needed_modules: + if mod + "_module" not in self.configurator.parser.modules: + self.configurator.enable_mod(mod, temp=True) def _mod_config(self): self.configurator.parser.add_include( diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 6e2e34ba9..02dffc3f7 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -140,5 +140,5 @@ class DebianConfigurator(configurator.ApacheConfigurator): "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), mod_name]) + temp, [self.conf("dismod"), "-f", mod_name]) util.run_script([self.conf("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index f52b9d0cc..4e2a5faff 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -40,7 +40,6 @@ class ApacheHttp01Test(util.ApacheTest): def setUp(self, *args, **kwargs): super(ApacheHttp01Test, self).setUp(*args, **kwargs) - self.maxDiff = None self.account_key = self.rsa512jwk self.achalls = [] @@ -53,21 +52,51 @@ class ApacheHttp01Test(util.ApacheTest): domain="example{0}.com".format(i), account_key=self.account_key)) + modules = ["alias", "authz_core", "authz_host"] + for mod in modules: + self.config.parser.modules.add("mod_{0}.c".format(mod)) + self.config.parser.modules.add(mod + "_module") + from certbot_apache.http_01 import ApacheHttp01 - self.config.parser.modules.add("mod_alias.c") - self.config.parser.modules.add("alias_module") self.http = ApacheHttp01(self.config) def test_empty_perform(self): self.assertFalse(self.http.perform()) @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_add_alias_module(self, mock_enmod): + def test_enable_modules_22(self, mock_enmod): + self.config.version = (2, 2) + self.config.parser.modules.remove("authz_host_module") + self.config.parser.modules.remove("mod_authz_host.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_host") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_enable_modules_24(self, mock_enmod): + self.config.parser.modules.remove("authz_core_module") + self.config.parser.modules.remove("mod_authz_core.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_core") + + def common_enable_modules_test(self, mock_enmod): + """Tests enabling mod_alias and other modules.""" self.config.parser.modules.remove("alias_module") self.config.parser.modules.remove("mod_alias.c") + self.http.prepare_http01_modules() + self.assertTrue(mock_enmod.called) - self.assertEqual(mock_enmod.call_args[0][0], "alias") + calls = mock_enmod.call_args_list + other_calls = [] + for call in calls: + if "alias" != call[0][0]: + other_calls.append(call) + + # If these lists are equal, we never enabled mod_alias + self.assertNotEqual(calls, other_calls) + return other_calls def common_perform_test(self, achalls): """Tests perform with the given achalls.""" diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 8f5162629..1ba1e2c34 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -103,7 +103,7 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, - http01_port="80", + http01_port=80, temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) From 28dad825afae8a079811114d72a53a9833155d6c Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 11 Jan 2018 20:44:40 +0200 Subject: [PATCH 274/631] Do not try to remove temp dir if it wasn't created --- certbot-apache/certbot_apache/http_01.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 804644975..87721d290 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -58,7 +58,8 @@ Alias /.well-known/acme-challenge {0} def cleanup(self): """Cleanup the challenge directory.""" - shutil.rmtree(self.challenge_dir, ignore_errors=True) + if self.challenge_dir: + shutil.rmtree(self.challenge_dir, ignore_errors=True) self.challenge_dir = None def prepare_http01_modules(self): From 2cb9d9e2aa332e4fd53aab32ad1d3946438a0dd3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 11 Jan 2018 17:06:23 -0800 Subject: [PATCH 275/631] Implement HTTP-01 challenge for Nginx (#5414) * get http01 challenge working * support multiple challenge types in configurator.py * update existing nginx tests * lint * refactor NginxHttp01 and NginxTlsSni01 to both now inherit from NginxChallengePerformer * remove TODO * challenges_test tests with both tlssni01 and http01 * Make challenges.py more abstract to make lint happier * add pylint disables to the tests to make pylint happier about the inheritance and abstraction situation * no need to cover raise NotImplementedError() lines * python3 compatibility * test that http01 perform is called * only remove ssl from addresses during http01 * Initialize addrs_to_add * Change Nginx http01 to modify server block so the site doesn't stop serving while getting a cert * pass existing unit tests * rename sni --> http01 in unit tests * lint * fix configurator test * select an http block instead of https * properly test for port number * use domains that have matching addresses * remove debugger * remove access_log and error_log cruft that wasn't being executed * continue to return None from choose_redirect_vhost when create_if_no_match is False * add nginx integration test --- certbot-nginx/certbot_nginx/configurator.py | 35 ++++-- certbot-nginx/certbot_nginx/http_01.py | 106 ++++++++++++++++ certbot-nginx/certbot_nginx/parser.py | 4 +- .../certbot_nginx/tests/configurator_test.py | 14 ++- .../certbot_nginx/tests/http_01_test.py | 113 ++++++++++++++++++ certbot-nginx/certbot_nginx/tests/util.py | 1 + .../tests/boulder-integration.conf.sh | 6 +- certbot-nginx/tests/boulder-integration.sh | 19 ++- 8 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/http_01.py create mode 100644 certbot-nginx/certbot_nginx/tests/http_01_test.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8af474c5e..a77cf2bc3 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -26,6 +26,7 @@ from certbot_nginx import constants from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 +from certbot_nginx import http_01 logger = logging.getLogger(__name__) @@ -208,7 +209,8 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -366,7 +368,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port): + def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -380,12 +382,19 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number + :param bool create_if_no_match: If we should create a new vhost from default + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. + :returns: vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` """ matches = self._get_redirect_ranked_matches(target_name, port) - return self._select_best_name_match(matches) + vhost = self._select_best_name_match(matches) + if not vhost and create_if_no_match: + vhost = self._vhost_from_duplicated_default(target_name) + return vhost def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -394,7 +403,7 @@ class NginxConfigurator(common.Installer): Rank by how well these match target_name. :param str target_name: The name to match - :param str port: port number + :param str port: port number as a string :returns: list of dicts containing the vhost, the matching name, and the numerical rank :rtype: list @@ -840,7 +849,7 @@ class NginxConfigurator(common.Installer): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] # Entry point in main.py for performing challenges def perform(self, achalls): @@ -853,15 +862,20 @@ class NginxConfigurator(common.Installer): """ self._chall_out += len(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.NginxTlsSni01(self) + sni_doer = tls_sni_01.NginxTlsSni01(self) + http_doer = http_01.NginxHttp01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() + sni_response = sni_doer.perform() + http_response = http_doer.perform() # Must restart in order to activate the challenges. # Handled here because we may be able to load up other challenge types self.restart() @@ -869,8 +883,9 @@ class NginxConfigurator(common.Installer): # Go through all of the challenges and assign them to the proper place # in the responses return value. All responses must be in the same order # as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)): + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp return responses diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py new file mode 100644 index 000000000..1f1e37891 --- /dev/null +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -0,0 +1,106 @@ +"""A class that performs HTTP-01 challenges for Nginx""" + +import logging +import os + +from acme import challenges + +from certbot.plugins import common + + +logger = logging.getLogger(__name__) + + +class NginxHttp01(common.ChallengePerformer): + """HTTP-01 authenticator for Nginx + + :ivar configurator: NginxConfigurator object + :type configurator: :class:`~nginx.configurator.NginxConfigurator` + + :ivar list achalls: Annotated + class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + challenges + + :param list indices: Meant to hold indices of challenges in a + larger array. NginxHttp01 is capable of solving many challenges + at once which causes an indexing issue within NginxConfigurator + who must return all responses in order. Imagine NginxConfigurator + maintaining state about where all of the http-01 Challenges, + TLS-SNI-01 Challenges belong in the response array. This is an + optional utility. + + """ + + def perform(self): + """Perform a challenge on Nginx. + + :returns: list of :class:`certbot.acme.challenges.HTTP01Response` + :rtype: list + + """ + if not self.achalls: + return [] + + responses = [x.response(x.account_key) for x in self.achalls] + + # Set up the configuration + self._mod_config() + + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def _add_bucket_directive(self): + """Modifies Nginx config to include server_names_hash_bucket_size directive.""" + root = self.configurator.parser.config_root + + bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] + + main = self.configurator.parser.parsed[root] + for line in main: + if line[0] == ['http']: + body = line[1] + found_bucket = False + posn = 0 + for inner_line in body: + if inner_line[0] == bucket_directive[1]: + if int(inner_line[1]) < int(bucket_directive[3]): + body[posn] = bucket_directive + found_bucket = True + posn += 1 + if not found_bucket: + body.insert(0, bucket_directive) + break + + def _mod_config(self): + """Modifies Nginx config to handle challenges. + + """ + self._add_bucket_directive() + + for achall in self.achalls: + self._mod_server_block(achall) + + def _get_validation_path(self, achall): + return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) + + def _mod_server_block(self, achall): + """Modifies a server block to respond to a challenge. + + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + """ + vhost = self.configurator.choose_redirect_vhost(achall.domain, + '%i' % self.configurator.config.http01_port, create_if_no_match=True) + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + location_directive = [[['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]] + + self.configurator.parser.add_server_directives(vhost, + location_directive, replace=False) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 9f13bc59f..5497f7e63 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -524,7 +524,7 @@ def _is_ssl_on_directive(entry): def _add_directives(directives, replace, block): """Adds or replaces directives in a config block. - When replace=False, it's an error to try and add a directive that already + When replace=False, it's an error to try and add a nonrepeatable directive that already exists in the config block with a conflicting value. When replace=True and a directive with the same name already exists in the @@ -545,7 +545,7 @@ def _add_directives(directives, replace, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e708b159a..7475df40c 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -100,7 +100,7 @@ class NginxConfiguratorTest(util.NginxTest): errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement') def test_get_chall_pref(self): - self.assertEqual([challenges.TLSSNI01], + self.assertEqual([challenges.TLSSNI01, challenges.HTTP01], self.config.get_chall_pref('myhost')) def test_save(self): @@ -291,9 +291,11 @@ class NginxConfiguratorTest(util.NginxTest): parsed_migration_conf[0]) @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform") + @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform") @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart") @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config") - def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform): + def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform, + mock_tls_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -304,7 +306,7 @@ class NginxConfiguratorTest(util.NginxTest): ), domain="localhost", account_key=self.rsa512jwk) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"), + chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", status=messages.Status("pending"), ), domain="example.com", account_key=self.rsa512jwk) @@ -314,10 +316,12 @@ class NginxConfiguratorTest(util.NginxTest): achall2.response(self.rsa512jwk), ] - mock_perform.return_value = expected + mock_tls_perform.return_value = expected[:1] + mock_http_perform.return_value = expected[1:] responses = self.config.perform([achall1, achall2]) - self.assertEqual(mock_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(mock_http_perform.call_count, 1) self.assertEqual(responses, expected) self.config.cleanup([achall1, achall2]) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py new file mode 100644 index 000000000..0f764e92e --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -0,0 +1,113 @@ +"""Tests for certbot_nginx.http_01""" +import unittest +import shutil + +import mock +import six + +from acme import challenges + +from certbot import achallenges + +from certbot.plugins import common_test +from certbot.tests import acme_util + +from certbot_nginx.tests import util + + +class HttpPerformTest(util.NginxTest): + """Test the NginxHttp01 challenge.""" + + account_key = common_test.AUTH_KEY + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), + domain="www.example.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01( + token=b"\xba\xa9\xda? Date: Mon, 15 Jan 2018 01:22:22 +0200 Subject: [PATCH 276/631] Use static directory under workdir for HTTP challenges (#5428) * Use static directory under workdir for HTTP challenges * Handle the reverter file registration before opening file handle --- certbot-apache/certbot_apache/configurator.py | 1 - certbot-apache/certbot_apache/http_01.py | 19 ++++++++----------- .../certbot_apache/tests/configurator_test.py | 5 ----- .../certbot_apache/tests/http_01_test.py | 6 +++--- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8d6995211..b87189dc0 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1956,7 +1956,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.revert_challenge_config() self.restart() self.parser.reset_modules() - self.http_doer.cleanup() def install_ssl_options_conf(self, options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 87721d290..3b942823d 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -1,8 +1,6 @@ """A class that performs HTTP-01 challenges for Apache""" import logging import os -import shutil -import tempfile from certbot.plugins import common @@ -35,7 +33,9 @@ Alias /.well-known/acme-challenge {0} self.challenge_conf = os.path.join( self.configurator.conf("challenge-location"), "le_http_01_challenge.conf") - self.challenge_dir = None + self.challenge_dir = os.path.join( + self.configurator.config.work_dir, + "http_challenges") def perform(self): """Perform all HTTP-01 challenges.""" @@ -56,12 +56,6 @@ Alias /.well-known/acme-challenge {0} return responses - def cleanup(self): - """Cleanup the challenge directory.""" - if self.challenge_dir: - shutil.rmtree(self.challenge_dir, ignore_errors=True) - self.challenge_dir = None - def prepare_http01_modules(self): """Make sure that we have the needed modules available for http01""" @@ -92,8 +86,9 @@ Alias /.well-known/acme-challenge {0} new_conf.write(config_text) def _set_up_challenges(self): - self.challenge_dir = tempfile.mkdtemp() - os.chmod(self.challenge_dir, 0o755) + if not os.path.isdir(self.challenge_dir): + os.makedirs(self.challenge_dir) + os.chmod(self.challenge_dir, 0o755) responses = [] for achall in self.achalls: @@ -105,6 +100,8 @@ Alias /.well-known/acme-challenge {0} response, validation = achall.response_and_validation() name = os.path.join(self.challenge_dir, achall.chall.encode("token")) + + self.configurator.reverter.register_file_creation(True, name) with open(name, 'wb') as f: f.write(validation.encode()) os.chmod(name, 0o644) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ea3867061..df67104da 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -747,7 +747,6 @@ class MultipleVhostsTest(util.ApacheTest): def test_cleanup(self, mock_cfg, mock_restart): mock_cfg.return_value = "" _, achalls = self.get_key_and_achalls() - self.config.http_doer = mock.MagicMock() for achall in achalls: self.config._chall_out.add(achall) # pylint: disable=protected-access @@ -756,10 +755,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.cleanup([achall]) if i == len(achalls) - 1: self.assertTrue(mock_restart.called) - self.assertTrue(self.config.http_doer.cleanup.called) else: self.assertFalse(mock_restart.called) - self.assertFalse(self.config.http_doer.cleanup.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") @@ -773,11 +770,9 @@ class MultipleVhostsTest(util.ApacheTest): self.config.cleanup([achalls[-1]]) self.assertFalse(mock_restart.called) - self.assertFalse(self.config.http_doer.cleanup.called) self.config.cleanup(achalls) self.assertTrue(mock_restart.called) - self.assertTrue(self.config.http_doer.cleanup.called) @mock.patch("certbot.util.run_script") def test_get_version(self, mock_script): diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 4e2a5faff..204d9a76c 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -100,6 +100,8 @@ class ApacheHttp01Test(util.ApacheTest): def common_perform_test(self, achalls): """Tests perform with the given achalls.""" + challenge_dir = self.http.challenge_dir + self.assertFalse(os.path.exists(challenge_dir)) for achall in achalls: self.http.add_chall(achall) @@ -114,9 +116,7 @@ class ApacheHttp01Test(util.ApacheTest): for achall in achalls: self._test_challenge_file(achall) - challenge_dir = self.http.challenge_dir - self.http.cleanup() - self.assertFalse(os.path.exists(challenge_dir)) + self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): self.assertEqual( From 368ca0c1096aa198b2e841dd52778646812e8141 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 Jan 2018 20:47:03 -0800 Subject: [PATCH 277/631] Small cleanup for Apache HTTP-01 * Remove http_doer from self * Refactor _find_best_vhost --- certbot-apache/certbot_apache/configurator.py | 29 +++++++++++-------- .../certbot_apache/tests/configurator_test.py | 3 +- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index b87189dc0..05c838123 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -164,9 +164,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} - # This will be set during the perform function - self.http_doer = None - @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" @@ -439,12 +436,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False - def _find_best_vhost(self, target_name): + def _find_best_vhost(self, target_name, vhosts=None): """Finds the best vhost for a target_name. This does not upgrade a vhost to HTTPS... it only finds the most appropriate vhost for the given target_name. + :param str target_name: domain handled by the desired vhost + :param vhosts: vhosts to consider + :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost` + :returns: VHost or None """ @@ -456,7 +457,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Points 1 - Address name with no SSL best_candidate = None best_points = 0 - for vhost in self.vhosts: + + if vhosts is None: + vhosts = self.vhosts + + for vhost in vhosts: if vhost.modmacro is True: continue names = vhost.get_names() @@ -481,7 +486,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # No winners here... is there only one reasonable vhost? if best_candidate is None: # reasonable == Not all _default_ addrs - vhosts = self._non_default_vhosts() + vhosts = self._non_default_vhosts(vhosts) # remove mod_macro hosts from reasonable vhosts reasonable_vhosts = [vh for vh in vhosts if vh.modmacro is False] @@ -490,9 +495,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return best_candidate - def _non_default_vhosts(self): + def _non_default_vhosts(self, vhosts): """Return all non _default_ only vhosts.""" - return [vh for vh in self.vhosts if not all( + return [vh for vh in vhosts if not all( addr.get_addr() == "_default_" for addr in vh.addrs )] @@ -1911,7 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self._chall_out.update(achalls) responses = [None] * len(achalls) - self.http_doer = http_01.ApacheHttp01(self) + http_doer = http_01.ApacheHttp01(self) sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): @@ -1919,11 +1924,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # challenge. This helps to put all of the responses back together # when they are all complete. if isinstance(achall.chall, challenges.HTTP01): - self.http_doer.add_chall(achall, i) + http_doer.add_chall(achall, i) else: # tls-sni-01 sni_doer.add_chall(achall, i) - http_response = self.http_doer.perform() + http_response = http_doer.perform() sni_response = sni_doer.perform() if http_response or sni_response: # Must reload in order to activate the challenges. @@ -1935,7 +1940,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # of identifying when the new configuration is being used. time.sleep(3) - self._update_responses(responses, http_response, self.http_doer) + self._update_responses(responses, http_response, http_doer) self._update_responses(responses, sni_response, sni_doer) return responses diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index df67104da..c846d2d42 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -305,7 +305,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_non_default_vhosts(self): # pylint: disable=protected-access - self.assertEqual(len(self.config._non_default_vhosts()), 8) + vhosts = self.config._non_default_vhosts(self.config.vhosts) + self.assertEqual(len(vhosts), 8) def test_deploy_cert_enable_new_vhost(self): # Create From 7e463bccad469c1fa7cda6aba09b1e459a514e2d Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 16 Jan 2018 14:58:45 -0800 Subject: [PATCH 278/631] Handle more edge cases for HTTP-01 support in Nginx (#5421) * only when using http01, only match default_server by port * import errors * put back in the code that creates a dummy block, but only when we can't find anything else --- certbot-nginx/certbot_nginx/configurator.py | 30 +++--- certbot-nginx/certbot_nginx/http_01.py | 107 ++++++++++++++++++-- 2 files changed, 113 insertions(+), 24 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index a77cf2bc3..bb2933a39 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -261,9 +261,9 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain): + def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost() + default_vhost = self._get_default_vhost(port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) self.new_vhost.names = set() @@ -278,15 +278,16 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.add_server_directives(vhost, name_block, replace=True) - def _get_default_vhost(self): + def _get_default_vhost(self, port): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - default_vhosts.append(vhost) - break + if port is None or self._port_matches(port, addr.get_port()): + default_vhosts.append(vhost) + break if len(default_vhosts) == 1: return default_vhosts[0] @@ -393,9 +394,17 @@ class NginxConfigurator(common.Installer): matches = self._get_redirect_ranked_matches(target_name, port) vhost = self._select_best_name_match(matches) if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + vhost = self._vhost_from_duplicated_default(target_name, port=port) return vhost + def _port_matches(self, test_port, matching_port): + # test_port is a number, matching is a number or "" or None + if matching_port == "" or matching_port is None: + # if no port is specified, Nginx defaults to listening on port 80. + return test_port == self.DEFAULT_LISTEN_PORT + else: + return test_port == matching_port + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -410,13 +419,6 @@ class NginxConfigurator(common.Installer): """ all_vhosts = self.parser.get_vhosts() - def _port_matches(test_port, matching_port): - # test_port is a number, matching is a number or "" or None - if matching_port == "" or matching_port is None: - # if no port is specified, Nginx defaults to listening on port 80. - return test_port == self.DEFAULT_LISTEN_PORT - else: - return test_port == matching_port def _vhost_matches(vhost, port): found_matching_port = False @@ -426,7 +428,7 @@ class NginxConfigurator(common.Installer): found_matching_port = (port == self.DEFAULT_LISTEN_PORT) else: for addr in vhost.addrs: - if _port_matches(port, addr.get_port()) and addr.ssl == False: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: found_matching_port = True if found_matching_port: diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 1f1e37891..c25081ae0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -5,8 +5,12 @@ import os from acme import challenges +from certbot import errors from certbot.plugins import common +from certbot_nginx import obj +from certbot_nginx import nginxparser + logger = logging.getLogger(__name__) @@ -31,6 +35,13 @@ class NginxHttp01(common.ChallengePerformer): """ + def __init__(self, configurator): + super(NginxHttp01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_http_01_cert_challenge.conf") + self._ipv6 = None + self._ipv6only = None + def perform(self): """Perform a challenge on Nginx. @@ -51,8 +62,16 @@ class NginxHttp01(common.ChallengePerformer): return responses - def _add_bucket_directive(self): - """Modifies Nginx config to include server_names_hash_bucket_size directive.""" + def _mod_config(self): + """Modifies Nginx config to include server_names_hash_bucket_size directive + and server challenge blocks. + + :raises .MisconfigurationError: + Unable to find a suitable HTTP block in which to include + authenticator hosts. + """ + included = False + include_directive = ['\n', 'include', ' ', self.challenge_conf] root = self.configurator.parser.config_root bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] @@ -71,21 +90,82 @@ class NginxHttp01(common.ChallengePerformer): posn += 1 if not found_bucket: body.insert(0, bucket_directive) + if include_directive not in body: + body.insert(0, include_directive) + included = True break + if not included: + raise errors.MisconfigurationError( + 'Certbot could not find a block to include ' + 'challenges in %s.' % root) + config = [self._make_or_mod_server_block(achall) for achall in self.achalls] + config = [x for x in config if x is not None] + config = nginxparser.UnspacedList(config) - def _mod_config(self): - """Modifies Nginx config to handle challenges. + self.configurator.reverter.register_file_creation( + True, self.challenge_conf) + with open(self.challenge_conf, "w") as new_conf: + nginxparser.dump(config, new_conf) + + def _default_listen_addresses(self): + """Finds addresses for a challenge block to listen on. + :returns: list of :class:`certbot_nginx.obj.Addr` to apply + :rtype: list """ - self._add_bucket_directive() + addresses = [] + default_addr = "%s" % self.configurator.config.http01_port + ipv6_addr = "[::]:{0}".format( + self.configurator.config.http01_port) + port = self.configurator.config.http01_port - for achall in self.achalls: - self._mod_server_block(achall) + if self._ipv6 is None or self._ipv6only is None: + self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) + ipv6, ipv6only = self._ipv6, self._ipv6only + + if ipv6: + # If IPv6 is active in Nginx configuration + if not ipv6only: + # If ipv6only=on is not already present in the config + ipv6_addr = ipv6_addr + " ipv6only=on" + addresses = [obj.Addr.fromstring(default_addr), + obj.Addr.fromstring(ipv6_addr)] + logger.info(("Using default addresses %s and %s for authentication."), + default_addr, + ipv6_addr) + else: + addresses = [obj.Addr.fromstring(default_addr)] + logger.info("Using default address %s for authentication.", + default_addr) + return addresses def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _mod_server_block(self, achall): + def _make_server_block(self, achall): + """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :param list addrs: addresses of challenged domain + :class:`list` of type :class:`~nginx.obj.Addr` + :returns: server block for the challenge host + :rtype: list + """ + addrs = self._default_listen_addresses() + block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs] + + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + block.extend([['server_name', ' ', achall.domain], + [['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]]) + # TODO: do we want to return something else if they otherwise access this block? + return [['server'], block] + + def _make_or_mod_server_block(self, achall): """Modifies a server block to respond to a challenge. :param achall: Annotated HTTP-01 challenge @@ -93,8 +173,15 @@ class NginxHttp01(common.ChallengePerformer): :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` """ - vhost = self.configurator.choose_redirect_vhost(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) + try: + vhost = self.configurator.choose_redirect_vhost(achall.domain, + '%i' % self.configurator.config.http01_port, create_if_no_match=True) + except errors.MisconfigurationError: + # Couldn't find either a matching name+port server block + # or a port+default_server block, so create a dummy block + return self._make_server_block(achall) + + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) From 314c5f19e51ccefc740f87751c589e2ca0fe9754 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 16 Jan 2018 20:33:25 +0200 Subject: [PATCH 279/631] Set up vhost discovery and overrides for HTTP-01 * Finalized HTTP vhost discovery and added overrides * Include overrides to every VirtualHost --- certbot-apache/certbot_apache/configurator.py | 14 ++++- certbot-apache/certbot_apache/http_01.py | 60 +++++++++++++------ certbot-apache/certbot_apache/parser.py | 17 ++++++ .../certbot_apache/tests/http_01_test.py | 6 +- .../certbot_apache/tests/parser_test.py | 17 ++++++ 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 05c838123..8f1aff8d7 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -436,6 +436,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False + def find_best_http_vhost(self, target): + """Returns non-HTTPS vhost objects found from the Apache config + + :param str target: Domain name of the desired VirtualHost + + :returns: VirtualHost object that's the best match for target name + :rtype: `obj.VirtualHost` or None + """ + nonssl_vhosts = [i for i in self.vhosts if not i.ssl] + return self._find_best_vhost(target, nonssl_vhosts) + + def _find_best_vhost(self, target_name, vhosts=None): """Finds the best vhost for a target_name. @@ -508,7 +520,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): virtual host addresses :rtype: set - """ + """ all_names = set() vhost_macro = [] diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 3b942823d..a2cd43845 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -7,26 +7,38 @@ from certbot.plugins import common logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): - """Class that performs HTPP-01 challenges within the Apache configurator.""" + """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE24 = """\ -Alias /.well-known/acme-challenge {0} + CONFIG_TEMPLATE_COMMON = """\ + Alias /.well-known/acme-challenge {0}" - - Require all granted - - -""" + + ProxyPass "/.well-known/acme-challenge" ! + + """ CONFIG_TEMPLATE22 = """\ -Alias /.well-known/acme-challenge {0} + + RewriteEngine on + RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999] + - - Order allow,deny - Allow from all - + + Order allow deny + Allow from all + + """ -""" + CONFIG_TEMPLATE24 = """\ + + RewriteEngine on + RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END] + + + + Require all granted + + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) @@ -50,6 +62,7 @@ Alias /.well-known/acme-challenge {0} self.prepare_http01_modules() responses = self._set_up_challenges() + self._mod_config() # Save reversible changes self.configurator.save("HTTP Challenge", True) @@ -70,21 +83,26 @@ Alias /.well-known/acme-challenge {0} self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - self.configurator.parser.add_include( - self.configurator.parser.loc["default"], self.challenge_conf) + for chall in self.achalls: + vh = self.configurator.find_best_http_vhost(chall.domain) + if vh: + self._set_up_include_directive(vh) + self.configurator.reverter.register_file_creation( True, self.challenge_conf) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22 else: - config_template = self.CONFIG_TEMPLATE24 + config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24 + config_text = config_template.format(self.challenge_dir) logger.debug("writing a config file with text:\n %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) + def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): os.makedirs(self.challenge_dir) @@ -107,3 +125,9 @@ Alias /.well-known/acme-challenge {0} os.chmod(name, 0o644) return response + + def _set_up_include_directive(self, vhost): + """Includes override configuration to the beginning of VirtualHost. + Note that this include isn't added to Augeas search tree""" + self.configurator.parser.add_dir_beginning(vhost.path, "Include", + self.challenge_conf) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 7715d2c35..d7da1e55e 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -332,6 +332,23 @@ class ApacheParser(object): else: self.aug.set(aug_conf_path + "/directive[last()]/arg", args) + def add_dir_beginning(self, aug_conf_path, dirname, args): + """Adds the directive to the beginning of defined aug_conf_path. + + :param str aug_conf_path: Augeas configuration path to add directive + :param str dirname: Directive to add + :param args: Value of the directive. ie. Listen 443, 443 is arg + :type args: list or str + """ + first_dir = aug_conf_path + "/directive[1]" + self.aug.insert(first_dir, "directive", True) + self.aug.set(first_dir, dirname) + if isinstance(args, list): + for i, value in enumerate(args, 1): + self.aug.set(first_dir + "/arg[%d]" % (i), value) + else: + self.aug.set(first_dir + "/arg", args) + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 204d9a76c..54b4ff208 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -119,9 +119,9 @@ class ApacheHttp01Test(util.ApacheTest): self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - self.assertEqual( - len(self.config.parser.find_dir( - "Include", self.http.challenge_conf)), 1) + #self.assertEqual( + # len(self.config.parser.find_dir( + # "Include", self.http.challenge_conf)), 1) with open(self.http.challenge_conf) as f: conf_contents = f.read() diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index a9eb129c2..4496781c9 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest): for i, match in enumerate(matches): self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_add_dir_beginning(self): + aug_default = "/files" + self.parser.loc["default"] + self.parser.add_dir_beginning(aug_default, + "AddDirectiveBeginning", + "testBegin") + + self.assertTrue( + self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default)) + + self.assertEqual( + self.parser.aug.get(aug_default+"/directive[1]"), + "AddDirectiveBeginning") + self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"]) + matches = self.parser.find_dir("AddList", None, aug_default) + for i, match in enumerate(matches): + self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_empty_arg(self): self.assertEquals(None, self.parser.get_arg("/files/whatever/nonexistent")) From f420b19492075b569eea349d5c6a4ec9bafa236f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Jan 2018 18:16:33 -0800 Subject: [PATCH 280/631] Apache HTTP01 Improvements * Fix docstring quote spacing * Remove unneeded directives * Enable mod_rewrite * Remove ifmod rewrite * Use stricter rewriterule * Uncomment tests * Fix order args * Remove S which doesn't seem to work across contexts * Use double backslash to make pylint * Fix enmod test * Fix http-01 tests * Test for rewrite * check for Include in vhost * add test_same_vhost * Don't add includes twice * Include default vhosts in search * Respect port in find_best_http_vhost * Add find_best_http_vhost port test * Filter by port in http01 --- certbot-apache/certbot_apache/configurator.py | 23 ++++--- certbot-apache/certbot_apache/http_01.py | 36 ++++------ .../certbot_apache/tests/configurator_test.py | 18 ++++- .../certbot_apache/tests/http_01_test.py | 67 ++++++++++++++----- .../apache2/sites-available/certbot.conf | 1 + certbot-apache/certbot_apache/tests/util.py | 2 +- 6 files changed, 97 insertions(+), 50 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8f1aff8d7..b32eda921 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -436,19 +436,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False - def find_best_http_vhost(self, target): + def find_best_http_vhost(self, target, filter_defaults, port="80"): """Returns non-HTTPS vhost objects found from the Apache config :param str target: Domain name of the desired VirtualHost + :param bool filter_defaults: whether _default_ vhosts should be + included if it is the best match + :param str port: port number the vhost should be listening on :returns: VirtualHost object that's the best match for target name :rtype: `obj.VirtualHost` or None """ - nonssl_vhosts = [i for i in self.vhosts if not i.ssl] - return self._find_best_vhost(target, nonssl_vhosts) + filtered_vhosts = [] + for vhost in self.vhosts: + if any(a.is_wildcard() or a.get_port() == port for a in vhost.addrs) and not vhost.ssl: + filtered_vhosts.append(vhost) + return self._find_best_vhost(target, filtered_vhosts, filter_defaults) - - def _find_best_vhost(self, target_name, vhosts=None): + def _find_best_vhost(self, target_name, vhosts=None, filter_defaults=True): """Finds the best vhost for a target_name. This does not upgrade a vhost to HTTPS... it only finds the most @@ -457,6 +462,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str target_name: domain handled by the desired vhost :param vhosts: vhosts to consider :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost` + :param bool filter_defaults: whether a vhost with a _default_ + addr is acceptable :returns: VHost or None @@ -497,8 +504,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # No winners here... is there only one reasonable vhost? if best_candidate is None: - # reasonable == Not all _default_ addrs - vhosts = self._non_default_vhosts(vhosts) + if filter_defaults: + vhosts = self._non_default_vhosts(vhosts) # remove mod_macro hosts from reasonable vhosts reasonable_vhosts = [vh for vh in vhosts if vh.modmacro is False] @@ -520,7 +527,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): virtual host addresses :rtype: set - """ + """ all_names = set() vhost_macro = [] diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index a2cd43845..edcca8c9c 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -9,31 +9,19 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE_COMMON = """\ - Alias /.well-known/acme-challenge {0}" - - - ProxyPass "/.well-known/acme-challenge" ! - - """ - CONFIG_TEMPLATE22 = """\ - - RewriteEngine on - RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999] - + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] - Order allow deny + Order Allow,Deny Allow from all """ CONFIG_TEMPLATE24 = """\ - - RewriteEngine on - RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END] - + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] Require all granted @@ -73,7 +61,7 @@ class ApacheHttp01(common.TLSSNI01): """Make sure that we have the needed modules available for http01""" if self.configurator.conf("handle-modules"): - needed_modules = ["alias"] + needed_modules = ["rewrite"] if self.configurator.version < (2, 4): needed_modules.append("authz_host") else: @@ -83,18 +71,22 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): + moded_vhosts = set() for chall in self.achalls: - vh = self.configurator.find_best_http_vhost(chall.domain) - if vh: + vh = self.configurator.find_best_http_vhost( + chall.domain, filter_defaults=False, + port=str(self.configurator.config.http01_port)) + if vh and vh not in moded_vhosts: self._set_up_include_directive(vh) + moded_vhosts.add(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22 + config_template = self.CONFIG_TEMPLATE22 else: - config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24 + config_template = self.CONFIG_TEMPLATE24 config_text = config_template.format(self.challenge_dir) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index c846d2d42..530d75a92 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -126,7 +126,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() self.assertEqual(names, set( ["certbot.demo", "ocspvhost.com", "encryption-example.demo", - "nonsym.link", "vhost.in.rootconf"] + "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"] )) @certbot_util.patch_get_utility() @@ -146,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() # Names get filtered, only 5 are returned - self.assertEqual(len(names), 7) + self.assertEqual(len(names), 8) self.assertTrue("zombo.com" in names) self.assertTrue("google.com" in names) self.assertTrue("certbot.demo" in names) @@ -260,6 +260,20 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.PluginError, self.config.choose_vhost, "none.com") + def test_find_best_http_vhost_default(self): + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + self.config.vhosts = [vh] + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) + + def test_find_best_http_vhost_port(self): + port = "8080" + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + False, True, "encryption-example.demo") + self.config.vhosts.append(vh) + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) + def test_findbest_continues_on_short_domain(self): # pylint: disable=protected-access chosen_vhost = self.config._find_best_vhost("purple.com") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 54b4ff208..768d904e8 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -22,8 +22,9 @@ class ApacheHttp01TestMeta(type): def _gen_test(num_achalls, minor_version): def _test(self): achalls = self.achalls[:num_achalls] + vhosts = self.vhosts[:num_achalls] self.config.version = (2, minor_version) - self.common_perform_test(achalls) + self.common_perform_test(achalls, vhosts) return _test for i in range(1, NUM_ACHALLS + 1): @@ -43,16 +44,30 @@ class ApacheHttp01Test(util.ApacheTest): self.account_key = self.rsa512jwk self.achalls = [] + self.vhosts = [] + vhost_index = 0 for i in range(NUM_ACHALLS): + domain = None + # Find a vhost with a name/alias we can use + for j in range(vhost_index + 1, len(self.config.vhosts)): + vhost = self.config.vhosts[j] + domain = vhost.name if vhost.name else next(iter(vhost.aliases), None) + if domain: + self.vhosts.append(vhost) + vhost_index = j + 1 + break + else: # pragma: no cover + # If we didn't find a domain, we shouldn't continue the test. + self.fail("No usable vhost found") + self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.HTTP01(token=((chr(ord('a') + i) * 16))), "pending"), - domain="example{0}.com".format(i), - account_key=self.account_key)) + domain=domain, account_key=self.account_key)) - modules = ["alias", "authz_core", "authz_host"] + modules = ["rewrite", "authz_core", "authz_host"] for mod in modules: self.config.parser.modules.add("mod_{0}.c".format(mod)) self.config.parser.modules.add(mod + "_module") @@ -81,9 +96,9 @@ class ApacheHttp01Test(util.ApacheTest): self.assertEqual(enmod_calls[0][0][0], "authz_core") def common_enable_modules_test(self, mock_enmod): - """Tests enabling mod_alias and other modules.""" - self.config.parser.modules.remove("alias_module") - self.config.parser.modules.remove("mod_alias.c") + """Tests enabling mod_rewrite and other modules.""" + self.config.parser.modules.remove("rewrite_module") + self.config.parser.modules.remove("mod_rewrite.c") self.http.prepare_http01_modules() @@ -91,14 +106,30 @@ class ApacheHttp01Test(util.ApacheTest): calls = mock_enmod.call_args_list other_calls = [] for call in calls: - if "alias" != call[0][0]: + if "rewrite" != call[0][0]: other_calls.append(call) - # If these lists are equal, we never enabled mod_alias + # If these lists are equal, we never enabled mod_rewrite self.assertNotEqual(calls, other_calls) return other_calls - def common_perform_test(self, achalls): + def test_same_vhost(self): + vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo") + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain=vhost.name, account_key=self.account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'b' * 16))), + "pending"), + domain=next(iter(vhost.aliases)), account_key=self.account_key) + ] + self.common_perform_test(achalls, [vhost]) + + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir self.assertFalse(os.path.exists(challenge_dir)) @@ -116,19 +147,21 @@ class ApacheHttp01Test(util.ApacheTest): for achall in achalls: self._test_challenge_file(achall) + for vhost in vhosts: + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf, + vhost.path) + self.assertEqual(len(matches), 1) + self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - #self.assertEqual( - # len(self.config.parser.find_dir( - # "Include", self.http.challenge_conf)), 1) - with open(self.http.challenge_conf) as f: conf_contents = f.read() - alias_fmt = "Alias /.well-known/acme-challenge {0}" - alias = alias_fmt.format(self.http.challenge_dir) - self.assertTrue(alias in conf_contents) + self.assertTrue("RewriteEngine on" in conf_contents) + self.assertTrue("RewriteRule" in conf_contents) + self.assertTrue(self.http.challenge_dir in conf_contents) if self.config.version < (2, 4): self.assertTrue("Allow from all" in conf_contents) else: diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf index b3147a523..965ca2222 100644 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf @@ -1,5 +1,6 @@ ServerName certbot.demo +ServerAlias www.certbot.demo ServerAdmin webmaster@localhost DocumentRoot /var/www-certbot-reworld/static/ diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 1ba1e2c34..1daaa00c5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -170,7 +170,7 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - "certbot.demo"), + "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, From b8f288a3720fda4f2f1ef56ada30f887f5b41a63 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 17 Jan 2018 14:08:45 +0200 Subject: [PATCH 281/631] Add include to every VirtualHost if definite one not found based on name --- certbot-apache/certbot_apache/http_01.py | 19 ++++++++++++++----- .../certbot_apache/tests/http_01_test.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index edcca8c9c..2b2b8e796 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -36,6 +36,7 @@ class ApacheHttp01(common.TLSSNI01): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") + self.moded_vhosts = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -71,14 +72,16 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - moded_vhosts = set() for chall in self.achalls: vh = self.configurator.find_best_http_vhost( chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) - if vh and vh not in moded_vhosts: + if vh: self._set_up_include_directive(vh) - moded_vhosts.add(vh) + else: + for vh in self.configurator.vhosts: + if not vh.ssl: + self._set_up_include_directive(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -121,5 +124,11 @@ class ApacheHttp01(common.TLSSNI01): def _set_up_include_directive(self, vhost): """Includes override configuration to the beginning of VirtualHost. Note that this include isn't added to Augeas search tree""" - self.configurator.parser.add_dir_beginning(vhost.path, "Include", - self.challenge_conf) + + if vhost not in self.moded_vhosts: + logger.debug( + "Adding a temporary challenge validation Include for name: %s " + + "in: %s", vhost.name, vhost.filep) + self.configurator.parser.add_dir_beginning( + vhost.path, "Include", self.challenge_conf) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 768d904e8..12f571354 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -129,6 +129,16 @@ class ApacheHttp01Test(util.ApacheTest): ] self.common_perform_test(achalls, [vhost]) + def test_anonymous_vhost(self): + vhosts = [v for v in self.config.vhosts if not v.ssl] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="something.nonexistent", account_key=self.account_key)] + self.common_perform_test(achalls, vhosts) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 2c379cd363143720644ff38eccc85a349c2f2b00 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Jan 2018 08:01:44 -0800 Subject: [PATCH 282/631] Add a rewrite directive for the .well-known location so we don't hit existing rewrites (#5436) --- certbot-nginx/certbot_nginx/http_01.py | 5 +++++ certbot-nginx/certbot_nginx/parser.py | 28 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c25081ae0..2885e7ac0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -191,3 +191,8 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.parser.add_server_directives(vhost, location_directive, replace=False) + + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, replace=False, insert_at_top=True) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 5497f7e63..fbd6c0ade 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -276,7 +276,7 @@ class NginxParser(object): return False - def add_server_directives(self, vhost, directives, replace): + def add_server_directives(self, vhost, directives, replace, insert_at_top=False): """Add or replace directives in the server block identified by vhost. This method modifies vhost to be fully consistent with the new directives. @@ -293,10 +293,12 @@ class NginxParser(object): whose information we use to match on :param list directives: The directives to add :param bool replace: Whether to only replace existing directives + :param bool insert_at_top: True if the directives need to be inserted at the top + of the server block instead of the bottom """ self._modify_server_directives(vhost, - functools.partial(_add_directives, directives, replace)) + functools.partial(_add_directives, directives, replace, insert_at_top)) def remove_server_directives(self, vhost, directive_name, match_func=None): """Remove all directives of type directive_name. @@ -521,7 +523,7 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(directives, replace, block): +def _add_directives(directives, replace, insert_at_top, block): """Adds or replaces directives in a config block. When replace=False, it's an error to try and add a nonrepeatable directive that already @@ -535,17 +537,18 @@ def _add_directives(directives, replace, block): :param list directives: The new directives. :param bool replace: Described above. + :param bool insert_at_top: Described above. :param list block: The block to replace in """ for directive in directives: - _add_directive(block, directive, replace) + _add_directive(block, directive, replace, insert_at_top) if block and '\n' not in block[-1]: # could be " \n " or ["\n"] ! block.append(nginxparser.UnspacedList('\n')) INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] @@ -597,7 +600,7 @@ def _find_location(block, directive_name, match_func=None): return next((index for index, line in enumerate(block) \ if line and line[0] == directive_name and (match_func is None or match_func(line))), None) -def _add_directive(block, directive, replace): +def _add_directive(block, directive, replace, insert_at_top): """Adds or replaces a single directive in a config block. See _add_directives for more documentation. @@ -619,7 +622,7 @@ def _add_directive(block, directive, replace): block[location] = directive comment_directive(block, location) return - # Append directive. Fail if the name is not a repeatable directive name, + # Append or prepend directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. @@ -652,8 +655,15 @@ def _add_directive(block, directive, replace): _comment_out_directive(block, included_dir_loc, directive[1]) if can_append(location, directive_name): - block.append(directive) - comment_directive(block, len(block) - 1) + if insert_at_top: + # Add a newline so the comment doesn't comment + # out existing directives + block.insert(0, nginxparser.UnspacedList('\n')) + block.insert(0, directive) + comment_directive(block, 0) + else: + block.append(directive) + comment_directive(block, len(block) - 1) elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) From e9b57e17833e19d36158ee7c1f9b7c684f118027 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Jan 2018 08:02:10 -0800 Subject: [PATCH 283/631] Add (nonexistent) document root so we don't use the default value (#5437) --- certbot-nginx/certbot_nginx/http_01.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 2885e7ac0..c0dec061a 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -155,10 +155,15 @@ class NginxHttp01(common.ChallengePerformer): addrs = self._default_listen_addresses() block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs] + # Ensure we 404 on any other request by setting a root + document_root = os.path.join( + self.configurator.config.work_dir, "http_01_nonexistent") + validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) block.extend([['server_name', ' ', achall.domain], + ['root', ' ', document_root], [['location', ' ', '=', ' ', validation_path], [['default_type', ' ', 'text/plain'], ['return', ' ', '200', ' ', validation]]]]) From bd231a3855c03d2943a2e3fe117605b4962726a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 09:27:36 -0800 Subject: [PATCH 284/631] Error without vhosts and fix tests token type --- certbot-apache/certbot_apache/http_01.py | 20 ++++++++++++++++--- .../certbot_apache/tests/http_01_test.py | 9 ++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 2b2b8e796..4aa3215e6 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -2,6 +2,8 @@ import logging import os +from certbot import errors + from certbot.plugins import common logger = logging.getLogger(__name__) @@ -79,9 +81,8 @@ class ApacheHttp01(common.TLSSNI01): if vh: self._set_up_include_directive(vh) else: - for vh in self.configurator.vhosts: - if not vh.ssl: - self._set_up_include_directive(vh) + for vh in self._relevant_vhosts(): + self._set_up_include_directive(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -97,6 +98,19 @@ class ApacheHttp01(common.TLSSNI01): with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) + def _relevant_vhosts(self): + http01_port = str(self.configurator.config.http01_port) + relevant_vhosts = [] + for vhost in self.configurator.vhosts: + if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs): + if not vhost.ssl: + relevant_vhosts.append(vhost) + if not relevant_vhosts: + raise errors.PluginError( + "Unable to find a virtual host listening on port {0}." + " Please add one.".format(http01_port)) + + return relevant_vhosts def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 12f571354..652b65a09 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -6,6 +6,7 @@ import unittest from acme import challenges from certbot import achallenges +from certbot import errors from certbot.tests import acme_util @@ -63,7 +64,7 @@ class ApacheHttp01Test(util.ApacheTest): self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( - challenges.HTTP01(token=((chr(ord('a') + i) * 16))), + challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))), "pending"), domain=domain, account_key=self.account_key)) @@ -139,6 +140,12 @@ class ApacheHttp01Test(util.ApacheTest): domain="something.nonexistent", account_key=self.account_key)] self.common_perform_test(achalls, vhosts) + def test_no_vhost(self): + for achall in self.achalls: + self.http.add_chall(achall) + self.config.config.http01_port = 12345 + self.assertRaises(errors.PluginError, self.http.perform) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 63136be2e55473a12a0de36ec6eaa72e9cd0ab70 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 17 Jan 2018 20:07:38 +0200 Subject: [PATCH 285/631] Make sure the HTTP tests do not use wrong vhosts for asserts --- certbot-apache/certbot_apache/tests/http_01_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 12f571354..af32697d1 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -161,7 +161,8 @@ class ApacheHttp01Test(util.ApacheTest): matches = self.config.parser.find_dir("Include", self.http.challenge_conf, vhost.path) - self.assertEqual(len(matches), 1) + if not vhost.ssl: + self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) From 522532dc300d4f6921320d1a56167b5b6372b562 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 10:33:51 -0800 Subject: [PATCH 286/631] Improve no vhost error message --- certbot-apache/certbot_apache/http_01.py | 6 ++++-- certbot-apache/certbot_apache/tests/http_01_test.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 4aa3215e6..e463f3880 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -107,8 +107,10 @@ class ApacheHttp01(common.TLSSNI01): relevant_vhosts.append(vhost) if not relevant_vhosts: raise errors.PluginError( - "Unable to find a virtual host listening on port {0}." - " Please add one.".format(http01_port)) + "Unable to find a virtual host listening on port {0} which is" + " currently needed for Certbot to prove to the CA that you" + " control your domain. Please add a virtual host for port" + " {0}.".format(http01_port)) return relevant_vhosts diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 00e05e9fc..ee8566396 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -165,10 +165,10 @@ class ApacheHttp01Test(util.ApacheTest): self._test_challenge_file(achall) for vhost in vhosts: - matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, - vhost.path) if not vhost.ssl: + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf, + vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) From 1bb2cfadf7d18ee01f70ebb3b34cedf356946837 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 15:34:34 -0800 Subject: [PATCH 287/631] hardcode vhosts and names for test (#5444) --- .../certbot_apache/tests/http_01_test.py | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index ee8566396..64a76649a 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -45,28 +45,19 @@ class ApacheHttp01Test(util.ApacheTest): self.account_key = self.rsa512jwk self.achalls = [] - self.vhosts = [] - vhost_index = 0 - for i in range(NUM_ACHALLS): - domain = None - # Find a vhost with a name/alias we can use - for j in range(vhost_index + 1, len(self.config.vhosts)): - vhost = self.config.vhosts[j] - domain = vhost.name if vhost.name else next(iter(vhost.aliases), None) - if domain: - self.vhosts.append(vhost) - vhost_index = j + 1 - break - else: # pragma: no cover - # If we didn't find a domain, we shouldn't continue the test. - self.fail("No usable vhost found") + vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multiple_vhosts") + # Takes the vhosts for encryption-example.demo, certbot.demo, and + # vhost.in.rootconf + self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]] + for i in range(NUM_ACHALLS): self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))), "pending"), - domain=domain, account_key=self.account_key)) + domain=self.vhosts[i].name, account_key=self.account_key)) modules = ["rewrite", "authz_core", "authz_host"] for mod in modules: From bf695d048dd1d3b14b1d521b5940d3c9a635fd64 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 15:55:29 -0800 Subject: [PATCH 288/631] Release 0.21.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 307 +++++++++++++----- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 10 +- letsencrypt-auto | 307 +++++++++++++----- letsencrypt-auto-source/certbot-auto.asc | 14 +- letsencrypt-auto-source/letsencrypt-auto | 26 +- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +- 22 files changed, 494 insertions(+), 224 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7ebdfe46e..a3e5fd744 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3270f2c79..6a874c813 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index 444bee1b9..558c330b2 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -816,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1092,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1431,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1466,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1503,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1529,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1537,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1604,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1faf30643..1015fec47 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 428271045..f7f1b0b38 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4a103193f..cd6f9458e 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 23098d4b6..35659acac 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4ed5a06ca..447a1c1f4 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a0b88aab..0def6d7cb 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b00bd1ac3..d51b9271b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b8f50254e..35c018706 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2a388e487..027b3bbae 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 78007afb5..5fb54922a 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 7d1eb0bc9..882834852 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 2ad7aaf08..b5585bf42 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index cbea701ee..b0a212646 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.21.0.dev0' +__version__ = '0.21.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index abaa95b9b..f7318f0b3 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,7 +107,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.20.0 (certbot; + "". (default: CertbotACMEClient/0.21.0 (certbot; Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags encoded in the user agent are: --duplicate, --force- @@ -331,6 +331,14 @@ revoke: --reason {unspecified,keycompromise,affiliationchanged,superseded,cessationofoperation} Specify reason for revoking certificate. (default: unspecified) + --delete-after-revoke + Delete certificates after revoking them. (default: + None) + --no-delete-after-revoke + Do not delete certificates after revoking them. This + option should be used with caution because the 'renew' + subcommand will attempt to renew undeleted revoked + certificates. (default: None) register: Options for account registration & modification diff --git a/letsencrypt-auto b/letsencrypt-auto index 444bee1b9..558c330b2 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -816,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1092,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1431,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1466,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1503,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1529,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1537,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1604,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index eeab78cd6..9df8013a0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt -XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf -p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX -OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4 -X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI -/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE= -=Is56 +iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n +e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW +AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL +RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn +IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf +yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= +=d/hS -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b86517bb6..558c330b2 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0.dev0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1190,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index e276aae5353e8c5bdf78fc91950e60649baa241f..2e52f03fdf9a1fbe2049d89f071a583bcf919976 100644 GIT binary patch literal 256 zcmV+b0ssD{xCAf=x;KV2@d03ctAnF7mn+aqUc{p=)VXkyhWU!~cN;%G^8WY$S?ta1 z2hmQE4T`~+Zwq%*K~cAA%;XQy8S-ox4a$2d0@tMjK|wQYPjC$Cv}rUZ3(4Cmz7$?G zU;e#Fi7%+uTC73=Hbzf*EfB0nOu(b3_b6CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&* zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz GZCemMz+EWKAt%U!XWg-L;RUoOlOZsAFb4Ic}LGE#!b+HMXRo%-QglHm~xc z7rA=rPZ17(d9O44~^dV!y) z3$?Ok9e8=A1v3-fBldYu$u&26@w%#BftaFQ;t)7s73=9Vm2g;i>-a0vSdi>_B;&sD z-BgJaKSacB+~@MWWUpduDsx8T Date: Wed, 17 Jan 2018 15:55:41 -0800 Subject: [PATCH 289/631] Bump version to 0.22.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index a3e5fd744..ce426cf74 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 6a874c813..38f41e9f1 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1015fec47..8f9f897cf 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index f7f1b0b38..612e7259f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index cd6f9458e..3157400c6 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 35659acac..1a68400fa 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 447a1c1f4..35de47308 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0def6d7cb..a946d00a4 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d51b9271b..8585fc848 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 35c018706..4fec37e29 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 027b3bbae..dca9ebf27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5fb54922a..bfa72b50b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 882834852..8df687972 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b5585bf42..152f77de8 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index b0a212646..2869d29b0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.21.0' +__version__ = '0.22.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 558c330b2..9c8ccf344 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.22.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 103039ca40c3a300816c5a957a0bf13a033234de Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 17:46:56 -0800 Subject: [PATCH 290/631] Add 0.21.0 changelog --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfc0401..f6f7be9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.0 - 2018-01-17 + +### Added + +* Support for the HTTP-01 challenge type was added to our Apache and Nginx + plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge + type which was what was previously being used by our Apache and Nginx plugins + last week due to a security issue. For more information about Let's Encrypt's + change, click + [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188). + Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no + changes need to be made to your Certbot configuration, however, you should + make sure your server is accessible on port 80 and isn't behind an external + proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to + HTTPS redirects inside Apache and Nginx are fine. +* IPv6 support was added to the Nginx plugin. +* Support for automatically creating server blocks based on the default server + block was added to the Nginx plugin. +* The flags --delete-after-revoke and --no-delete-after-revoke were added + allowing users to control whether the revoke subcommand also deletes the + certificates it is revoking. + +### Changed + +* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME + library. Support for these versions of Python will be removed in the next + major release of Certbot. If you are using certbot-auto on a RHEL 6 based + system, it will guide you through the process of installing Python 3. +* We split our implementation of JOSE (Javascript Object Signing and + Encryption) out of our ACME library and into a separate package named josepy. + This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and + on [GitHub](https://github.com/certbot/josepy). +* We updated the ciphersuites used in Apache to the new [values recommended by + Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29). + The major change here is adding ChaCha20 to the list of supported + ciphersuites. + +### Fixed + +* An issue with our Apache plugin on Gentoo due to differences in their + apache2ctl command have been resolved. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/47?closed=1 + ## 0.20.0 - 2017-12-06 ### Added From b0aa8b7c0bcddb0837d9a57d021152bc05473b8e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 24 Jan 2018 02:46:36 +0200 Subject: [PATCH 291/631] Work around Basic Authentication for challenge dir in Apache (#5461) Unfortunately, the way that Apache merges the configuration directives is different for mod_rewrite and / directives. To work around basic auth in VirtualHosts, the challenge override Include had to be split in two. The first part handles overrides for RewriteRule and the other part will handle overrides for and directives. --- certbot-apache/certbot_apache/http_01.py | 60 +++++++++++++------ .../certbot_apache/tests/http_01_test.py | 24 +++++--- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index e463f3880..cce93a646 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -11,30 +11,43 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE22 = """\ + CONFIG_TEMPLATE22_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] + """ + CONFIG_TEMPLATE22_POST = """\ Order Allow,Deny Allow from all + + Order Allow,Deny + Allow from all + """ - CONFIG_TEMPLATE24 = """\ + CONFIG_TEMPLATE24_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] - + """ + CONFIG_TEMPLATE24_POST = """\ Require all granted + + Require all granted + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) - self.challenge_conf = os.path.join( + self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), - "le_http_01_challenge.conf") + "le_http_01_challenge_pre.conf") + self.challenge_conf_post = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_post.conf") self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") @@ -79,24 +92,32 @@ class ApacheHttp01(common.TLSSNI01): chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) if vh: - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) else: for vh in self._relevant_vhosts(): - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) self.configurator.reverter.register_file_creation( - True, self.challenge_conf) + True, self.challenge_conf_pre) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_post) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template_pre = self.CONFIG_TEMPLATE22_PRE + config_template_post = self.CONFIG_TEMPLATE22_POST else: - config_template = self.CONFIG_TEMPLATE24 + config_template_pre = self.CONFIG_TEMPLATE24_PRE + config_template_post = self.CONFIG_TEMPLATE24_POST - config_text = config_template.format(self.challenge_dir) + config_text_pre = config_template_pre.format(self.challenge_dir) + config_text_post = config_template_post.format(self.challenge_dir) - logger.debug("writing a config file with text:\n %s", config_text) - with open(self.challenge_conf, "w") as new_conf: - new_conf.write(config_text) + logger.debug("writing a pre config file with text:\n %s", config_text_pre) + with open(self.challenge_conf_pre, "w") as new_conf: + new_conf.write(config_text_pre) + logger.debug("writing a post config file with text:\n %s", config_text_post) + with open(self.challenge_conf_post, "w") as new_conf: + new_conf.write(config_text_post) def _relevant_vhosts(self): http01_port = str(self.configurator.config.http01_port) @@ -137,14 +158,17 @@ class ApacheHttp01(common.TLSSNI01): return response - def _set_up_include_directive(self, vhost): - """Includes override configuration to the beginning of VirtualHost. - Note that this include isn't added to Augeas search tree""" + def _set_up_include_directives(self, vhost): + """Includes override configuration to the beginning and to the end of + VirtualHost. Note that this include isn't added to Augeas search tree""" if vhost not in self.moded_vhosts: logger.debug( "Adding a temporary challenge validation Include for name: %s " + "in: %s", vhost.name, vhost.filep) self.configurator.parser.add_dir_beginning( - vhost.path, "Include", self.challenge_conf) + vhost.path, "Include", self.challenge_conf_pre) + self.configurator.parser.add_dir( + vhost.path, "Include", self.challenge_conf_post) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 64a76649a..9ed4ee509 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -158,23 +158,31 @@ class ApacheHttp01Test(util.ApacheTest): for vhost in vhosts: if not vhost.ssl: matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - with open(self.http.challenge_conf) as f: - conf_contents = f.read() + with open(self.http.challenge_conf_pre) as f: + pre_conf_contents = f.read() - self.assertTrue("RewriteEngine on" in conf_contents) - self.assertTrue("RewriteRule" in conf_contents) - self.assertTrue(self.http.challenge_dir in conf_contents) + with open(self.http.challenge_conf_post) as f: + post_conf_contents = f.read() + + self.assertTrue("RewriteEngine on" in pre_conf_contents) + self.assertTrue("RewriteRule" in pre_conf_contents) + + self.assertTrue(self.http.challenge_dir in post_conf_contents) if self.config.version < (2, 4): - self.assertTrue("Allow from all" in conf_contents) + self.assertTrue("Allow from all" in post_conf_contents) else: - self.assertTrue("Require all granted" in conf_contents) + self.assertTrue("Require all granted" in post_conf_contents) def _test_challenge_file(self, achall): name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) From c0068791ce2a4edbe4b7293c6eab21d8b62347ca Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 24 Jan 2018 13:56:40 -0800 Subject: [PATCH 292/631] add let's encrypt status to footer and fix link --- docs/_templates/footer.html | 52 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 ++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 docs/_templates/footer.html diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 000000000..8fd0f127d --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,52 @@ +
+ {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} + +
+ +
+

+ + © Copyright 2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license. + +
+
+ + Let's Encrypt Status + + + {%- if build_id and build_url %} + {% trans build_url=build_url, build_id=build_id %} + + Build + {{ build_id }}. + + {% endtrans %} + {%- elif commit %} + {% trans commit=commit %} + + Revision {{ commit }}. + + {% endtrans %} + {%- elif last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} + {%- endif %} + +

+
+ + {%- if show_sphinx %} + {% trans %}Built with Sphinx using a theme provided by Read the Docs{% endtrans %}. + {%- endif %} + + {%- block extrafooter %} {% endblock %} + +
diff --git a/docs/conf.py b/docs/conf.py index 73df47dbd..09bb44285 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,8 @@ master_doc = 'index' # General information about the project. project = u'Certbot' -copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' +# this is now overridden by the footer.html template +#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 0a4f926b160cf118c81974e91df7f196bc7a379d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 24 Jan 2018 15:01:42 -0800 Subject: [PATCH 293/631] Remove Default Detector log line. (#5372) This produces a super-long log line that wraps to 30-60 lines, depending on screen width. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Remove Default Detector log line. This produces about 30 lines of log output. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Add more useful logs. --- certbot/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7e..dec7474f9 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,13 +207,15 @@ def set_by_cli(var): # propagate plugin requests: eg --standalone modifies config.authenticator detector.authenticator, detector.installer = ( plugin_selection.cli_plugin_requests(detector)) - logger.debug("Default Detector is %r", detector) if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) return True for modifier in VAR_MODIFIERS.get(var, []): if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) return True return False From 8a9f21cdd36fb9a424430f9c807bacb27c1cfbf4 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Jan 2018 22:19:32 -0800 Subject: [PATCH 294/631] Fix Nginx redirect issue (#5479) * wrap redirect in if host matches * return 404 if we've created a new block * change domain matching to exact match * insert new redirect directive at the top * add a redirect block to the top if it doesn't already exist, even if there's an existing redirect * fix obj tests * remove active parameter * update tests * add back spaces * move imports * remove unused code --- certbot-nginx/certbot_nginx/configurator.py | 50 ++++---- certbot-nginx/certbot_nginx/obj.py | 21 ---- .../certbot_nginx/tests/configurator_test.py | 110 +++--------------- certbot-nginx/certbot_nginx/tests/obj_test.py | 14 +-- 4 files changed, 44 insertions(+), 151 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index bb2933a39..9f091c0fd 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -31,16 +31,6 @@ from certbot_nginx import http_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [ - ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - ['\n'] -] - -REDIRECT_COMMENT_BLOCK = [ - ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' return 301 https://$host$request_uri;'], - ['\n'] -] @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) @@ -571,24 +561,17 @@ class NginxConfigurator(common.Installer): logger.warning("Failed %s for %s", enhancement, domain) raise - def _has_certbot_redirect(self, vhost): - test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + def _has_certbot_redirect(self, vhost, domain): + test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) - def _has_certbot_redirect_comment(self, vhost): - test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) - return vhost.contains_list(test_redirect_comment_block) - - def _add_redirect_block(self, vhost, active=True): + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ - if active: - redirect_block = REDIRECT_BLOCK - else: - redirect_block = REDIRECT_COMMENT_BLOCK + redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False) + vhost, redirect_block, replace=False, insert_at_top=True) def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -615,6 +598,7 @@ class NginxConfigurator(common.Installer): self.DEFAULT_LISTEN_PORT) return + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, only_directives=['listen', 'server_name']) @@ -631,20 +615,18 @@ class NginxConfigurator(common.Installer): # remove all non-ssl addresses from the existing block self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + # Add this at the bottom to get the right order of directives + return_404_directive = [['\n ', 'return', ' ', '404']] + self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + vhost = new_vhost - if self._has_certbot_redirect(vhost): + if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) else: # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) + self._add_redirect_block(vhost, domain) logger.info("Redirecting all traffic on port %s to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) @@ -907,6 +889,14 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] +def _redirect_block_for_domain(domain): + redirect_block = [[ + ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + '\n ']], + ['\n']] + return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index f5ac5c2e3..e8dc8936d 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,15 +193,6 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False - def has_redirect(self): - """Determine if this vhost has a redirecting statement - """ - for directive_name in REDIRECT_DIRECTIVES: - found = _find_directive(self.raw, directive_name) - if found is not None: - return True - return False - def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -225,15 +216,3 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True - -def _find_directive(directives, directive_name): - """Find a directive of type directive_name in directives - """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: - return None - - if directives[0] == directive_name: - return directives - - matches = (_find_directive(line, directive_name) for line in directives) - return next((m for m in matches if m is not None), None) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 7475df40c..acb7ee282 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser +from certbot_nginx.configurator import _redirect_block_for_domain +from certbot_nginx.nginxparser import UnspacedList from certbot_nginx.tests import util @@ -447,7 +449,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -460,6 +462,8 @@ class NginxConfiguratorTest(util.NginxTest): migration_conf = self.config.parser.abs_path('sites-enabled/migration.com') self.config.enhance("migration.com", "redirect") + expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0] + generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -484,101 +488,27 @@ class NginxConfiguratorTest(util.NginxTest): ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], [], []]], [['server'], [ + [['if', '($host', '=', 'www.example.com)'], [ + ['return', '301', 'https://$host$request_uri']]], + ['#', ' managed by Certbot'], [], ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], - [], []]]], + ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): + def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a # redirect in the block that is managed by certbot # Has a certbot redirect - mock_has_redirect.return_value = True mock_contains_list.return_value = True with mock.patch("certbot_nginx.configurator.logger") as mock_logger: self.config.enhance("www.example.com", "redirect") self.assertEqual(mock_logger.info.call_args[0][0], "Traffic on port %s already redirecting to ssl in %s") - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - self.config.deploy_cert( - "example.org", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block') - def test_redirect_comment_exists(self, mock_add_redirect_block, - mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list): - # Test that we add nothing if there is a non-Certbot redirect and a - # preexisting comment - # Has a non-Certbot redirect and a comment - mock_has_redirect.return_value = True - mock_contains_list.return_value = False # self._has_certbot_redirect(vhost): - mock_has_cb_redirect_comment.return_value = True - - # assert _add_redirect_block not called - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertFalse(mock_add_redirect_block.called) - self.assertTrue(mock_logger.info.called) - def test_redirect_dont_enhance(self): # Test that we don't accidentally add redirect to ssl-only block with mock.patch("certbot_nginx.configurator.logger") as mock_logger: @@ -586,22 +516,18 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_logger.info.call_args[0][0], 'No matching insecure server blocks listening on port %s found.') - def test_no_double_redirect(self): - # Test that we don't also add the commented redirect if we've just added - # a redirect to that vhost this run + def test_double_redirect(self): + # Test that we add one redirect for each domain example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("example.com", "redirect") self.config.enhance("example.org", "redirect") - unexpected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] - ] + expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0] + expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0] + generated_conf = self.config.parser.parsed[example_conf] - for line in unexpected: - self.assertFalse(util.contains_at_depth(generated_conf, line, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2)) def test_staple_ocsp_bad_version(self): self.config.version = (1, 3, 1) @@ -763,7 +689,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 92cb0e086..b30338b5b 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -162,17 +162,15 @@ class VirtualHostTest(unittest.TestCase): 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) - def test_has_redirect(self): - self.assertTrue(self.vhost1.has_redirect()) - self.assertTrue(self.vhost2.has_redirect()) - self.assertTrue(self.vhost3.has_redirect()) - self.assertFalse(self.vhost4.has_redirect()) - def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block - test_needle = _test_block_from_block(REDIRECT_BLOCK) + from certbot_nginx.configurator import _test_block_from_block + test_block = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] + ] + test_needle = _test_block_from_block(test_block) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], From a1aba5842e979379d98f3128d140d1270fb951c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:23:20 -0800 Subject: [PATCH 295/631] Fix --no-bootstrap on CentOS/RHEL 6 (#5476) * fix --no-bootstrap on RHEL6 * Add regression test --- letsencrypt-auto-source/letsencrypt-auto | 20 ++++++++++++------- .../letsencrypt-auto.template | 20 ++++++++++++------- .../tests/centos6_tests.sh | 8 ++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9c8ccf344..b5cac23e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 96e5c2db0..2ce337002 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -300,13 +300,8 @@ DeterminePythonVersion() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -402,6 +397,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index a0e96edf8..2c6dcf734 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -69,5 +69,13 @@ fi echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." echo "" +export VENV_PATH=$(mktemp -d) +"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 +if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then + echo "Python 3 wasn't used with --no-bootstrap!" + exit 1 +fi +unset VENV_PATH + # test using python3 pytest -v -s certbot/letsencrypt-auto-source/tests From a2239baa45de446ab246047171ad86e014bf580c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:38:36 -0800 Subject: [PATCH 296/631] fix test_tests.sh (#5478) --- tests/letstest/scripts/test_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 4bed2dd3a..e6ab836b8 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,11 +1,13 @@ #!/bin/sh -xe +LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" VENV_NAME=venv # *-auto respects VENV_PATH -letsencrypt/certbot-auto --debug --os-packages-only --non-interactive -LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version +$LE_AUTO --os-packages-only +LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate # change to an empty directory to ensure CWD doesn't affect tests From 43bbaadd1154c0cd44eeb761bf33563953e793ae Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Jan 2018 15:29:38 -0800 Subject: [PATCH 297/631] Update certbot-auto and help (#5487) * Release 0.21.1 (cherry picked from commit ff60d70e68f7b4ddc60a61848190fbb6e55b5d2b) * Bump version to 0.22.0 --- certbot-auto | 46 ++++++++++-------- docs/cli-help.txt | 18 +++---- letsencrypt-auto | 46 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 16 +++--- letsencrypt-auto-source/letsencrypt-auto | 24 ++++----- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++----- 7 files changed, 92 insertions(+), 82 deletions(-) diff --git a/certbot-auto b/certbot-auto index 558c330b2..d3a5c23e5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f7318f0b3..abebdb9c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.21.0 (certbot; - Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + "". (default: CertbotACMEClient/0.21.1 (certbot; + darwin 10.13.3) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -448,11 +448,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary. (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: - a2dismod) + Path to the Apache 'a2dismod' binary. (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -466,13 +464,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: True) + you.(Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you.(Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index 558c330b2..d3a5c23e5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9df8013a0..f28fd9893 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n -e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW -AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL -RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn -IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf -yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= -=d/hS +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X +dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 +JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc +a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 +0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR +zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR +F/j/631OC6yLRasr+xcL1gc+SCryfA== +=tkZT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b5cac23e6..8ff7944b5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1196,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 2e52f03fdf9a1fbe2049d89f071a583bcf919976..8dd6837754301180039bdcd2192ec2ffa9911f1e 100644 GIT binary patch literal 256 zcmV+b0ssEOC2SB&#jbL|t)G85E<)sA{+|s~F5m$!7y+-fA{(45p3@Fk=k3Xi+RklE z=idnaoUY1-Gy6T(2&LYe?9T(@a*CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&* zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz GZCemMz Date: Wed, 22 Nov 2017 18:01:56 -0800 Subject: [PATCH 298/631] Add expiration date to skipped message --- certbot/renewal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/renewal.py b/certbot/renewal.py index 7d0240f73..024a815cc 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -425,7 +425,10 @@ def handle_renewal_request(config): main.renew_cert(lineage_config, plugins, renewal_candidate) renew_successes.append(renewal_candidate.fullchain) else: - renew_skipped.append(renewal_candidate.fullchain) + expiry = crypto_util.notAfter(renewal_candidate.version( + "cert", renewal_candidate.latest_common_version())) + renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, + expiry.strftime("%Y-%m-%d"))) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " From 45613fd31c3649ac5fb08a52153a8893f394af0b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 26 Jan 2018 16:02:19 -0800 Subject: [PATCH 299/631] update changelog for 0.21.1 (#5504) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f7be9fe..1369b0907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.1 - 2018-01-25 + +### Fixed + +* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host + header of the request is set to an expected value before redirecting users to + the domain found in the header. The previous way Certbot configured Nginx + redirects was a potential security issue which you can read more about at + https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493. +* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges + if basic authentication is configured for the domain you request a + certificate for. +* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6 + based systems rather than Python 2.6. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/49?closed=1 + ## 0.21.0 - 2018-01-17 ### Added From 72b63ca5ac19cc8da0b1aba3a55dea85bca7a0eb Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 1 Feb 2018 13:14:43 -0800 Subject: [PATCH 300/631] Use "certificate" instead of "cert" in docs. --- docs/install.rst | 2 +- docs/using.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index a914586ff..c18c3cdbc 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -116,7 +116,7 @@ certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a cert for resolves +from the server that the domain you're requesting a certficate for resolves to, `install Docker`_, then issue the following command: .. code-block:: shell diff --git a/docs/using.rst b/docs/using.rst index ab4670052..e8f84e2d7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -557,8 +557,8 @@ apologize for any inconvenience you encounter in integrating these commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. - This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. - If you write a custom script and expect to run a command only after a cert was actually renewed + This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. + If you write a custom script and expect to run a command only after a certificate was actually renewed you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. From e085ff06a12a0be6033f38f092d42253c7b21515 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 5 Feb 2018 16:27:21 -0800 Subject: [PATCH 301/631] Update old issue link to point to letsencrypt community forums. (#5538) --- certbot/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index b735421f5..bc25da549 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -556,11 +556,11 @@ class Client(object): self.installer.rollback_checkpoints() self.installer.restart() except: - # TODO: suggest letshelp-letsencrypt here reporter.add_message( "An error occurred and we failed to restore your config and " - "restart your server. Please submit a bug report to " - "https://github.com/letsencrypt/letsencrypt", + "restart your server. Please post to " + "https://community.letsencrypt.org/c/server-config " + "with details about your configuration and this error you received.", reporter.HIGH_PRIORITY) raise reporter.add_message(success_msg, reporter.HIGH_PRIORITY) From 9baf75d6c887a9b69bd92c7bffe585f091c9b08e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 6 Feb 2018 16:45:33 -0800 Subject: [PATCH 302/631] client.py changes for ACMEv2 (#5287) * Implement ACMEv2 signing of POST bodies. * Add account, and make acme_version explicit. * Remove separate NewAccount. * Rename to add v2. * Add terms_of_service_agreed. * Split out wrap_in_jws_v2 test. * Re-add too-many-public-methods. * Split Client into ClientBase / Client / ClientV2 * Use camelCase for newAccount. * Make acme_version optional parameter on .post(). This allows us to instantiate a ClientNetwork before knowing the version. * Add kid unconditionally. --- acme/acme/client.py | 324 ++++++++++++++++++++++++--------------- acme/acme/client_test.py | 74 ++++++--- acme/acme/messages.py | 1 + 3 files changed, 258 insertions(+), 141 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..5d1add5a3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -39,39 +39,24 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. + :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net, acme_version): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + :param int acme_version: ACME protocol version. 1 or 2. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net + self.acme_version = acme_version @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,28 +68,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body) + response = self.net.post(regr.uri, body, acme_version=self.acme_version) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -153,21 +118,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), @@ -176,42 +126,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri=None): - """Request challenges. - - :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - if new_authzr_uri is not None: - logger.debug("request_challenges with new_authzr_uri deprecated.") - new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) - # TODO: handle errors - assert response.status_code == http_client.CREATED - return self._authzr_from_response(response, identifier) - - def request_domain_challenges(self, domain, new_authzr_uri=None): - """Request challenges for domain names. - - This is simply a convenience function that wraps around - `request_challenges`, but works with domain names instead of - generic identifiers. See ``request_challenges`` for more - documentation. - - :param str domain: Domain name to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) - def answer_challenge(self, challb, response): """Answer challenge. @@ -227,7 +141,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response) + response = self.net.post(challb.uri, response, + acme_version=self.acme_version) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -288,6 +203,133 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + response = self.net.post(self.directory[messages.Revocation], + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None, + acme_version=self.acme_version) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') + +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `.JWK` (private) + :ivar alg: `.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, + net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + directory = messages.Directory.from_json( + self.net.get(directory).json()) + super(Client, self).__init__(directory=directory, + net=net, acme_version=1) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self.net.post(self.directory[new_reg], new_reg, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + new_authz = messages.NewAuthorization(identifier=identifier) + response = self.net.post(self.directory.new_authz, new_authz, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) + def request_issuance(self, csr, authzrs): """Request issuance. @@ -311,7 +353,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}) + headers={'Accept': content_type}, + acme_version=1) cert_chain_uri = response.links.get('up', {}).get('url') @@ -481,37 +524,65 @@ class Client(object): # pylint: disable=too-many-instance-attributes "Recursion limit reached. Didn't get {0}".format(uri)) return chain - def revoke(self, cert, rsn): - """Revoke certificate. - :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in - `.ComparableX509` - :param int rsn: Reason code for certificate revocation. +class ClientV2(ClientBase): + """ACME client for a v2 API. - :raises .ClientError: If revocation is unsuccessful. + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. + """ + + def __init__(self, directory, net): + """Initialize. + + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, + net=net, acme_version=2) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + response = self.net.post(self.directory['newAccount'], new_account, + acme_version=2) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) class ClientNetwork(object): # pylint: disable=too-many-instance-attributes - """Client network.""" + """Wrapper around requests that signs POSTs for authentication. + + Also adds user agent, and handles Content-Type. + """ JSON_CONTENT_TYPE = 'application/json' JOSE_CONTENT_TYPE = 'application/jose+json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, + """Initialize. + + :param key: Account private key + :param messages.Registration account: Account object. Required if you are + planning to use .post() with acme_version=2. + :param josepy.JWASignature alg: Algoritm to use in signing JWS. + :param bool verify_ssl: Whether to verify certificates on SSL connections. + :param str user_agent: String to send as User-Agent header. + :param float timeout: Timeout for requests. + """ + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() @@ -527,21 +598,29 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url, acme_version): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. :param .JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: :rtype: `.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if acme_version == 2: + kwargs["url"] = url + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -714,8 +793,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise - def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, + acme_version=1, **kwargs): + data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..662c32942 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -104,6 +104,23 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments + def test_new_account_v2(self): + directory = messages.Directory({ + "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', + }) + from acme.client import ClientV2 + client = ClientV2(directory, self.net) + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, client.new_account(self.regr)) + def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -142,20 +159,23 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_deprecated_arg(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier, new_authzr_uri="hi") self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY, + acme_version=1) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -417,7 +437,8 @@ class ClientTest(unittest.TestCase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None) + self.directory[messages.Revocation], mock.ANY, content_type=None, + acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -432,9 +453,22 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +487,27 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=1) jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=2) + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +737,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6daf55094..98993c4e1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -251,6 +251,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) + terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' From 0416382633c7b37f05193c0bc42d10d03ac13b5e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Feb 2018 17:01:58 -0800 Subject: [PATCH 303/631] Update leauto_upgrades with tests from #5402. (#5407) --- .../letstest/scripts/test_leauto_upgrades.sh | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51472f2e6..355fead2e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -65,6 +65,7 @@ iQIDAQAB " if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + RUN_PYTHON3_TESTS=1 if command -v python3; then echo "Didn't expect Python 3 to be installed!" exit 1 @@ -85,11 +86,25 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; exit 1 fi unset VENV_PATH - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) - if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then - echo "Certbot didn't upgrade as expected!" - exit 1 - fi +fi + +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi + +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then + echo upgrade appeared to fail + exit 1 +fi + +if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then + echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ + exit 1 +fi + +if [ "$RUN_PYTHON3_TESTS" = 1 ]; then if ! command -v python3; then echo "Python3 wasn't properly installed" exit 1 @@ -98,11 +113,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; echo "Python3 wasn't used in venv!" exit 1 fi -elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo upgrade appeared to fail - exit 1 fi - echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then From 530a9590e6b9a912162facdae58924f0e750d314 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 7 Feb 2018 14:08:03 -0800 Subject: [PATCH 304/631] Add sudo to certbot-auto instructions. (#5501) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c144a4f74..a3e5d530a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -32,7 +32,7 @@ a new plugin is introduced. .. code-block:: shell cd certbot - ./certbot-auto --os-packages-only + sudo ./certbot-auto --os-packages-only ./tools/venv.sh Then in each shell where you're working on the client, do: From 4f0aeb12fa30d56c3926713901bc02cc53d6b9d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Feb 2018 14:14:26 -0800 Subject: [PATCH 305/631] Add find-duplicative-certs docs (#5547) * add find-duplicative-certs docs * address review feedback --- certbot/cert_manager.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index da85ed783..4240a0523 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -121,7 +121,28 @@ def domains_for_certname(config, certname): return lineage.names() if lineage else None def find_duplicative_certs(config, domains): - """Find existing certs that duplicate the request.""" + """Find existing certs that match the given domain names. + + This function searches for certificates whose domains are equal to + the `domains` parameter and certificates whose domains are a subset + of the domains in the `domains` parameter. If multiple certificates + are found whose names are a subset of `domains`, the one whose names + are the largest subset of `domains` is returned. + + If multiple certificates' domains are an exact match or equally + sized subsets, which matching certificates are returned is + undefined. + + :param config: Configuration. + :type config: :class:`certbot.configuration.NamespaceConfig` + :param domains: List of domain names + :type domains: `list` of `str` + + :returns: lineages representing the identically matching cert and the + largest subset if they exist + :rtype: `tuple` of `storage.RenewableCert` or `None` + + """ def update_certs_for_domain_matches(candidate_lineage, rv): """Return cert as identical_names_cert if it matches, or subset_names_cert if it matches as subset From d6b247c002d9e5986098c44c771f2177a0823aae Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 9 Feb 2018 12:54:15 -0800 Subject: [PATCH 306/631] Set ClientNetwork.account after registering (#5558) --- acme/acme/client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 5d1add5a3..77e89e535 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -95,6 +95,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) + self.net.account = updated_regr return updated_regr def deactivate_registration(self, regr): @@ -555,8 +556,9 @@ class ClientV2(ClientBase): acme_version=2) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member - return self._regr_from_response(response) - + regr = self._regr_from_response(response) + self.net.account = regr + return regr class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. @@ -571,8 +573,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. :param key: Account private key - :param messages.Registration account: Account object. Required if you are - planning to use .post() with acme_version=2. + :param messages.RegistrationResource account: Account object. Required if you are + planning to use .post() with acme_version=2 for anything other than creating a new + account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. From 1f45832460550d0982676368870af9cac69592a5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Feb 2018 16:41:05 -0800 Subject: [PATCH 307/631] Suggest people try the community forum. (#5561) --- ISSUE_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 55780c0ac..03917f8ca 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,9 @@ +If you're having trouble using Certbot and aren't sure you've found a bug or +request for a new feature, please first try asking for help at +https://community.letsencrypt.org/. There is a much larger community there of +people familiar with the project who will be able to more quickly answer your +questions. + ## My operating system is (include version): From abc4a27613f1e8196a52f75b051c05c3d2153a2a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 12 Feb 2018 14:07:33 -0800 Subject: [PATCH 308/631] [Docs] restore docs for ppl just using Certbot git master (#5420) - Dev / test cycles are one use case for the "running a local copy of the client" instructions, but simply running bleeding edge Certbot is another - So edit the docs to once again explain how to just run bleeding edge Certbot, without (say) always getting staging certs. --- docs/contributing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index a3e5d530a..83b607e15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -35,7 +35,10 @@ a new plugin is introduced. sudo ./certbot-auto --os-packages-only ./tools/venv.sh -Then in each shell where you're working on the client, do: +You can now run the copy of Certbot from git either by executing +``venv/bin/certbot``, or by activating the virtual environment. If you're +actively modifying and testing the code, you may want to run commands like this in +each shell where you're working: .. code-block:: shell @@ -43,7 +46,8 @@ Then in each shell where you're working on the client, do: export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh -After that, your shell will be using the virtual environment, and you run the +After that, your shell will be using the virtual environment, your copy of +Certbot will default to requesting test (staging) certificates, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that includes several flags useful for testing. For instance, it sets various output directories to point to /tmp/, and uses non-privileged ports for challenges, so From 789be8f9bc36241eee88849a6c496dc898185030 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 12 Feb 2018 14:55:41 -0800 Subject: [PATCH 309/631] Change "Attempting to parse" warning to info. (#5557) * Change "Attempting to parse" warning to info. This message shows up on every renewal run when the config was updated by a newer version of Certbot than the one being run. For instance, if a user has the certbot packages installed from PPA (currently 0.18.2), but runs certbot-auto once to try out the latest version (0.21.1), they will start getting this message via email every 12 hours. --- certbot/storage.py | 2 +- certbot/tests/storage_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 67d2155ae..ed3922c58 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -417,7 +417,7 @@ class RenewableCert(object): conf_version = self.configuration.get("version") if (conf_version is not None and util.get_strict_version(conf_version) > CURRENT_VERSION): - logger.warning( + logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " "work.", conf_version, config_filename, certbot.__version__) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c8f775e2..6c0970e72 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -158,8 +158,8 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertTrue(mock_logger.warning.called) - self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + self.assertTrue(mock_logger.info.called) + self.assertTrue("version" in mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=too-many-statements,protected-access From 90664f196f992b20bdfdfb8914697e4456de7510 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Mon, 12 Feb 2018 16:43:11 -0800 Subject: [PATCH 310/631] Remove autodocs for long-removed acme.other module (#5529) This module was removed in 22a9c7e3c2a811698ed7d63fae1cd43d0bd5d088. The autodocs are therefore unnecessary. Furthermore, they are starting to cause build failures for Fedora. --- acme/docs/api/other.rst | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 acme/docs/api/other.rst diff --git a/acme/docs/api/other.rst b/acme/docs/api/other.rst deleted file mode 100644 index eb27a5d53..000000000 --- a/acme/docs/api/other.rst +++ /dev/null @@ -1,5 +0,0 @@ -Other ACME objects ------------------- - -.. automodule:: acme.other - :members: From 932ecbb9c20bdc4e9d05c2e2bf78ea1b1608b67d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 13 Feb 2018 02:43:59 +0200 Subject: [PATCH 311/631] Fix test inconsistence in Apache plugin configurator_test (#5520) --- certbot-apache/certbot_apache/tests/configurator_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 530d75a92..7fbfcb617 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -474,7 +474,11 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 3) self.assertTrue(mock_add_dir.called) self.assertEqual(mock_add_dir.call_args[0][1], "Listen") - self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + call_found = False + for mock_call in mock_add_dir.mock_calls: + if mock_call[1][2] == ['1.2.3.4:8080']: + call_found = True + self.assertTrue(call_found) def test_prepare_server_https(self): mock_enable = mock.Mock() From 49edf17cb77455f9f7b8227735939f693ceff303 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 13 Feb 2018 09:52:04 -0800 Subject: [PATCH 312/631] ignore .docker (#5477) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a01d2e1c7..4dff20caf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ tests/letstest/venv/ # pytest cache .cache + +# docker files +.docker From ad0a99a1f5aa3cc28988685428bfdd475d48df58 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 10:50:04 -0800 Subject: [PATCH 313/631] Proper webroot directory cleanup (#5453) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..15889a25c 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 9277710f6f984f6464ee375ea72d3a11b689853c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 11:15:08 -0800 Subject: [PATCH 314/631] Added install-only flag (#5531) --- letsencrypt-auto-source/letsencrypt-auto | 9 +++++++++ letsencrypt-auto-source/letsencrypt-auto.template | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..c84499eab 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -1428,6 +1431,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..365fc6752 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -587,6 +590,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else From ac464a58e5591f917ab66e2c30b2ba92cdc85e80 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 18:16:20 +0200 Subject: [PATCH 315/631] Only add Include for TLS configuration if not already there (#5498) * Only add Include for TLS configuration if not already there * Add tests to prevent future regression --- certbot-apache/certbot_apache/configurator.py | 5 +++- .../certbot_apache/tests/configurator_test.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index b32eda921..4bb2cbebd 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1269,7 +1269,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + # Only include the TLS configuration if not already included + existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path) + if not existing_inc: + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7fbfcb617..8f34d33d3 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -335,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest): "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) + def test_no_duplicate_include(self): + def mock_find_dir(directive, argument, _): + """Mock method for parser.find_dir""" + if directive == "Include" and argument.endswith("options-ssl-apache.conf"): + return ["/path/to/whatever"] + + mock_add = mock.MagicMock() + self.config.parser.add_dir = mock_add + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = False + for a in mock_add.call_args_list: + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + tried_to_add = True + # Include should be added, find_dir is not patched, and returns falsy + self.assertTrue(tried_to_add) + + self.config.parser.find_dir = mock_find_dir + mock_add.reset_mock() + + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = [] + for a in mock_add.call_args_list: + tried_to_add.append(a[0][1] == "Include" and + a[0][2] == self.config.mod_ssl_conf) + # Include shouldn't be added, as patched find_dir "finds" existing one + self.assertFalse(any(tried_to_add)) + def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") From fbace69b5e8fed5ee32cb029478d6b11aea84842 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 19:28:36 +0200 Subject: [PATCH 316/631] Fix install verb (#5536) * Fix install verb * Fix error message, tests and remove global pylint change * Fix boulder integration test keypath * Also use chain_path from lineage if not defined on CLI --- certbot/cli.py | 5 +- certbot/main.py | 41 ++++++++++++++-- certbot/tests/main_test.py | 90 +++++++++++++++++++++++++++++++++--- tests/boulder-integration.sh | 2 +- 4 files changed, 125 insertions(+), 13 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index dec7474f9..09dd71d13 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1278,14 +1278,13 @@ def _paths_parser(helpful): elif verb == "revoke": add(section, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, - help=cph, required=(verb == "install")) + add(section, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): section = verb # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", required=(verb == "install"), + add(section, "--key-path", type=((verb == "revoke" and read_file) or os.path.abspath), help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..13234e0d2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,4 +1,5 @@ """Certbot main entry point.""" +# pylint: disable=too-many-lines from __future__ import print_function import functools import logging.handlers @@ -779,11 +780,45 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) - domains, _ = _find_domains_or_certname(config, installer) - le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + # If cert-path is defined, populate missing (ie. not overridden) values. + # Unfortunately this can't be done in argument parser, as certificate + # manager needs the access to renewal directory paths + if config.certname: + config = _populate_from_certname(config) + if config.key_path and config.cert_path: + _check_certificate_and_key(config) + domains, _ = _find_domains_or_certname(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + _install_cert(config, le_client, domains) + else: + raise errors.ConfigurationError("Path to certificate or key was not defined. " + "If your certificate is managed by Certbot, please use --cert-name " + "to define which certificate you would like to install.") +def _populate_from_certname(config): + """Helper function for install to populate missing config values from lineage + defined by --cert-name.""" + lineage = cert_manager.lineage_for_certname(config, config.certname) + if not lineage: + return config + if not config.key_path: + config.namespace.key_path = lineage.key_path + if not config.cert_path: + config.namespace.cert_path = lineage.cert_path + if not config.chain_path: + config.namespace.chain_path = lineage.chain_path + if not config.fullchain_path: + config.namespace.fullchain_path = lineage.fullchain_path + return config + +def _check_certificate_and_key(config): + if not os.path.isfile(os.path.realpath(config.cert_path)): + raise errors.ConfigurationError("Error while reading certificate from path " + "{0}".format(config.cert_path)) + if not os.path.isfile(os.path.realpath(config.key_path)): + raise errors.ConfigurationError("Error while reading private key from path " + "{0}".format(config.key_path)) def plugins_cmd(config, plugins): """List server software plugins. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f..7368e8513 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -592,11 +592,30 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met super(MainTest, self).tearDown() - def _call(self, args, stdout=None): - "Run the cli with output streams and actual client mocked out" - with mock.patch('certbot.main.client') as client: - ret, stdout, stderr = self._call_no_clientmock(args, stdout) - return ret, stdout, stderr, client + def _call(self, args, stdout=None, mockisfile=False): + """Run the cli with output streams, actual client and optionally + os.path.isfile() mocked out""" + + if mockisfile: + orig_open = os.path.isfile + def mock_isfile(fn, *args, **kwargs): + """Mock os.path.isfile()""" + if (fn.endswith("cert") or + fn.endswith("chain") or + fn.endswith("privkey")): + return True + else: + return orig_open(fn, *args, **kwargs) + + with mock.patch("os.path.isfile") as mock_if: + mock_if.side_effect = mock_isfile + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client + else: + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client def _call_no_clientmock(self, args, stdout=None): "Run the client with output streams mocked out" @@ -680,9 +699,68 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_selection(self, mock_pick_installer, _rec): self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert', - '--key-path', 'key', '--chain-path', 'chain']) + '--key-path', 'privkey', '--chain-path', 'chain'], mockisfile=True) self.assertEqual(mock_pick_installer.call_count, 1) + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_certname(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_override(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever', + '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.chain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + + mock_install.reset() + + self._call(['install', '--cert-name', 'whatever', + '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_error(self, _inst, _rec): + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--key-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-name', 'notfound', + '--key-path', 'invalid']) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..24d224cb0 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -251,7 +251,7 @@ openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/csr/key.pem" + --key-path "${root}/key.pem" CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` From 99aec1394df3813646a2734405043aeada160a49 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 14 Feb 2018 12:09:17 -0800 Subject: [PATCH 317/631] Revert "Proper webroot directory cleanup (#5453)" (#5574) This reverts commit ad0a99a1f5aa3cc28988685428bfdd475d48df58. --- certbot/plugins/util.py | 18 ---------- certbot/plugins/util_test.py | 8 ----- certbot/plugins/webroot.py | 63 ++++++++++++++++----------------- certbot/plugins/webroot_test.py | 31 ---------------- 4 files changed, 30 insertions(+), 90 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..f0e2f4c5b 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,24 +6,6 @@ from certbot import util logger = logging.getLogger(__name__) -def get_prefixes(path): - """Retrieves all possible path prefixes of a path, in descending order - of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] - :param str path: the path to break into prefixes - - :returns: all possible path prefixes of given path in descending order - :rtype: `list` of `str` - """ - prefix = path - prefixes = [] - while len(prefix) > 0: - prefixes.append(prefix) - prefix, _ = os.path.split(prefix) - # break once we hit '/' - if prefix == prefixes[-1]: - break - return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..947f24697 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,14 +5,6 @@ import unittest import mock -class GetPrefixTest(unittest.TestCase): - """Tests for certbot.plugins.get_prefixes.""" - def test_get_prefix(self): - from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) - class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 15889a25c..714d83cce 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,7 +18,6 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common -from certbot.plugins import util logger = logging.getLogger(__name__) @@ -66,8 +65,6 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) - # stack of dirs successfully created by this authenticator - self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -164,26 +161,27 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) + try: - stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): - try: - # This is coupled with the "umask" call above because - # os.mkdir's "mode" parameter may not always work: - # https://docs.python.org/3/library/os.html#os.mkdir - os.mkdir(prefix, 0o0755) - self._created_dirs.append(prefix) - # Set owner as parent directory if possible - try: - os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + # This is coupled with the "umask" call above because + # os.makedirs's "mode" parameter may not always work: + # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python + os.makedirs(self.full_roots[name], 0o0755) + + # Set owner as parent directory if possible + try: + stat_path = os.stat(path) + os.chown(self.full_roots[name], stat_path.st_uid, + stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -219,17 +217,16 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] - while len(self._created_dirs) > 0: - path = self._created_dirs.pop() - try: - os.rmdir(path) - except OSError as exc: - not_removed.insert(0, path) - logger.info("Challenge directory %s was not empty, didn't remove", path) - logger.debug("Error was: %s", exc) - self._created_dirs = not_removed - logger.debug("All challenges cleaned up") + for root_path, achalls in six.iteritems(self.performed): + if not achalls: + try: + os.rmdir(root_path) + logger.debug("All challenges cleaned up, removing %s", + root_path) + except OSError as exc: + logger.info( + "Unable to clean up challenge directory %s", root_path) + logger.debug("Error was: %s", exc) class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 59133f0aa..36e2ffba6 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,8 +36,6 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() - self.partial_root_challenge_path = os.path.join( - self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -201,35 +199,6 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) - self.assertFalse(os.path.exists(self.partial_root_challenge_path)) - - def test_perform_cleanup_existing_dirs(self): - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([self.achall]) - self.auth.cleanup([self.achall]) - - # Ensure we don't "clean up" directories that previously existed - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) - - def test_perform_cleanup_multiple_challenges(self): - bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.HTTP01(token=b"bingo"), "pending"), - domain="thing.com", account_key=KEY) - - bingo_validation_path = "YmluZ28" - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([bingo_achall, self.achall]) - - self.auth.cleanup([self.achall]) - self.assertFalse(os.path.exists(bingo_validation_path)) - self.assertTrue(os.path.exists(self.root_challenge_path)) - self.auth.cleanup([bingo_achall]) - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 608875cd655e5719a56d178e4046539c98b91348 Mon Sep 17 00:00:00 2001 From: Sydney Li Date: Wed, 14 Feb 2018 15:14:24 -0800 Subject: [PATCH 318/631] Add test for skipped certs --- certbot/tests/main_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..8a2ffd1ca 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -926,7 +926,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False): + quiet_mode=False, expiry_date=datetime.datetime.now()): # pylint: disable=too-many-locals,too-many-arguments cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -959,7 +959,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_latest = mock.MagicMock() mock_latest.get_issuer.return_value = "Fake fake" mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('certbot.main.renewal.crypto_util'): + with mock.patch('certbot.main.renewal.crypto_util') as mock_crypto_util: + mock_crypto_util.notAfter.return_value = expiry_date if not args: args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: @@ -1027,6 +1028,16 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + @mock.patch('certbot.renewal.should_renew') + def test_renew_skips_recent_certs(self, should_renew): + should_renew.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + expiry = datetime.datetime.now() + datetime.timedelta(days=90) + _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, + args=['renew'], expiry_date=expiry) + self.assertTrue('No renewals were attempted.' in stdout.getvalue()) + self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] From 09b5927e6ac881359f6df9a1792a598fd94ad8d6 Mon Sep 17 00:00:00 2001 From: cclauss Date: Thu, 15 Feb 2018 20:07:35 +0100 Subject: [PATCH 319/631] from botocore.exceptions import ClientError (#5507) Fixes undefined name 'botocore' in flake8 testing of https://github.com/certbot/certbot $ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__ ``` ./tests/letstest/multitester.py:144:12: F821 undefined name 'botocore' except botocore.exceptions.ClientError as e: ^ 1 F821 undefined name 'botocore' ``` --- tests/letstest/multitester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index d9491939c..17740cde8 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -38,6 +38,7 @@ from multiprocessing import Manager import urllib2 import yaml import boto3 +from botocore.exceptions import ClientError import fabric from fabric.api import run, execute, local, env, sudo, cd, lcd from fabric.operations import get, put @@ -141,7 +142,7 @@ def make_instance(instance_name, # give instance a name try: new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) - except botocore.exceptions.ClientError as e: + except ClientError as e: if "InvalidInstanceID.NotFound" in str(e): # This seems to be ephemeral... retry time.sleep(1) From d5efefd979b7c660130b04e5c588d2f4a63a991c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 15 Feb 2018 15:55:08 -0800 Subject: [PATCH 320/631] Re-land proper webroot directory cleanup (#5577) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally * Fix for MacOS --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..6328b16ef 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From d467a4ae95680220286c5ce7e6e840b48c8f74a5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 15 Feb 2018 19:04:17 -0800 Subject: [PATCH 321/631] Add mechanism to detect acme version (#5554) Detects acme version by checking for newNonce field in the directory, since it's mandatory. Also updates ClientNetwork.account on register and update_registration. * add mechanism to detect acme version * update ClientNetwork.account comment * switch to MultiVersionClient object in acme * add shim methods * add returns * use backwards-compatible format and implement register * update to actual representation of tos v2 * add tos fields and pass through to v1 for partial updates * update tests * pass more tests * allow instance variable pass-through and lint * update certbot and tests to use new_account_and_tos method * remove --agree-tos test from main_test for now because we moved the callback into acme * add docstrings * use hasattr * all most review comments * use terms_of_service for both v1 and v2 * add tests for acme/client.py * tests for acme/messages.py --- acme/acme/client.py | 56 ++++++++++++ acme/acme/client_test.py | 171 ++++++++++++++++++++++++++++------- acme/acme/messages.py | 25 ++++- acme/acme/messages_test.py | 7 ++ certbot/client.py | 24 ++--- certbot/main.py | 11 ++- certbot/tests/client_test.py | 36 ++++---- certbot/tests/main_test.py | 17 ++-- certbot/tests/util.py | 2 +- 9 files changed, 262 insertions(+), 87 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 77e89e535..d843feaa7 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -560,6 +560,62 @@ class ClientV2(ClientBase): self.net.account = regr return regr +class BackwardsCompatibleClientV2(object): + """ACME client wrapper that tends towards V2-style calls, but + supports V1 servers. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 + """ + + def __init__(self, net, key, server): + directory = messages.Directory.from_json(net.get(server).json()) + self.acme_version = self._acme_version_from_directory(directory) + if self.acme_version == 1: + self.client = Client(directory, key=key, net=net) + else: + self.client = ClientV2(directory, net=net) + + def __getattr__(self, name): + if name in vars(self.client): + return getattr(self.client, name) + elif name in dir(ClientBase): + return getattr(self.client, name) + # temporary, for breaking changes into smaller pieces + elif name in dir(Client): + return getattr(self.client, name) + else: + raise AttributeError() + + def new_account_and_tos(self, regr, check_tos_cb=None): + """Combined register and agree_tos for V1, new_account for V2 + + :param .NewRegistration regr: + :param callable check_tos_cb: callback that raises an error if + the check does not work + """ + def _assess_tos(tos): + if check_tos_cb is not None: + check_tos_cb(tos) + if self.acme_version == 1: + regr = self.client.register(regr) + if regr.terms_of_service is not None: + _assess_tos(regr.terms_of_service) + return self.client.agree_to_tos(regr) + return regr + else: + if "terms_of_service" in self.client.directory.meta: + _assess_tos(self.client.directory.meta.terms_of_service) + regr = regr.update(terms_of_service_agreed=True) + return self.client.new_account(regr) + + def _acme_version_from_directory(self, directory): + if hasattr(directory, 'newNonce'): + return 2 + else: + return 1 + + class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 662c32942..eb03d9261 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -21,10 +21,25 @@ CERT_DER = test_util.load_vector('cert.der') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) +DIRECTORY_V1 = messages.Directory({ + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', +}) -class ClientTest(unittest.TestCase): - """Tests for acme.client.Client.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods +DIRECTORY_V2 = messages.Directory({ + 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' +}) + + +class ClientTestBase(unittest.TestCase): + """Base for tests in acme.client.""" def setUp(self): self.response = mock.MagicMock( @@ -33,21 +48,6 @@ class ClientTest(unittest.TestCase): self.net.post.return_value = self.response self.net.get.return_value = self.response - self.directory = messages.Directory({ - messages.NewRegistration: - 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: - 'https://www.letsencrypt-demo.org/acme/revoke-cert', - messages.NewAuthorization: - 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', - }) - - from acme.client import Client - self.client = Client( - directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) - self.identifier = messages.Identifier( typ=messages.IDENTIFIER_FQDN, value='example.com') @@ -84,6 +84,124 @@ class ClientTest(unittest.TestCase): # Reason code for revocation self.rsn = 1 + +class BackwardsCompatibleClientV2Test(ClientTestBase): + """Tests for acme.client.BackwardsCompatibleClientV2.""" + + def _init(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + return BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + + def test_init_downloads_directory(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + self.net.get.assert_called_once_with(uri) + + def test_init_acme_version(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.acme_version, 1) + + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.assertEqual(client.acme_version, 2) + + def test_forwarding(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.directory, client.client.directory) + self.assertEqual(client.key, KEY) + # delete this line once we finish migrating to new API: + self.assertEqual(client.register, client.client.register) + self.assertEqual(client.update_registration, client.client.update_registration) + self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') + self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') + self.assertRaises(AttributeError, client.__getattr__, 'new_account') + + def test_new_account_and_tos(self): + # v2 no tos + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().new_account.assert_called_with(self.new_reg) + + # v2 tos good + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + client.new_account_and_tos(self.new_reg, lambda x: True) + mock_client().new_account.assert_called_with( + self.new_reg.update(terms_of_service_agreed=True)) + + # v2 tos bad + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + def _tos_cb(tos): + raise errors.Error + self.assertRaises(errors.Error, client.new_account_and_tos, + self.new_reg, _tos_cb) + mock_client().new_account.assert_not_called() + + # v1 yes tos + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service="TOS") + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_called_once_with(regr) + + # v1 no tos + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service=None) + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_not_called() + + +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientV2Test, self).setUp() + from acme.client import ClientV2 + self.directory = DIRECTORY_V2 + self.client = ClientV2(directory=self.directory, net=self.net) + + def test_new_account_v2(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, self.client.new_account(self.regr)) + + +class ClientTest(ClientTestBase): + """Tests for acme.client.Client.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientTest, self).setUp() + from acme.client import Client + self.directory = DIRECTORY_V1 + self.client = Client( + directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) + def test_init_downloads_directory(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import Client @@ -104,23 +222,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments - def test_new_account_v2(self): - directory = messages.Directory({ - "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', - }) - from acme.client import ClientV2 - client = ClientV2(directory, self.net) - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, client.new_account(self.regr)) - def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 98993c4e1..21702f6a3 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -171,10 +171,31 @@ class Directory(jose.JSONDeSerializable): class Meta(jose.JSONObjectWithFields): """Directory Meta.""" - terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) caa_identities = jose.Field('caa-identities', omitempty=True) + def __init__(self, **kwargs): + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) + # pylint: disable=star-args + super(Directory.Meta, self).__init__(**kwargs) + + @property + def terms_of_service(self): + """URL for the CA TOS""" + return self._terms_of_service or self._terms_of_service_v2 + + def __iter__(self): + # When iterating over fields, use the external name 'terms_of_service' instead of + # the internal '_terms_of_service'. + for name in super(Directory.Meta, self).__iter__(): + yield name[1:] if name == '_terms_of_service' else name + + def _internal_name(self, name): + return '_' + name if name == 'terms_of_service' else name + + @classmethod def _canon_key(cls, key): return getattr(key, 'resource_type', key) @@ -251,7 +272,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) - terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) + terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index fa885d3c2..4bc60a67b 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -165,6 +165,13 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + def test_iter_meta(self): + result = False + for k in self.dir.meta: + if k == 'terms_of_service': + result = self.dir.meta[k] == 'https://example.com/acme/terms' + self.assertTrue(result) + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" diff --git a/certbot/client.py b/certbot/client.py index bc25da549..67ee8f7fa 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -37,12 +37,12 @@ from certbot.plugins import selection as plugin_selection logger = logging.getLogger(__name__) -def acme_from_config_key(config, key): +def acme_from_config_key(config, key, regr=None): "Wrangle ACME client construction" # TODO: Allow for other alg types besides RS256 - net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl), + net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl), user_agent=determine_user_agent(config)) - return acme_client.Client(config.server, key=key, net=net) + return acme_client.BackwardsCompatibleClientV2(net, key, config.server) def determine_user_agent(config): @@ -162,14 +162,7 @@ def register(config, account_storage, tos_cb=None): backend=default_backend()))) acme = acme_from_config_key(config, key) # TODO: add phone? - regr = perform_registration(acme, config) - - if regr.terms_of_service is not None: - if tos_cb is not None and not tos_cb(regr): - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") - regr = acme.agree_to_tos(regr) + regr = perform_registration(acme, config, tos_cb) acc = account.Account(regr, key) account.report_new_account(config) @@ -180,7 +173,7 @@ def register(config, account_storage, tos_cb=None): return acc, acme -def perform_registration(acme, config): +def perform_registration(acme, config, tos_cb): """ Actually register new account, trying repeatedly if there are email problems @@ -192,7 +185,8 @@ def perform_registration(acme, config): :rtype: `acme.messages.RegistrationResource` """ try: - return acme.register(messages.NewRegistration.from_data(email=config.email)) + return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), + tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: @@ -202,7 +196,7 @@ def perform_registration(acme, config): raise errors.Error(msg) else: config.email = display_ops.get_email(invalid=True) - return perform_registration(acme, config) + return perform_registration(acme, config, tos_cb) else: raise @@ -232,7 +226,7 @@ class Client(object): # Initialize ACME if account is provided if acme is None and self.account is not None: - acme = acme_from_config_key(config, self.account.key) + acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme if auth is not None: diff --git a/certbot/main.py b/certbot/main.py index 13234e0d2..ff3758985 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -496,17 +496,20 @@ def _determine_account(config): if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - def _tos_cb(regr): + def _tos_cb(terms_of_service): if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( - regr.terms_of_service, config.server)) + terms_of_service, config.server)) obj = zope.component.getUtility(interfaces.IDisplay) - return obj.yesno(msg, "Agree", "Cancel", + result = obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos", force_interactive=True) - + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..a9a87b80b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -30,31 +30,27 @@ class RegisterTest(test_util.ConfigTestCase): self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" self.account_storage = account.AccountMemoryStorage() - self.tos_cb = mock.MagicMock() def _call(self): from certbot.client import register - return register(self.config, self.account_storage, self.tos_cb) + tos_cb = mock.MagicMock() + return register(self.config, self.account_storage, tos_cb) def test_no_tos(self): - with mock.patch("certbot.client.acme_client.Client") as mock_client: - mock_client.register().terms_of_service = "http://tos" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client.new_account_and_tos().terms_of_service = "http://tos" with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): - self.tos_cb.return_value = False + mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) self.assertFalse(mock_handle.called) - self.tos_cb.return_value = True + mock_client().new_account_and_tos.side_effect = None self._call() self.assertTrue(mock_handle.called) - self.tos_cb = None - self._call() - self.assertEqual(mock_handle.call_count, 2) - def test_it(self): - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -66,9 +62,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = False msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) self.assertTrue(mock_handle.called) @@ -79,9 +75,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = True msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription"): - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) def test_needs_email(self): @@ -91,7 +87,7 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -104,9 +100,9 @@ class RegisterTest(test_util.ConfigTestCase): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) self.assertFalse(mock_handle.called) @@ -122,7 +118,7 @@ class ClientTestCommon(test_util.ConfigTestCase): self.account = mock.MagicMock(**{"key.pem": KEY}) from certbot.client import Client - with mock.patch("certbot.client.acme_client.Client") as acme: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as acme: self.acme_client = acme self.acme = acme.return_value = mock.MagicMock() self.client = Client( @@ -140,7 +136,7 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] def test_init_acme_verify_ssl(self): - net = self.acme_client.call_args[1]["net"] + net = self.acme_client.call_args[0][0] self.assertTrue(net.verify_ssl) def _mock_obtain_certificate(self): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e84de54d0..518653a53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -225,7 +225,7 @@ class RevokeTest(test_util.TempDirTestCase): 'cert_512.pem')) self.patches = [ - mock.patch('acme.client.Client', autospec=True), + mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot.client.Client'), mock.patch('certbot.main._determine_account'), mock.patch('certbot.main.display_ops.success_revocation') @@ -267,7 +267,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_with_reason(self, mock_acme_client, mock_delete_if_appropriate): mock_delete_if_appropriate.return_value = False - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): self._call("--reason " + reason) @@ -661,10 +661,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - with mock.patch('certbot.main._get_and_save_cert'): - with mock.patch('certbot.main.client.acme_from_config_key'): - args.extend(['--email', 'io@io.is']) - self._cli_missing_flag(args, "--agree-tos") @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @@ -693,7 +689,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met ua = "bandersnatch" args += ["--user-agent", ua] self._call_no_clientmock(args) - acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua) + acme_net.assert_called_once_with(mock.ANY, account=mock.ANY, verify_ssl=True, + user_agent=ua) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') @@ -1352,11 +1349,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['--cert-path', SS_CERT_PATH, '--key-path', RSA2048_KEY_PATH, '--server', server, 'revoke']) with open(RSA2048_KEY_PATH, 'rb') as f: - mock_acme_client.Client.assert_called_once_with( - server, key=jose.JWK.load(f.read()), net=mock.ANY) + mock_acme_client.BackwardsCompatibleClientV2.assert_called_once_with( + mock.ANY, jose.JWK.load(f.read()), server) with open(SS_CERT_PATH, 'rb') as f: cert = crypto_util.pyopenssl_load_certificate(f.read())[0] - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke mock_revoke.assert_called_once_with( jose.ComparableX509(cert), mock.ANY) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..60d8d6084 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -340,7 +340,7 @@ class ConfigTestCase(TempDirTestCase): self.config.cert_path = constants.CLI_DEFAULTS['auth_cert_path'] self.config.fullchain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] - self.config.server = "example.com" + self.config.server = "https://example.com" def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. From e48898a8c82e995470b909f13cc020ae7c32b5cd Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 14 Feb 2018 19:19:23 -0800 Subject: [PATCH 322/631] ACMEv2: Add Order support This adds two new classes in messages: Order and OrderResource. It also adds methods to ClientV2 to create orders, and poll orders then request issuance. The CSR is stored on the OrderResource so it can be carried along and submitted when it's time to finalize the order. --- acme/acme/client.py | 98 ++++++++++++++++++++++++++++++++++---- acme/acme/errors.py | 21 ++++++++ acme/acme/messages.py | 48 ++++++++++++++++++- acme/acme/messages_test.py | 18 ++++++- 4 files changed, 173 insertions(+), 12 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d843feaa7..219667d53 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,6 +1,7 @@ """ACME client API.""" import base64 import collections +import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -119,11 +120,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), uri=response.headers.get('Location', uri)) - if authzr.body.identifier != identifier: + if identifier is not None and authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr @@ -233,8 +234,8 @@ class Client(ClientBase): instances of `.DeserializationError` raised in `from_json()`. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` + :ivar key: `josepy.JWK` (private) + :ivar alg: `josepy.JWASignature` :ivar bool verify_ssl: Verify SSL certificates? :ivar .ClientNetwork net: Client network. Useful for testing. If not supplied, it will be initialized using `key`, `alg` and @@ -550,7 +551,6 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` - """ response = self.net.post(self.directory['newAccount'], new_account, acme_version=2) @@ -560,6 +560,84 @@ class ClientV2(ClientBase): self.net.account = regr return regr + def new_order(self, csr_pem): + """Request a new Order object from the server. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + csr = cryptography.x509.load_pem_x509_csr(csr_pem, + cryptography.hazmat.backends.default_backend()) + san_extension = next(ext for ext in csr.extensions + if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + + identifiers = [] + for name in dnsNames: + identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, + value=name)) + order = messages.NewOrder(identifiers=identifiers) + response = self.net.post(self.directory['newOrder'], order) + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url))) + return messages.OrderResource( + body=body, + uri=response.headers.get('Location', uri), + fullchain_pem=fullchain_pem, + authorizations=authorizations, + csr_pem=csr_pem) + + def poll_and_finalize(self, orderr, deadline=None): + if deadline is None: + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.poll_authorizations(orderr, deadline) + return self.finalize_order(orderr, deadline) + + def poll_authorizations(self, orderr, deadline): + """Poll Order Resource for status.""" + responses = [] + for url in orderr.body.authorizations: + while datetime.datetime.now() < deadline: + authzr = self._authzr_from_response(self.net.get(url), uri=url) + if authzr.body.status != messages.STATUS_PENDING: + responses.append(authzr) + break + time.sleep(1) + # If we didn't get a response for every authorization, we fell through + # the bottom of the loop due to hitting the deadline. + if len(responses) > orderr.body.authorizations: + raise TimeoutError() + failed = [] + for authzr in responses: + if authzr.body.status != messages.STATUS_VALID: + for chall in authzr.body.challenges: + if chall.error != None: + failed.append(authzr) + if len(failed) > 0: + raise ValidationError(failed) + return orderr.update(authorizations=responses) + + def finalize_order(self, orderr, deadline): + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) + wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) + self.net.post(latest.body.finalize, wrapped_csr) + while datetime.datetime.now() < deadline: + time.sleep(1) + response = self.net.get(orderr.uri) + body = messages.Order.from_json(response.json()) + if body.error is not None: + raise IssuanceError(body.error) + if body.certificate is not None: + certificate_response = self.net.get(body.certificate).text + return orderr.update(fullchain_pem=certificate_response) + raise TimeoutError() + + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -628,10 +706,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. - :param key: Account private key + :param josepy.JWK key: Account private key :param messages.RegistrationResource account: Account object. Required if you are - planning to use .post() with acme_version=2 for anything other than creating a new - account; may be set later after registering. + planning to use .post() with acme_version=2 for anything other than + creating a new account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. @@ -662,10 +740,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes .. todo:: Implement ``acmePath``. - :param .JSONDeSerializable obj: + :param josepy.JSONDeSerializable obj: :param str url: The URL to which this object will be POSTed :param bytes nonce: - :rtype: `.JWS` + :rtype: `josepy.JWS` """ jobj = obj.json_dumps(indent=2).encode() diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..624ccb3d9 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -83,6 +83,27 @@ class PollError(ClientError): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) +class ValidationError(Error): + """Error for authorization failures. Contains a list of authorization + resources, each of which is invalid and should have an error field. + """ + def __init__(self, failed_authzrs): + self.failed_authzrs = failed_authzrs + super(ClientError, self).__init__() + +class TimeoutError(Error): + """Error for when polling an authorization or an order times out.""" + +class IssuanceError(Error): + """Error sent by the server after requesting issuance of a certificate.""" + + def __init__(self, error): + """Initialize. + + :param messages.Error error: The error provided by the server. + """ + self.error = error + class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 21702f6a3..418bcead9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -174,7 +174,7 @@ class Directory(jose.JSONDeSerializable): _terms_of_service = jose.Field('terms-of-service', omitempty=True) _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) - caa_identities = jose.Field('caa-identities', omitempty=True) + caa_identities = jose.Field('caaIdentities', omitempty=True) def __init__(self, **kwargs): kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) @@ -504,3 +504,49 @@ class Revocation(jose.JSONObjectWithFields): certificate = jose.Field( 'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert) reason = jose.Field('reason') + + +class Order(ResourceBody): + """Order Resource Body. + + .. note:: Parsing of identifiers on response doesn't work right now; to make + it work we would need to set up the equivalent of Identifier.from_json, but + for a list. + :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: + :ivar list of str authorizations: URLs of authorizations. + :ivar str certificate: URL to download certificate as a fullchain PEM. + :ivar str finalize: URL to POST to to request issuance once all + authorizations have "valid" status. + :ivar datetime.datetime expires: When the order expires. + :ivar .Error error: Any error that occurred during finalization, if applicable. + """ + identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + authorizations = jose.Field('authorizations', omitempty=True) + certificate = jose.Field('certificate', omitempty=True) + finalize = jose.Field('finalize', omitempty=True) + expires = fields.RFC3339Field('expires', omitempty=True) + error = jose.Field('error', omitempty=True, decoder=Error.from_json) + +class OrderResource(ResourceWithURI): + """Order Resource. + + :ivar acme.messages.Order body: + :ivar str csr_pem: The CSR this Order will be finalized with. + :ivar list of acme.messages.AuthorizationResource authorizations: + Fully-fetched AuthorizationResource objects. + :ivar str fullchain_pem: The fetched contents of the certificate URL + produced once the order was finalized, if it's present. + """ + body = jose.Field('body', decoder=Order.from_json) + csr_pem = jose.Field('csr_pem', omitempty=True) + authorizations = jose.Field('authorizations') + fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + +@Directory.register +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 4bc60a67b..64bc81efd 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -157,7 +157,7 @@ class DirectoryTest(unittest.TestCase): 'meta': { 'terms-of-service': 'https://example.com/acme/terms', 'website': 'https://www.example.com/', - 'caa-identities': ['example.com'], + 'caaIdentities': ['example.com'], }, }) @@ -408,5 +408,21 @@ class RevocationTest(unittest.TestCase): hash(Revocation.from_json(self.rev.to_json())) +class OrderResourceTest(unittest.TestCase): + """Tests for acme.messages.OrderResource.""" + + def setUp(self): + from acme.messages import OrderResource + self.regr = OrderResource( + body=mock.sentinel.body, uri=mock.sentinel.uri) + + def test_to_partial_json(self): + self.assertEqual(self.regr.to_json(), { + 'body': mock.sentinel.body, + 'uri': mock.sentinel.uri, + 'authorizations': None, + }) + + if __name__ == '__main__': unittest.main() # pragma: no cover From 70a75ebe9dffb7d59e701e2dcb04ce23a1bcc0fc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Feb 2018 19:40:14 -0800 Subject: [PATCH 323/631] Add tests and fix minor bugs in Order support * delint * refactor client tests * Add test for new order and fix identifiers parsing. * Add poll_and_finalize test * Test and fix poll_authorizations timeout * Add test_failed_authorizations * Add test_poll_authorizations_success * Test and fix finalize_order success * add test_finalize_order_timeout * add test_finalize_order_error * test sleep code --- acme/acme/client.py | 38 ++++++-- acme/acme/client_test.py | 181 +++++++++++++++++++++++++++++++-------- acme/acme/errors.py | 3 +- acme/acme/messages.py | 7 +- 4 files changed, 180 insertions(+), 49 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 219667d53..bc93ca06f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -586,12 +586,23 @@ class ClientV2(ClientBase): authorizations.append(self._authzr_from_response(self.net.get(url))) return messages.OrderResource( body=body, - uri=response.headers.get('Location', uri), - fullchain_pem=fullchain_pem, + uri=response.headers.get('Location'), authorizations=authorizations, csr_pem=csr_pem) def poll_and_finalize(self, orderr, deadline=None): + """Poll authorizations and finalize the order. + + If no deadline is provided, this method will timeout after 90 + seconds. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ if deadline is None: deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.poll_authorizations(orderr, deadline) @@ -609,8 +620,8 @@ class ClientV2(ClientBase): time.sleep(1) # If we didn't get a response for every authorization, we fell through # the bottom of the loop due to hitting the deadline. - if len(responses) > orderr.body.authorizations: - raise TimeoutError() + if len(responses) < len(orderr.body.authorizations): + raise errors.TimeoutError() failed = [] for authzr in responses: if authzr.body.status != messages.STATUS_VALID: @@ -618,24 +629,33 @@ class ClientV2(ClientBase): if chall.error != None: failed.append(authzr) if len(failed) > 0: - raise ValidationError(failed) + raise errors.ValidationError(failed) return orderr.update(authorizations=responses) def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(latest.body.finalize, wrapped_csr) + self.net.post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) body = messages.Order.from_json(response.json()) if body.error is not None: - raise IssuanceError(body.error) + raise errors.IssuanceError(body.error) if body.certificate is not None: certificate_response = self.net.get(body.certificate).text - return orderr.update(fullchain_pem=certificate_response) - raise TimeoutError() + return orderr.update(body=body, fullchain_pem=certificate_response) + raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index eb03d9261..11516c02f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +import copy import datetime import json import unittest @@ -18,6 +19,8 @@ from acme import test_util CERT_DER = test_util.load_vector('cert.der') +CERT_SAN_PEM = test_util.load_vector('cert-san.pem') +CSR_SAN_PEM = test_util.load_vector('csr-san.pem') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) @@ -34,7 +37,8 @@ DIRECTORY_V1 = messages.Directory({ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', - 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', + 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', }) @@ -57,8 +61,7 @@ class ClientTestBase(unittest.TestCase): contact=self.contact, key=KEY.public_key()) self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( - body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - terms_of_service='https://www.letsencrypt-demo.org/tos') + body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') # Authorization authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1' @@ -75,15 +78,6 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) - # Request issuance - self.certr = messages.CertificateResource( - body=messages_test.CERT, authzrs=(self.authzr,), - uri='https://www.letsencrypt-demo.org/acme/cert/1', - cert_chain_uri='https://www.letsencrypt-demo.org/ca') - - # Reason code for revocation - self.rsn = 1 - class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -168,37 +162,29 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().agree_to_tos.assert_not_called() -class ClientV2Test(ClientTestBase): - """Tests for acme.client.ClientV2.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods - - def setUp(self): - super(ClientV2Test, self).setUp() - from acme.client import ClientV2 - self.directory = DIRECTORY_V2 - self.client = ClientV2(directory=self.directory, net=self.net) - - def test_new_account_v2(self): - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, self.client.new_account(self.regr)) - - class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" # pylint: disable=too-many-instance-attributes,too-many-public-methods def setUp(self): super(ClientTest, self).setUp() - from acme.client import Client + self.directory = DIRECTORY_V1 + + # Registration + self.regr = self.regr.update( + terms_of_service='https://www.letsencrypt-demo.org/tos') + + # Request issuance + self.certr = messages.CertificateResource( + body=messages_test.CERT, authzrs=(self.authzr,), + uri='https://www.letsencrypt-demo.org/acme/cert/1', + cert_chain_uri='https://www.letsencrypt-demo.org/ca') + + # Reason code for revocation + self.rsn = 1 + + from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -554,6 +540,129 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + + def setUp(self): + super(ClientV2Test, self).setUp() + + self.directory = DIRECTORY_V2 + + from acme.client import ClientV2 + self.client = ClientV2(self.directory, self.net) + + self.new_reg = self.new_reg.update(terms_of_service_agreed=True) + + self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2' + self.authz2 = self.authz.update(identifier=messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='www.example.com'), + status=messages.STATUS_PENDING) + self.authzr2 = messages.AuthorizationResource( + body=self.authz2, uri=self.authzr_uri2) + + self.order = messages.Order( + identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, + authorizations=(self.authzr.uri, self.authzr_uri2), + finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') + self.orderr = messages.OrderResource( + body=self.order, + uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1', + authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM) + + def test_new_account(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + + def test_new_order(self): + order_response = copy.deepcopy(self.response) + order_response.status_code = http_client.CREATED + order_response.json.return_value = self.order.to_json() + order_response.headers['Location'] = self.orderr.uri + self.net.post.return_value = order_response + + authz_response = copy.deepcopy(self.response) + authz_response.json.return_value = self.authz.to_json() + authz_response.headers['Location'] = self.authzr.uri + authz_response2 = self.response + authz_response2.json.return_value = self.authz2.to_json() + authz_response2.headers['Location'] = self.authzr2.uri + self.net.get.side_effect = (authz_response, authz_response2) + + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + + @mock.patch('acme.client.datetime') + def test_poll_and_finalize(self, mock_datetime): + mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15) + mock_datetime.timedelta = datetime.timedelta + expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90) + + self.client.poll_authorizations = mock.Mock(return_value=self.orderr) + self.client.finalize_order = mock.Mock(return_value=self.orderr) + + self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr) + self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline) + self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline) + + @mock.patch('acme.client.datetime') + def test_poll_authorizations_timeout(self, mock_datetime): + now_side_effect = [datetime.datetime(2018, 2, 15), + datetime.datetime(2018, 2, 16), + datetime.datetime(2018, 2, 17)] + mock_datetime.datetime.now.side_effect = now_side_effect + self.response.json.side_effect = [ + self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()] + + self.assertRaises( + errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1]) + + def test_poll_authorizations_failure(self): + deadline = datetime.datetime(9999, 9, 9) + challb = self.challr.body.update(status=messages.STATUS_INVALID, + error=messages.Error.with_code('unauthorized')) + authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,)) + self.response.json.return_value = authz.to_json() + + self.assertRaises( + errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline) + + def test_poll_authorizations_success(self): + deadline = datetime.datetime(9999, 9, 9) + updated_authz2 = self.authz2.update(status=messages.STATUS_VALID) + updated_authzr2 = messages.AuthorizationResource( + body=updated_authz2, uri=self.authzr_uri2) + updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2]) + + self.response.json.side_effect = ( + self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json()) + self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr) + + def test_finalize_order_success(self): + updated_order = self.order.update( + certificate='https://www.letsencrypt-demo.org/acme/cert/') + updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM) + + self.response.json.return_value = updated_order.to_json() + self.response.text = CERT_SAN_PEM + + deadline = datetime.datetime(9999, 9, 9) + self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr) + + def test_finalize_order_error(self): + updated_order = self.order.update(error=messages.Error.with_code('unauthorized')) + self.response.json.return_value = updated_order.to_json() + + deadline = datetime.datetime(9999, 9, 9) + self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline) + + def test_finalize_order_timeout(self): + deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 624ccb3d9..991335958 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -89,7 +89,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ClientError, self).__init__() + super(ValidationError, self).__init__() class TimeoutError(Error): """Error for when polling an authorization or an order times out.""" @@ -103,6 +103,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error + super(IssuanceError, self).__init__() class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 418bcead9..23cd66c63 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,9 +509,6 @@ class Revocation(jose.JSONObjectWithFields): class Order(ResourceBody): """Order Resource Body. - .. note:: Parsing of identifiers on response doesn't work right now; to make - it work we would need to set up the equivalent of Identifier.from_json, but - for a list. :ivar list of .Identifier: List of identifiers for the certificate. :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. @@ -530,6 +527,10 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) + @identifiers.decoder + def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + return tuple(Identifier.from_json(identifier) for identifier in value) + class OrderResource(ResourceWithURI): """Order Resource. From adec7a8feda1b6ad4f989d4c293fdb78a646917a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 09:51:27 -0800 Subject: [PATCH 324/631] Cleanup dockerfile-dev (#5435) * cleanup dockerfile-dev * map port 80 * remove python3-dev package --- Dockerfile-dev | 65 ++++++---------------------------------------- docker-compose.yml | 1 + 2 files changed, 9 insertions(+), 57 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 581b58f11..9e35ebec8 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,70 +1,21 @@ # This Dockerfile builds an image for development. -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington -MAINTAINER Yan +FROM ubuntu:xenial -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt +# Note: this only exposes the port to other docker containers. +EXPOSE 80 443 WORKDIR /opt/certbot/src -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev git -y && \ +COPY . . +RUN apt-get update && \ + apt-get install apache2 git nginx-light -y && \ + letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ - -# all above files are necessary for setup.py, however, package source -# code directory has to be copied separately to a subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/ -COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ -COPY tests /opt/certbot/src/tests/ - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ - /opt/certbot/venv/bin/pip install -U pip && \ - /opt/certbot/venv/bin/pip install -U setuptools && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx \ - -e /opt/certbot/src/letshelp-certbot \ - -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +RUN VENV_NAME="../venv" tools/venv.sh ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/docker-compose.yml b/docker-compose.yml index 00d3d4c72..75a5b9aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: context: . dockerfile: Dockerfile-dev ports: + - "80:80" - "443:443" volumes: - .:/opt/certbot/src From 2a142aa93288d0db87b8f12dc71fce70ee3ce482 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 14:47:10 -0800 Subject: [PATCH 325/631] Make Certbot depend on josepy (#5542) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ce505a62e..47b5b0b2c 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2', # load_pem_x509_certificate + 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', From e95e963ad62735a0cd23a46e27a7cbef3790afb2 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 16 Feb 2018 16:05:16 -0800 Subject: [PATCH 326/631] Get common name from CSR in new_order in ClientV2 (#5587) * switch new_order to use crypto_util._pyopenssl_cert_or_req_san * move certbot.crypto_util._get_names_from_loaded_cert_or_req functionality to acme.crypto_util._pyopenssl_cert_or_req_all_names --- acme/acme/client.py | 10 ++++------ acme/acme/crypto_util.py | 9 +++++++++ acme/acme/crypto_util_test.py | 24 ++++++++++++++++++++++++ acme/acme/testdata/cert-nocn.der | Bin 0 -> 1397 bytes certbot/crypto_util.py | 8 +------- 5 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 acme/acme/testdata/cert-nocn.der diff --git a/acme/acme/client.py b/acme/acme/client.py index bc93ca06f..1f4ae4fad 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,7 +1,6 @@ """ACME client API.""" import base64 import collections -import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -17,6 +16,7 @@ import re import requests import sys +from acme import crypto_util from acme import errors from acme import jws from acme import messages @@ -568,11 +568,9 @@ class ClientV2(ClientBase): :returns: The newly created order. :rtype: OrderResource """ - csr = cryptography.x509.load_pem_x509_csr(csr_pem, - cryptography.hazmat.backends.default_backend()) - san_extension = next(ext for ext in csr.extensions - if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) identifiers = [] for name in dnsNames: diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..a986721f0 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -186,6 +186,15 @@ def make_csr(private_key_pem, domains, must_staple=False): return OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr) +def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): + common_name = loaded_cert_or_req.get_subject().CN + sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..14aaac8b5 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -65,6 +65,30 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): + """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" + + @classmethod + def _call(cls, loader, name): + # pylint: disable=protected-access + from acme.crypto_util import _pyopenssl_cert_or_req_all_names + return _pyopenssl_cert_or_req_all_names(loader(name)) + + def _call_cert(self, name): + return self._call(test_util.load_cert, name) + + def test_cert_one_san_no_common(self): + self.assertEqual(self._call_cert('cert-nocn.der'), + ['no-common-name.badssl.com']) + + def test_cert_no_sans_yes_common(self): + self.assertEqual(self._call_cert('cert.pem'), ['example.com']) + + def test_cert_two_sans_yes_common(self): + self.assertEqual(self._call_cert('cert-san.pem'), + ['example.com', 'www.example.com']) + + class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" diff --git a/acme/acme/testdata/cert-nocn.der b/acme/acme/testdata/cert-nocn.der new file mode 100644 index 0000000000000000000000000000000000000000..59da83ccc6dc3f2a05fe762d6d3372cb446d88ac GIT binary patch literal 1397 zcmXqLVl6aiVu@V9%*4pVB#@u9Us#IesM(YUabM(~3ac9MvT?DFX?R2$!&+v%jyui@$=iqk>OnZe~epilL2x6-bU**cd7o6zr(rUzDDhmsyoq zl9`{U5SEyenF3)3rzV#cr78rc7L@_*baph56X!KFH!wCbHUNVtab6<>12ZEdBV$ub zQ_CoWY$SJw1{(?+2twS=Ed4`e(;vm)B!UDdfIVG98F8R5MnRyDq!9E5}j7rD>$H>aS+{DPw02Jq9 zYGPz$IK=YiN6$N!Lz`~|TrG$=Gg;zw%!&-p4N6;oPMx<=Tc^w}=YV&DkI$=3TMqCy zY}@d&=wuty<y8VeR_DBi?&nPSFc6UzRSD*z?WN>8RUfv3B+OC9k>W3I?eu7U#71 zRE9penEisy)$G-j&A)YDMK2k9 zJXb0_en_M$N8^>b*Zo&HTZ^{TMJ*D$lesb6U)Rz-AiH|l^)LeDLwhwS|v73 zy^Xs(Wq8!*E&kp8-JytK%_NSSJ>f0EjjX4i{mTxS;h^4nTwr%`!TU@JM|RiHnx855 zwmOGb*1Vo9<z44-7_GWflnou?CS@pS5qDU+ce3f47#0 zhnCQGX{UoK2C^Upd@N!tB6HsztZwACa?dQ3s1-Dyyl1Jj{cmt0ljUb*{LjL|%*49D zfCr>p7{q5XV1Q^=Wf3zFVdKzdV`ODzXJ&-6m<$3yiWOKq4crZ^*?52oSQs}MGBPnT zvlwU_Xu=dSF^b7%l#~<{Tj}c;gOi?Ka(-@pN?+{ z-k{bX6vj7TLb9iXGJ8r4RN%%kwn;#ppPyV@fMPCC70{=GEV2gDO_F)}y1<~w&Ck=# zOUzBxOG->BE(UuSIU@i|5MV}NWXSxZ>ULmW+UMOr#Oj|(nYf(rnDb6zkyW|Xz_ z$9{(K%Jus_l6IO3g=HA){+awKO)O|fmTT1Z+#7Ryp6LWmIpcR=gMC-d`J0n;Teo@| z6zW||Pht60z4=n>rj4&JFaGB1)RMV*x5}HhJC+@K>piVA&uZReo2JynpBFM}&U!55 z^j+hWs<_`F(Vy8@W!9tBm(mqy|NpbeP?kwtdY#U4)jc=2tozb#`ZwppYdhvmr}>U7 l*UbLhr)aUs$L-SL{YJ0C_h)*Gh(7yrDof|BxSMr#8URDu=Gy=O literal 0 HcmV?d00001 diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..8368855cd 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -340,14 +340,8 @@ def _get_names_from_cert_or_req(cert_or_req, load_func, typ): def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): - common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access - sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) - - if common_name is None: - return sans - else: - return [common_name] + [d for d in sans if d != common_name] + return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): From 42638afc7568c161f467a4f6f385d597a6e329d9 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 26 Jan 2018 11:47:25 +0200 Subject: [PATCH 327/631] Drop support for EOL Python 2.6 and 3.3 * Drop support for EOL Python 2.6 * Use more helpful assertIn/NotIn instead of assertTrue/False * Drop support for EOL Python 3.3 * Remove redundant Python 3.3 code * Restore code for RHEL 6 and virtualenv for Py2.7 * Revert pipstrap.py to upstream * Merge py26_packages and non_py26_packages into all_packages * Revert changes to *-auto in root * Update by calling letsencrypt-auto-source/build.py * Revert permissions for pipstrap.py --- .travis.yml | 8 ---- acme/acme/__init__.py | 10 ---- acme/acme/crypto_util.py | 4 +- acme/acme/crypto_util_test.py | 12 ++--- acme/setup.py | 9 ---- certbot-apache/setup.py | 2 - certbot-compatibility-test/setup.py | 2 - certbot-dns-cloudflare/setup.py | 2 - certbot-dns-cloudxns/setup.py | 1 - certbot-dns-digitalocean/setup.py | 2 - certbot-dns-dnsimple/setup.py | 1 - certbot-dns-dnsmadeeasy/setup.py | 1 - certbot-dns-google/setup.py | 2 - certbot-dns-luadns/setup.py | 1 - certbot-dns-nsone/setup.py | 1 - certbot-dns-rfc2136/setup.py | 2 - certbot-dns-route53/setup.py | 2 - certbot-nginx/setup.py | 2 - certbot/log.py | 48 ++++--------------- certbot/main.py | 12 ----- docs/contributing.rst | 2 +- docs/install.rst | 2 +- letsencrypt-auto-source/letsencrypt-auto | 35 ++++---------- .../letsencrypt-auto.template | 35 ++++---------- letsencrypt-auto-source/tests/auto_test.py | 16 +++---- letshelp-certbot/letshelp_certbot/apache.py | 4 +- letshelp-certbot/setup.py | 2 - setup.py | 9 ---- tests/letstest/scripts/test_tox.sh | 8 +--- tests/run_http_server.py | 2 +- tools/install_and_test.sh | 4 -- tox.ini | 41 ++++------------ 32 files changed, 56 insertions(+), 228 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35666d8e6..1077d99d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,18 +25,10 @@ matrix: addons: - python: "2.7" env: TOXENV=lint - - python: "2.6" - env: TOXENV=py26 - sudo: required - services: docker - python: "2.7" env: TOXENV=py27-oldest sudo: required services: docker - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..78ba41d0f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,7 +5,6 @@ import logging import os import re import socket -import sys import OpenSSL @@ -130,8 +129,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..22a507811 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -170,9 +170,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +188,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..ba5c8e6fb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -25,13 +25,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -58,10 +51,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..d7c223a0a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8f9f897cf..7e1b059e2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -40,10 +40,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..d619f1872 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..5d14f3e29 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..ce8fedd46 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..06af16759 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..7c0f3ed86 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..de881ad84 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -44,10 +44,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..0d580b7ee 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..c0ba11470 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..5161e7a94 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..09f8a7d52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -32,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..37c477ef6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..76a90d499 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,6 @@ import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -1218,17 +1217,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/docs/contributing.rst b/docs/contributing.rst index 83b607e15..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -422,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index c18c3cdbc..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..aed15a8ef 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -781,20 +781,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -965,18 +956,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..b3d6ab740 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -320,20 +320,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -504,18 +495,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..8c2bfc079 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertNotIn('Creating virtual environment...', out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..7c8c39068 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -31,10 +31,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/setup.py b/setup.py index ce505a62e..e3824a7f7 100644 --- a/setup.py +++ b/setup.py @@ -52,13 +52,6 @@ install_requires = [ 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -98,10 +91,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tox.ini b/tox.ini index 20f5cda32..971aa7631 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,22 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ certbot-dns-cloudflare \ + certbot-dns-cloudxns \ certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +51,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -104,7 +84,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will From 73bd801f352fb9bad7fe8bc35c368f573c15a21e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:21:02 -0800 Subject: [PATCH 328/631] add and use request_authorizations --- acme/acme/client.py | 14 ++++++++++++++ certbot/auth_handler.py | 10 ++++++---- certbot/client.py | 23 ++++++++--------------- certbot/main.py | 2 +- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1f4ae4fad..6b4d65233 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,6 +671,7 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) + self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -705,6 +706,19 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) + def request_authorizations(self, csr_pem): + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return authorizations + else: + self.orderr = self.client.new_order(csr_pem) + return self.orderr.authorizations + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..4f88199e3 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,10 +48,10 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, domains, best_effort=False): + def get_authorizations(self, csr_pem, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param list csr_pem: CSR containing domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -62,8 +62,10 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + authzrs = self.acme.request_authorizations(csr_pem) + for authzr in authzrs: + self.authzr[authzr.body.identifier.value] = authzr + domains = self.authzr.keys() self._choose_challenges(domains) config = zope.component.getUtility(interfaces.IConfig) diff --git a/certbot/client.py b/certbot/client.py index 67ee8f7fa..8e3ec6c62 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,9 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, authzr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. @@ -261,10 +257,10 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + authzr = self.auth_handler.get_authorizations(csr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -307,13 +303,6 @@ class Client(object): :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -326,8 +315,12 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + authzr = self.auth_handler.get_authorizations( + csr, + self.config.allow_subset_of_names) + certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + csr, authzr=authzr) return certr, chain, key, csr diff --git a/certbot/main.py b/certbot/main.py index ff3758985..d01f68920 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + certr, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) From eaf739184cf517d5a3f5103caa072bb0cd39b4e9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:29:42 -0800 Subject: [PATCH 329/631] pass pem to auth_handler --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 8e3ec6c62..d7d2acb14 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,7 +260,7 @@ class Client(object): logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(csr) + authzr = self.auth_handler.get_authorizations(csr.data) certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,7 +316,7 @@ class Client(object): csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) authzr = self.auth_handler.get_authorizations( - csr, + csr.data, self.config.allow_subset_of_names) certr, chain = self.obtain_certificate_from_csr( From ea2022588b4d95f1d14849b918b2cd3788cc6084 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:32:49 -0800 Subject: [PATCH 330/631] add docstring --- acme/acme/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6b4d65233..e7cf016bb 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -707,6 +707,16 @@ class BackwardsCompatibleClientV2(object): return self.client.new_account(regr) def request_authorizations(self, csr_pem): + """Request authorizations for the domains in csr_pem. + + Calls request_domain_challenges for each domain for V1, and + calls new_order and saves the result for V2. + + :param str csr_pem: A CSR in PEM format. + + :returns: List of Authorization Resources. + :rtype: list of `.AuthorizationResource` + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access From 20d0b91c710bd8110aa5ee23082f45583e7789a4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:35:10 -0800 Subject: [PATCH 331/631] switch interface to new_order and remove best_effort flag --- acme/acme/client.py | 18 ++++++++---------- certbot/auth_handler.py | 27 ++++++++++----------------- certbot/client.py | 15 ++++----------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e7cf016bb..1838fab42 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,7 +671,6 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) - self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -706,16 +705,16 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) - def request_authorizations(self, csr_pem): - """Request authorizations for the domains in csr_pem. + def new_order(self, csr_pem): + """Request a new Order object from the server. - Calls request_domain_challenges for each domain for V1, and - calls new_order and saves the result for V2. + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. :param str csr_pem: A CSR in PEM format. - :returns: List of Authorization Resources. - :rtype: list of `.AuthorizationResource` + :returns: The newly created order. + :rtype: OrderResource """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) @@ -724,10 +723,9 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return authorizations + return messages.OrderResource(authorizations=authorizations) else: - self.orderr = self.client.new_order(csr_pem) - return self.orderr.authorizations + return self.client.new_order(csr_pem) def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 4f88199e3..825513329 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,12 +48,11 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, csr_pem, best_effort=False): + def handle_authorizations(self, orderr): """Retrieve all authorizations for challenges. - :param list csr_pem: CSR containing domains for authorization - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + :param acme.messages.OrderResource orderr: must have + authorizations filled in :returns: List of authorization resources :rtype: list @@ -62,7 +61,7 @@ class AuthHandler(object): authorizations """ - authzrs = self.acme.request_authorizations(csr_pem) + authzrs = orderr.authorizations for authzr in authzrs: self.authzr[authzr.body.identifier.value] = authzr domains = self.authzr.keys() @@ -80,7 +79,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(resp) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -124,7 +123,7 @@ class AuthHandler(object): return resp - def _respond(self, resp, best_effort): + def _respond(self, resp): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -137,7 +136,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(chall_update) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -169,7 +168,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + self, chall_update, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -190,14 +189,8 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - if best_effort: - comp_domains.add(domain) - logger.warning( - "Challenge failed for domain %s", - domain) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index d7d2acb14..61e9db635 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,14 +235,12 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, authzr=None): + def obtain_certificate_from_csr(self, csr): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -259,8 +257,8 @@ class Client(object): logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(csr.data) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -315,12 +313,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - authzr = self.auth_handler.get_authorizations( - csr.data, - self.config.allow_subset_of_names) - - certr, chain = self.obtain_certificate_from_csr( - csr, authzr=authzr) + certr, chain = self.obtain_certificate_from_csr(csr) return certr, chain, key, csr From 68e24a8ea7eeb405592e40ed340baff6f3f3821b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:59:51 -0800 Subject: [PATCH 332/631] start test updates --- certbot/tests/auth_handler_test.py | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..d424e59ca 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,8 +57,8 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, "failure.com", [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -92,12 +92,11 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -115,14 +114,13 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -146,7 +144,11 @@ class GetAuthorizationsTest(unittest.TestCase): mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -169,31 +171,33 @@ class GetAuthorizationsTest(unittest.TestCase): def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) @@ -201,20 +205,20 @@ class GetAuthorizationsTest(unittest.TestCase): self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): + def _validate_all(self, unused_1): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 9c84fe1144cc5ccec5fc9cfcdf7c5adf8cfdf15c Mon Sep 17 00:00:00 2001 From: Matt Christian Date: Sun, 18 Feb 2018 15:45:22 -0600 Subject: [PATCH 333/631] Add override class for ID="ol" AKA Oracle Linux Server, a clone of CentOS/RHEL. --- certbot-apache/certbot_apache/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, From d6b4e2001b404b1f9bd4c3929e888726d583ee57 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:19:04 -0800 Subject: [PATCH 334/631] put back in best_effort code, with a todo for actually supporting it in ACMEv2 --- certbot/auth_handler.py | 22 +++++++++++++++------- certbot/client.py | 11 ++++++++--- certbot/tests/auth_handler_test.py | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 825513329..662cadc65 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,11 +48,13 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def handle_authorizations(self, orderr): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: Whether or not all authorizations are + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -79,7 +81,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp) + self._respond(resp, best_effort) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -123,7 +125,7 @@ class AuthHandler(object): return resp - def _respond(self, resp): + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -136,7 +138,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update) + self._poll_challenges(chall_update, best_effort) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -168,7 +170,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, min_sleep=3, max_rounds=15): + self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -189,8 +191,14 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + if best_effort: + comp_domains.add(domain) + logger.warning( + "Challenge failed for domain %s", + domain) + else: + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index 61e9db635..5feea662d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,12 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, csr, best_effort=False): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. + :param bool best_effort: Whether or not all authorizations are required :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,7 +259,11 @@ class Client(object): logger.debug("CSR: %s", csr) orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + if best_effort: + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop + pass certr = self.acme.request_issuance( jose.ComparableX509( @@ -313,7 +318,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr) + certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) return certr, chain, key, csr diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index d424e59ca..ea8b006c4 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -218,7 +218,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1): + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 11f2f1e576243a255afbcd166b042cab2e2f1c4b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:20:41 -0800 Subject: [PATCH 335/631] remove extra spaces --- certbot/auth_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 662cadc65..47d806b94 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -194,8 +194,8 @@ class AuthHandler(object): if best_effort: comp_domains.add(domain) logger.warning( - "Challenge failed for domain %s", - domain) + "Challenge failed for domain %s", + domain) else: all_failed_achalls.update( updated for _, updated in failed_achalls) From a0e84e65ce9cfc041f341a6bedf36525e49fa1b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:29:04 -0800 Subject: [PATCH 336/631] auth_handler tests are happy --- certbot/tests/auth_handler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index ea8b006c4..3633b673d 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -118,7 +118,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) mock_order = mock.MagicMock(authorizations=[authzr]) authzr = self.handler.handle_authorizations(mock_order) From 76a0cbf9c23b9827c4bf94e9a6521a4bf049a466 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:43:12 -0800 Subject: [PATCH 337/631] client tests passing --- certbot/tests/client_test.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a9a87b80b..b6cbca367 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -141,16 +141,17 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] + self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.new_order.return_value = mock.sentinel.orderr def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, + self.client.auth_handler.handle_authorizations.assert_called_once_with( + mock.sentinel.orderr, self.config.allow_subset_of_names) - authzr = self.client.auth_handler.get_authorizations() + authzr = self.client.auth_handler.handle_authorizations() self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( @@ -167,31 +168,19 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + best_effort=False)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) - # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) @@ -204,13 +193,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) + test_csr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -220,13 +206,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) + test_csr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -276,7 +259,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) - self.client.auth_handler.get_authorizations.return_value = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) From 3dfeb483ee853333b11e829ef7d985ebf5ad7269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:49:23 -0800 Subject: [PATCH 338/631] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b6cbca367..570080e3b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -166,7 +166,6 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -191,7 +190,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -204,7 +202,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertRaises( acme_errors.Error, From d6af978472d7519b615e8894903383610ab41269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:52:11 -0800 Subject: [PATCH 339/631] remove if/pass --- certbot/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 5feea662d..65e85a159 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,10 +260,8 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - if best_effort: - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop - pass + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop certr = self.acme.request_issuance( jose.ComparableX509( From d29c637bf94cf083ef3dc1648bed3697229b8025 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:36:35 -0800 Subject: [PATCH 340/631] support best_effort --- certbot/client.py | 29 +++++++++++++++++++++-------- certbot/tests/client_test.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 65e85a159..d00055eae 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, best_effort=False): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param bool best_effort: Whether or not all authorizations are required + :param acme.messages.OrderResource orderr: contains authzrs :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,10 +258,12 @@ class Client(object): logger.debug("CSR: %s", csr) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop + if orderr is None: + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) + else: + authzr = orderr.authorizations + certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,9 +318,20 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + if successful_domains != domains: + if not self.config.dry_run: + # TODO: delete keys + pass + return self.obtain_certificate(successful_domains) + else: + certr, chain = self.obtain_certificate_from_csr(csr, orderr) + + return certr, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 570080e3b..376bf5a90 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -134,6 +134,7 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock(authorizations=[None]) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -144,11 +145,11 @@ class ClientTest(ClientTestCommon): self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain - self.acme.new_order.return_value = mock.sentinel.orderr + self.acme.new_order.return_value = self.eg_order def _check_obtain_certificate(self): self.client.auth_handler.handle_authorizations.assert_called_once_with( - mock.sentinel.orderr, + self.eg_order, self.config.allow_subset_of_names) authzr = self.client.auth_handler.handle_authorizations() @@ -166,15 +167,26 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, - best_effort=False)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() + # Test for orderr=None + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + test_csr, + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order) + # Test for no auth_handler self.client.auth_handler = None self.assertRaises( @@ -190,11 +202,15 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - test_csr)) + test_csr, + orderr=orderr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -202,11 +218,15 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - test_csr) + test_csr, + orderr=orderr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -256,6 +276,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) + self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From 7c073dbcaf6d6789da01734f0a32951b18b3e4d6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:38:18 -0800 Subject: [PATCH 341/631] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 376bf5a90..f0ef077b2 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -205,7 +204,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -221,7 +219,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, From 051664a142a8c77121c79cb07307379a63d76874 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:39:30 -0800 Subject: [PATCH 342/631] lint --- certbot/tests/client_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f0ef077b2..f10053616 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -201,7 +201,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertEqual( @@ -216,7 +215,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertRaises( From d5a90c5a6e58a068c0fbd8bf800f9953425a9693 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:43:27 -0800 Subject: [PATCH 343/631] delete key and csr before trying again --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d00055eae..404e1e0d9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -325,8 +325,8 @@ class Client(object): if successful_domains != domains: if not self.config.dry_run: - # TODO: delete keys - pass + os.remove(key.file) + os.remove(csr.file) return self.obtain_certificate(successful_domains) else: certr, chain = self.obtain_certificate_from_csr(csr, orderr) From 26bcaff85cae0b38280f576ac732bd5a4744d2f8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:59:58 -0800 Subject: [PATCH 344/631] add test for new_order for v2 --- acme/acme/client_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 11516c02f..1ba41cd7d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,6 +161,21 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() + def test_new_order_v1(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" From 65d0b9674cb4008e624c1a58cc7c8e4ee91fcdb6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:01:35 -0800 Subject: [PATCH 345/631] Fix client test --- certbot/tests/client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f10053616..6b90eab83 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,6 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From a7eadf88629d9cbba611731b9fe555bc0ba5cb84 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:08:46 -0800 Subject: [PATCH 346/631] add new order test for v1 --- acme/acme/client_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1ba41cd7d..773d59aa2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,10 +161,18 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() - def test_new_order_v1(self): + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) def test_new_order_v2(self): self.response.json.return_value = DIRECTORY_V2.to_json() From dea43e90b629f629a19e6f865fbac2930421a733 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:11:36 -0800 Subject: [PATCH 347/631] lint --- acme/acme/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 773d59aa2..1b33ca5d7 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -164,7 +164,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): @mock.patch('OpenSSL.crypto.load_certificate_request') @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, - mock_load_certificate_request): + unused_mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] mock_csr_pem = mock.MagicMock() From df50f2d5fa3ddd70cfcd01cf63c305431c577d46 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:12:15 -0800 Subject: [PATCH 348/631] client test --- certbot/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6b90eab83..ecd77bdeb 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - auth_handler.handle_authorizations(orderr) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From d13a4ed18da3f2a0b8f88076bf15f778337259ea Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:50:18 -0800 Subject: [PATCH 349/631] add tests for if partial auth success --- certbot/tests/client_test.py | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ecd77bdeb..f4a8a5c8a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -147,10 +147,13 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.return_value = mock.sentinel.chain self.acme.new_order.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.handle_authorizations.assert_called_once_with( - self.eg_order, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) authzr = self.client.auth_handler.handle_authorizations() @@ -238,6 +241,21 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + + authzr = self._authzr_from_domains(["example.com"]) + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") def test_obtain_certificate_dry_run(self, mock_acme_crypto, mock_crypto): @@ -255,22 +273,25 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. - + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr + + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr @@ -281,7 +302,7 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') From ea3b78e3c9ff11da1d6919f2c5c36d53623512d3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 20 Feb 2018 18:53:48 -0800 Subject: [PATCH 350/631] update order object with returned authorizations (#5598) --- certbot/client.py | 6 +++--- certbot/tests/client_test.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 404e1e0d9..dd11f2204 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -261,9 +261,8 @@ class Client(object): if orderr is None: orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr) - else: - authzr = orderr.authorizations - + orderr = orderr.update(authorizations=authzr) + authzr = orderr.authorizations certr = self.acme.request_issuance( jose.ComparableX509( @@ -320,6 +319,7 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + orderr = orderr.update(authorizations=authzr) auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f4a8a5c8a..a65341692 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -294,6 +294,7 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr + self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From f1b7017c0c632054c61a231f8cd83db092d063fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 16:20:45 -0800 Subject: [PATCH 351/631] Finish dropping Python 2.6 and 3.3 support * Undo letsencrypt-auto changes * Remove ordereddict import * Add Python 3.4 tests to replace 3.3 * Add python_requires * update pipstrap --- .travis.yml | 4 + acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot/plugins/disco.py | 8 +- certbot/util.py | 8 +- letsencrypt-auto-source/letsencrypt-auto | 124 ++++++++++++------ .../letsencrypt-auto.template | 33 +++-- letsencrypt-auto-source/pieces/pipstrap.py | 91 ++++++++----- letshelp-certbot/setup.py | 1 + setup.py | 1 + 22 files changed, 192 insertions(+), 92 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1077d99d9..42b8d679d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ matrix: env: TOXENV=py27-oldest sudo: required services: docker + - python: "3.4" + env: TOXENV=py34 + sudo: required + services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/setup.py b/acme/setup.py index ba5c8e6fb..1f16f3b99 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -45,6 +45,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index d7c223a0a..86b0c646e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 7e1b059e2..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -34,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d619f1872..6db6cc48f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5d14f3e29..bf337c3d0 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index ce8fedd46..12d55f660 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 06af16759..79c93c942 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 7c0f3ed86..5d0970af1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index de881ad84..cdfa205aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -36,6 +36,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0d580b7ee..6c0dfb68f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index c0ba11470..09a4e2cf7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5161e7a94..06efc373d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 09f8a7d52..8bd157166 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 37c477ef6..58f687aea 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..5a7e07ec0 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..b81799373 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index aed15a8ef..85659cfad 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -781,11 +781,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -956,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1220,6 +1237,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1253,33 +1271,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1300,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1315,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1329,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1336,17 +1373,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b3d6ab740..618e8f6bd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -320,11 +320,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -495,10 +504,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..ed55b37e9 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,33 +57,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +103,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +119,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +134,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,17 +159,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 7c8c39068..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', diff --git a/setup.py b/setup.py index e3824a7f7..f314449e6 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', From f3b23662f1fddedcea702c6ea5b54d5f97d1b0a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Feb 2018 20:52:04 -0800 Subject: [PATCH 352/631] Don't error immediately on wildcards. (#5600) --- certbot/tests/display/ops_test.py | 4 ++-- certbot/tests/main_test.py | 7 ++----- certbot/util.py | 10 ---------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 518653a53..c31a3fb33 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -939,10 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, @@ -1232,7 +1229,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..70f402a72 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -552,16 +552,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): From c3659c300b0720d29939331c1c7c86586fd629ae Mon Sep 17 00:00:00 2001 From: Marcus LaFerrera Date: Thu, 22 Feb 2018 13:09:06 -0500 Subject: [PATCH 353/631] Return str rather than bytes (#5585) * Return str rather than bytes Project id is returned as bytes, which causes issues when constructing the google cloud API url, converting `b'PROJECT_ID'` to `b%27PROJECT_ID%27` causing the request to fail. * Ensure we handle both bytes and str types * project_id should be a str or bytes, not int --- certbot-dns-google/certbot_dns_google/dns_google.py | 5 ++++- certbot-dns-google/certbot_dns_google/dns_google_test.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..cea754c06 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -224,4 +224,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..53f84dd6e 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -223,9 +223,13 @@ class GoogleClientTest(unittest.TestCase): response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 From 457269b0052f68b7e1b4a75414175a2e22f27ae6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 10:14:29 -0800 Subject: [PATCH 354/631] Add finalize_order to shim object, update Certbot to use it (#5601) * update order object with returned authorizations * major structure of finalize_order shim refactor * util methods and imports for finalize_order shim refactor * update certbot.tests.client_test.py * extraneous client_test imports * remove correct import * update renewal call * add test for acme.dump_pyopenssl_chain * Add test for certbot.crypto_util.cert_and_chain_from_fullchain * add tests for acme.client and change to fetch chain failure to TimeoutError * s/rytpe/rtype * remove ClientV1 passthrough * dump the wrapped cert * remove dead code * remove the correct dead code * support earlier mock --- acme/acme/client.py | 43 ++++++++++++++-- acme/acme/client_test.py | 71 ++++++++++++++++++++++++- acme/acme/crypto_util.py | 22 ++++++++ acme/acme/crypto_util_test.py | 28 ++++++++++ certbot/client.py | 81 +++++++++-------------------- certbot/crypto_util.py | 25 +++++---- certbot/main.py | 4 +- certbot/renewal.py | 5 +- certbot/tests/client_test.py | 86 +++++++++++-------------------- certbot/tests/crypto_util_test.py | 13 +++++ certbot/tests/util.py | 5 -- 11 files changed, 240 insertions(+), 143 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1838fab42..97f529aae 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -677,9 +677,6 @@ class BackwardsCompatibleClientV2(object): return getattr(self.client, name) elif name in dir(ClientBase): return getattr(self.client, name) - # temporary, for breaking changes into smaller pieces - elif name in dir(Client): - return getattr(self.client, name) else: raise AttributeError() @@ -723,10 +720,48 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return messages.OrderResource(authorizations=authorizations) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) else: return self.client.new_order(csr_pem) + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) + chain = crypto_util.dump_pyopenssl_chain(chain) + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1b33ca5d7..acc5193ca 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -8,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -82,6 +83,29 @@ class ClientTestBase(unittest.TestCase): class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded) + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + def _init(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import BackwardsCompatibleClientV2 @@ -109,8 +133,6 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - # delete this line once we finish migrating to new API: - self.assertEqual(client.register, client.client.register) self.assertEqual(client.update_registration, client.client.update_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') @@ -182,7 +204,52 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.new_order(mock_csr_pem) mock_client().new_order.assert_called_once_with(mock_csr_pem) + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) class ClientTest(ClientTestBase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a986721f0..f13c5109c 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -8,6 +8,8 @@ import socket import sys import OpenSSL +import josepy as jose + from acme import errors @@ -280,3 +282,23 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 14aaac8b5..e8dd3b20c 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -225,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/client.py b/certbot/client.py index dd11f2204..fc3848a5c 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -11,7 +12,6 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util -from acme import errors as acme_errors from acme import messages import certbot @@ -243,8 +243,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM strings :rtype: tuple """ @@ -264,32 +263,9 @@ class Client(object): orderr = orderr.update(authorizations=authzr) authzr = orderr.authorizations - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -298,10 +274,9 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ @@ -329,9 +304,9 @@ class Client(object): os.remove(csr.file) return self.obtain_certificate(successful_domains) else: - certr, chain = self.obtain_certificate_from_csr(csr, orderr) + cert, chain = self.obtain_certificate_from_csr(csr, orderr) - return certr, chain, key, csr + return cert, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -350,7 +325,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -365,19 +340,16 @@ class Client(object): return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -394,8 +366,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -406,20 +376,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 8368855cd..11721cc10 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -367,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -443,3 +433,16 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/main.py b/certbot/main.py index d01f68920..eff4c9c8f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,13 +1064,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/renewal.py b/certbot/renewal.py index 024a815cc..ea5d87a5e 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -294,15 +294,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a65341692..ed9c140e7 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -134,7 +130,10 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] - self.eg_order = mock.MagicMock(authorizations=[None]) + self.eg_order = mock.MagicMock( + authorizations=[None], + fullchain_pem=mock.sentinel.fullchain_pem, + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -143,9 +142,9 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() self.client.auth_handler.handle_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.finalize_order.return_value = self.eg_order self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order def _check_obtain_certificate(self, auth_count=1): if auth_count == 1: @@ -155,27 +154,24 @@ class ClientTest(ClientTestCommon): else: self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.handle_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=orderr)) @@ -184,7 +180,7 @@ class ClientTest(ClientTestCommon): # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=None)) @@ -198,41 +194,13 @@ class ClientTest(ClientTestCommon): test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - test_csr, - orderr=orderr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - test_csr, - orderr=orderr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -240,6 +208,8 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + mock.sentinel.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -248,6 +218,8 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) @@ -255,6 +227,7 @@ class ClientTest(ClientTestCommon): self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -263,6 +236,8 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -272,6 +247,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) def _authzr_from_domains(self, domains): authzr = [] @@ -294,7 +270,6 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr - self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): @@ -302,13 +277,12 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): domains = ["example.com", "www.example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), @@ -324,7 +298,6 @@ class ClientTest(ClientTestCommon): self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -333,9 +306,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -345,7 +317,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..00303fab3 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT + chain_pem = CERT + SS_CERT + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 60d8d6084..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( From 990b211a76efb5a376cc4076bbbc2c2f0a2b3f2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 12:33:55 -0800 Subject: [PATCH 355/631] Remove extra `:returns:` (#5611) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index fc3848a5c..2d7288ce3 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -274,7 +274,7 @@ class Client(object): :param list domains: domains to get a certificate - :returns: :returns: certificate as PEM string, chain as PEM string, + :returns: certificate as PEM string, chain as PEM string, newly generated private key (`.util.Key`), and DER-encoded Certificate Signing Request (`.util.CSR`). :rtype: tuple From 1e46d26ac3511ca92e14c98c823a82286142ebb6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Feb 2018 16:28:50 -0800 Subject: [PATCH 356/631] Fix ACMEv2 issues (#5612) * Add post wrapper to automatically add acme_version * Add uri to authzr. * Only add kid when account is set. * Add content_type when downloading certificate. * Only save new_authz URL when it exists. * Handle combinations in ACMEv1 and ACMEv2. * Add tests for ACMEv2 "combinations". --- acme/acme/client.py | 61 ++++++++++++-------- certbot/account.py | 17 ++++-- certbot/auth_handler.py | 12 +++- certbot/client.py | 4 +- certbot/tests/auth_handler_test.py | 93 ++++++++++++++++++++++++------ 5 files changed, 134 insertions(+), 53 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 97f529aae..9854aae31 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -70,7 +70,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes terms_of_service=terms_of_service) def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body, acme_version=self.acme_version) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -82,6 +82,13 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -143,8 +150,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response, - acme_version=self.acme_version) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -216,12 +222,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], + response = self._post(self.directory[messages.Revocation], messages.Revocation( certificate=cert, reason=rsn), - content_type=None, - acme_version=self.acme_version) + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -271,8 +276,7 @@ class Client(ClientBase): """ new_reg = messages.NewRegistration() if new_reg is None else new_reg - response = self.net.post(self.directory[new_reg], new_reg, - acme_version=1) + response = self._post(self.directory[new_reg], new_reg) # TODO: handle errors assert response.status_code == http_client.CREATED @@ -308,8 +312,7 @@ class Client(ClientBase): if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz, - acme_version=1) + response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) @@ -351,12 +354,11 @@ class Client(ClientBase): req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}, - acme_version=1) + headers={'Accept': content_type}) cert_chain_uri = response.links.get('up', {}).get('url') @@ -552,8 +554,7 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory['newAccount'], new_account, - acme_version=2) + response = self._post(self.directory['newAccount'], new_account) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) @@ -577,11 +578,11 @@ class ClientV2(ClientBase): identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=name)) order = messages.NewOrder(identifiers=identifiers) - response = self.net.post(self.directory['newOrder'], order) + response = self._post(self.directory['newOrder'], order) body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url))) + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) return messages.OrderResource( body=body, uri=response.headers.get('Location'), @@ -643,7 +644,7 @@ class ClientV2(ClientBase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(orderr.body.finalize, wrapped_csr) + self._post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) @@ -651,17 +652,29 @@ class ClientV2(ClientBase): if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self.net.get(body.certificate).text + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but - supports V1 servers. + supports V1 servers. - :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint - :ivar .ClientBase client: either Client or ClientV2 + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 """ def __init__(self, net, key, server): @@ -829,7 +842,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes } if acme_version == 2: kwargs["url"] = url - kwargs["kid"] = self.account["uri"] + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key # pylint: disable=star-args return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 47d806b94..9cc10d4b4 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -24,7 +24,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -100,10 +100,16 @@ class AuthHandler(object): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") for dom in domains: + dom_challenges = self.authzr[dom].body.challenges + if self.acme.acme_version == 1: + combinations = self.authzr[dom].body.combinations + else: + combinations = tuple((i,) for i in range(len(dom_challenges))) + path = gen_challenge_path( - self.authzr[dom].body.challenges, + dom_challenges, self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + combinations) dom_achalls = self._challenge_factory( dom, path) diff --git a/certbot/client.py b/certbot/client.py index 2d7288ce3..0f4fa760d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -212,8 +212,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 3633b673d..394002206 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -81,6 +81,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,13 +91,13 @@ class HandleAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - mock_poll.side_effect = self._validate_all - - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - authzr = self.handler.handle_authorizations(mock_order) + + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -112,8 +113,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -138,17 +146,43 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) + + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][0] + self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - authzr = self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -167,6 +201,13 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( @@ -194,30 +235,44 @@ class HandleAuthorizationsTest(unittest.TestCase): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] From f3a0deba840f1c6bc1510ee71a0e70035fa8488c Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Fri, 23 Feb 2018 15:26:11 -0600 Subject: [PATCH 357/631] Remove min version of setuptools (#5617) --- acme/setup.py | 4 +--- certbot-apache/setup.py | 4 +--- certbot-dns-cloudflare/setup.py | 4 +--- certbot-dns-cloudxns/setup.py | 4 +--- certbot-dns-digitalocean/setup.py | 4 +--- certbot-dns-dnsimple/setup.py | 4 +--- certbot-dns-dnsmadeeasy/setup.py | 4 +--- certbot-dns-google/setup.py | 4 +--- certbot-dns-luadns/setup.py | 4 +--- certbot-dns-nsone/setup.py | 4 +--- certbot-dns-rfc2136/setup.py | 4 +--- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 4 +--- setup.py | 4 +--- 14 files changed, 14 insertions(+), 42 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..51bbf0f71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,9 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..76d7f5ca5 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..e1b84d1be 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..53ceb58ea 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..3330bdd67 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..00a3c032a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..36119ade0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..10fccd1ea 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -15,9 +15,7 @@ install_requires = [ 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..b094e1818 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..e777d821b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..6fc6dca73 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..0fbeab31b 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -10,9 +10,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..a84efe2c3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/setup.py b/setup.py index 47b5b0b2c..736ef467f 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,7 @@ install_requires = [ 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] From 57bdc590dfcbe44da5565e8369997af633debadc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 26 Feb 2018 16:27:38 -0800 Subject: [PATCH 358/631] Add DNS Dockerfiles --- certbot-dns-cloudflare/Dockerfile | 5 +++++ certbot-dns-cloudxns/Dockerfile | 5 +++++ certbot-dns-digitalocean/Dockerfile | 5 +++++ certbot-dns-dnsimple/Dockerfile | 5 +++++ certbot-dns-dnsmadeeasy/Dockerfile | 5 +++++ certbot-dns-google/Dockerfile | 5 +++++ certbot-dns-luadns/Dockerfile | 5 +++++ certbot-dns-nsone/Dockerfile | 5 +++++ certbot-dns-rfc2136/Dockerfile | 5 +++++ certbot-dns-route53/Dockerfile | 5 +++++ 10 files changed, 50 insertions(+) create mode 100644 certbot-dns-cloudflare/Dockerfile create mode 100644 certbot-dns-cloudxns/Dockerfile create mode 100644 certbot-dns-digitalocean/Dockerfile create mode 100644 certbot-dns-dnsimple/Dockerfile create mode 100644 certbot-dns-dnsmadeeasy/Dockerfile create mode 100644 certbot-dns-google/Dockerfile create mode 100644 certbot-dns-luadns/Dockerfile create mode 100644 certbot-dns-nsone/Dockerfile create mode 100644 certbot-dns-rfc2136/Dockerfile create mode 100644 certbot-dns-route53/Dockerfile diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile new file mode 100644 index 000000000..27dcc8751 --- /dev/null +++ b/certbot-dns-cloudflare/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudflare + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile new file mode 100644 index 000000000..cc84ea65b --- /dev/null +++ b/certbot-dns-cloudxns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudxns + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile new file mode 100644 index 000000000..8bdd0619f --- /dev/null +++ b/certbot-dns-digitalocean/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-digitalocean + +RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile new file mode 100644 index 000000000..38d2be80e --- /dev/null +++ b/certbot-dns-dnsimple/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsimple + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile new file mode 100644 index 000000000..ff7936925 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsmadeeasy + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile new file mode 100644 index 000000000..4a258d0ee --- /dev/null +++ b/certbot-dns-google/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-google + +RUN pip install --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile new file mode 100644 index 000000000..6efb4d777 --- /dev/null +++ b/certbot-dns-luadns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-luadns + +RUN pip install --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile new file mode 100644 index 000000000..88fc13c57 --- /dev/null +++ b/certbot-dns-nsone/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-nsone + +RUN pip install --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile new file mode 100644 index 000000000..1b8feb2f8 --- /dev/null +++ b/certbot-dns-rfc2136/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-rfc2136 + +RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile new file mode 100644 index 000000000..a1b8d6caf --- /dev/null +++ b/certbot-dns-route53/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-route53 + +RUN pip install --no-cache-dir --editable src/certbot-dns-route53 From 6f86267a26c9b748bd90113fe62157a9a455cdd2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 12:42:13 -0800 Subject: [PATCH 359/631] Fix revocation in ACMEv2 (#5626) * Allow revoke to pass in a url * Add revocation support to ACMEv2. * Provide regr for account based revocation. * Add revoke wrapper to BackwardsCompat client --- acme/acme/client.py | 52 +++++++++++++++++++++++++++++++++++----- acme/acme/client_test.py | 26 +++++++++++++++++--- certbot/main.py | 4 ++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9854aae31..e3f6e845d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -211,7 +211,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response - def revoke(self, cert, rsn): + def _revoke(self, cert, rsn, url): """Revoke certificate. :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in @@ -219,14 +219,16 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :param int rsn: Reason code for certificate revocation. + :param str url: ACME URL to post to + :raises .ClientError: If revocation is unsuccessful. """ - response = self._post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -528,6 +530,18 @@ class Client(ClientBase): "Recursion limit reached. Didn't get {0}".format(uri)) return chain + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory[messages.Revocation]) class ClientV2(ClientBase): @@ -657,6 +671,19 @@ class ClientV2(ClientBase): return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but @@ -775,6 +802,19 @@ class BackwardsCompatibleClientV2(object): else: return self.client.finalize_order(orderr, deadline) + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acc5193ca..1e4db2884 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,6 +40,7 @@ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', }) @@ -79,6 +80,9 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -251,6 +255,19 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.finalize_order(mock_orderr, mock_deadline) mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -271,9 +288,6 @@ class ClientTest(ClientTestBase): uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 - from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -752,6 +766,12 @@ class ClientV2Test(ClientTestBase): deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, content_type=None, + acme_version=2) + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/certbot/main.py b/certbot/main.py index 33c7730c6..7be852e83 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -982,11 +982,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) From b18696b6a0967ddb609aeb5841b93e6214099080 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 16:47:43 -0800 Subject: [PATCH 360/631] Don't run tests with Python 2.6 (#5627) * Don't run tests with Python 2.6. * Revert "Don't run tests with Python 2.6." This reverts commit 4a9d778cca62ae2bec4cf060726e88f1fd66f374. * Revert changes to auto_test.py. --- letsencrypt-auto-source/tests/auto_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 8c2bfc079..d187452a1 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertTrue('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertNotIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertFalse('Creating virtual environment...' in out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" From a39d2fe55b760707718dcd8225b7dc42dcef4c9c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 18:05:33 -0800 Subject: [PATCH 361/631] Fix wildcard issuance (#5620) * Add is_wildcard_domain to certbot.util. * Error with --allow-subset-of-names and wildcards. * Fix issue preventing wildcard cert issuance. * Kill assumption domain is unique in auth_handler * fix typo and add test * update comments --- certbot/auth_handler.py | 158 ++++++++++++++++------------- certbot/cli.py | 5 + certbot/client.py | 7 +- certbot/tests/auth_handler_test.py | 80 +++++++-------- certbot/tests/cli_test.py | 4 + certbot/tests/client_test.py | 1 + certbot/tests/util_test.py | 20 ++++ certbot/util.py | 18 ++++ 8 files changed, 180 insertions(+), 113 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 9cc10d4b4..2b38e4af5 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -29,10 +34,8 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` + :ivar aauthzrs: ACME Authorization Resources and their active challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,12 +45,9 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() + self.aauthzrs = [] self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. @@ -63,17 +63,15 @@ class AuthHandler(object): authorizations """ - authzrs = orderr.authorizations - for authzr in authzrs: - self.authzr[authzr.body.identifier.value] = authzr - domains = self.authzr.keys() + for authzr in orderr.authorizations: + self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) - self._choose_challenges(domains) + self._choose_challenges() config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: + while self._has_challenges(): resp = self._solve_challenges() logger.info("Waiting for verification...") if config.debug_challenges: @@ -87,8 +85,8 @@ class AuthHandler(object): self.verify_authzr_complete() # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -96,41 +94,54 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: - dom_challenges = self.authzr[dom].body.challenges + for aauthzr in self.aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: - combinations = self.authzr[dom].body.combinations + combinations = aauthzr.authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(dom_challenges))) + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) path = gen_challenge_path( - dom_challenges, - self._get_chall_pref(dom), + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), combinations) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) + + def _has_challenges(self): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in self.aauthzrs) def _solve_challenges(self): """Get Responses for challenges from authenticators.""" resp = [] + all_achalls = self._get_all_achalls() with error_handler.ErrorHandler(self._cleanup_challenges): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp + def _get_all_achalls(self): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in self.aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. @@ -139,69 +150,67 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(resp, chall_update) # Check for updated status... try: self._poll_challenges(chall_update, best_effort) finally: - # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, resps, chall_update): """Send responses and make sure errors are handled. :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(self.aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls def _poll_challenges( self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + self.aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -210,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = self.aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -285,14 +296,17 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls() else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in self.aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break def verify_authzr_complete(self): """Verifies that all authorizations have been decided. @@ -301,15 +315,16 @@ class AuthHandler(object): :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in self.aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -323,8 +338,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls diff --git a/certbot/cli.py b/certbot/cli.py index 09dd71d13..1c2273c8a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -599,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args diff --git a/certbot/client.py b/certbot/client.py index 0f4fa760d..81fc0b802 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -298,7 +298,12 @@ class Client(object): auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] - if successful_domains != domains: + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: if not self.config.dry_run: os.remove(key.file) os.remove(csr.file) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 394002206..7650d2c95 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,32 +29,31 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) class HandleAuthorizationsTest(unittest.TestCase): @@ -103,7 +102,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -134,7 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -160,7 +159,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -190,12 +189,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -274,14 +273,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self._test_preferred_challenges_not_supported_common(combos=False) def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + for i, aauthzr in enumerate(self.handler.aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -290,7 +290,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -298,40 +298,38 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[0], [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( + [messages.STATUS_PENDING] * 2, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.handler.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid self.handler._poll_challenges(self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.handler._poll_challenges(self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -345,7 +343,7 @@ class PollChallengesTest(unittest.TestCase): def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ed9c140e7..b51275d9e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -222,6 +222,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index f47c5da9c..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -601,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. From e9bc4a319b9989a300dd574466a15edd581ee3c4 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 28 Feb 2018 21:31:47 +0200 Subject: [PATCH 362/631] Apache plugin wildcard support for ACMEv2 (#5608) In `deploy_cert()` and `enhance()`, the user will be presented with a dialog to choose from the VirtualHosts that can be covered by the wildcard domain name. The (multiple) selection result will then be handled in a similar way that we previously handled a single VirtualHost that was returned by the `_find_best_vhost()`. Additionally the selected VirtualHosts are added to a dictionary that maps selections to a wildcard domain to be reused in the later `enhance()` call and not forcing the user to select the same VirtualHosts again. * Apache plugin wildcard support * Present dialog only once per domain, added tests * Raise exception if no VHosts selected for wildcard domain --- certbot-apache/certbot_apache/configurator.py | 141 +++++++++++++++++- certbot-apache/certbot_apache/display_ops.py | 48 +++++- certbot-apache/certbot_apache/obj.py | 13 ++ .../certbot_apache/tests/configurator_test.py | 100 +++++++++++++ .../certbot_apache/tests/display_ops_test.py | 30 ++++ certbot/tests/main_test.py | 1 - 6 files changed, 316 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 4bb2cbebd..6377bb114 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -152,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -262,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -280,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -311,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -327,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -391,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -1376,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 8f34d33d3..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1337,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index c31a3fb33..b778f05ea 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -940,7 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, From 78735fa2c3ebaee6c7aba02bbf939597a9075cbb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 28 Feb 2018 16:08:06 -0800 Subject: [PATCH 363/631] Suggest DNS authenticator when it's needed (#5638) --- certbot/auth_handler.py | 16 ++++++++++++---- certbot/tests/auth_handler_test.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 2b38e4af5..67d36c8cc 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -433,7 +433,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -454,15 +454,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7650d2c95..b6af3d0f5 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -272,6 +272,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_net.acme_version = 2 self._test_preferred_challenges_not_supported_common(combos=False) + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def _validate_all(self, unused_1, unused_2): for i, aauthzr in enumerate(self.handler.aauthzrs): azr = aauthzr.authzr From 38d5144fff79e10507ee7d53cb6664c8180d2245 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 08:25:32 -0800 Subject: [PATCH 364/631] Drop min coverage to 63 (#5641) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 24d224cb0..ea412b6b9 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -437,4 +437,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 63 -m From 559220c2eff90975fb671945d780c2a757d0e167 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 10:11:15 -0800 Subject: [PATCH 365/631] Add basic ACMEv2 integration tests (#5635) * Use newer boulder config * Use ACMEv2 endpoint if requested * Add v2 integration tests * Work with unset variables * Add wildcard issuance test * quote domains --- .travis.yml | 6 +++++- tests/boulder-fetch.sh | 7 +++++++ tests/boulder-integration.sh | 6 ++++++ tests/integration/_common.sh | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42b8d679d..c62664180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + sudo: required + services: docker + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ea412b6b9..f2b0dcf60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -430,6 +430,12 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..236090a14 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,6 +16,11 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*," From f0b337532cdc232add47c7bf98401eb7d75ca615 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 14:05:50 -0800 Subject: [PATCH 366/631] Nginx plugin wildcard support for ACMEv2 (#5619) * support wildcards for deploy_cert * support wildcards for enhance * redirect enhance and some tests * update tests * add display_ops and display_repr * update display_ops_test and errors found * say server block * match redirects properly * functional code * start adding tests and lint errors * add configurator tests * lint * change message to be generic to installation and enhancement * remove _wildcard_domain * take selecting vhosts out of loop * remove extra newline * filter wildcard vhosts by port * lint * don't filter by domain * [^.]+ * lint * make vhost hashable * one more tuple --- certbot-nginx/certbot_nginx/configurator.py | 199 ++++++++++++++---- certbot-nginx/certbot_nginx/display_ops.py | 44 ++++ certbot-nginx/certbot_nginx/http_01.py | 6 +- certbot-nginx/certbot_nginx/obj.py | 17 ++ .../certbot_nginx/tests/configurator_test.py | 100 ++++++++- .../certbot_nginx/tests/display_ops_test.py | 45 ++++ .../certbot_nginx/tests/tls_sni_01_test.py | 4 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 7 +- 8 files changed, 370 insertions(+), 52 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/display_ops.py create mode 100644 certbot-nginx/certbot_nginx/tests/display_ops_test.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 9f091c0fd..e4d87744e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,6 +23,7 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants +from certbot_nginx import display_ops from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 @@ -92,6 +93,11 @@ class NginxConfigurator(common.Installer): # For creating new vhosts if no names match self.new_vhost = None + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = {} + self._wildcard_redirect_vhosts = {} + # Add number of outstanding challenges self._chall_out = 0 @@ -146,6 +152,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError( 'Unable to lock %s', self.conf('server-root')) + # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): @@ -166,14 +173,24 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, create_if_no_match=True) + vhosts = self.choose_vhosts(domain, create_if_no_match=True) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + # pylint: disable=unused-argument + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) - logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -181,10 +198,61 @@ class NginxConfigurator(common.Installer): self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path + def _choose_vhosts_wildcard(self, domain, prefer_ssl, no_ssl_filter_port=None): + """Prompts user to choose vhosts to install a wildcard certificate for""" + if prefer_ssl: + vhosts_cache = self._wildcard_vhosts + preference_test = lambda x: x.ssl + else: + vhosts_cache = self._wildcard_redirect_vhosts + preference_test = lambda x: not x.ssl + + # Caching! + if domain in vhosts_cache: + # Vhosts for a wildcard domain were already selected + return vhosts_cache[domain] + + # Get all vhosts whether or not they are covered by the wildcard domain + vhosts = self.parser.get_vhosts() + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL or non-SSL vhosts + filtered_vhosts = {} + for vhost in vhosts: + # Ensure we're listening non-sslishly on no_ssl_filter_port + if no_ssl_filter_port is not None: + if not self._vhost_listening_on_port_no_ssl(vhost, no_ssl_filter_port): + continue + for name in vhost.names: + if preference_test(vhost): + # Prefer either SSL or non-SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + return_vhosts = display_ops.select_vhost_multiple(list(dialog_input)) + + for vhost in return_vhosts: + if domain not in vhosts_cache: + vhosts_cache[domain] = [] + vhosts_cache[domain].append(vhost) + + return return_vhosts + ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, create_if_no_match=False): + def _choose_vhost_single(self, target_name): + matches = self._get_ranked_matches(target_name) + vhosts = [x for x in [self._select_best_name_match(matches)] if x is not None] + return vhosts + + def choose_vhosts(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -202,17 +270,19 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: ssl vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: ssl vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - vhost = None - - matches = self._get_ranked_matches(target_name) - vhost = self._select_best_name_match(matches) - if not vhost: + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to support. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=True) + else: + vhosts = self._choose_vhost_single(target_name) + if not vhosts: if create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + # result will not be [None] because it errors on failure + vhosts = [self._vhost_from_duplicated_default(target_name)] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -222,10 +292,11 @@ class NginxConfigurator(common.Installer): "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + for vhost in vhosts: + if not vhost.ssl: + self._make_server_ssl(vhost) - return vhost + return vhosts def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) @@ -359,7 +430,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -377,15 +448,20 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - matches = self._get_redirect_ranked_matches(target_name, port) - vhost = self._select_best_name_match(matches) - if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name, port=port) - return vhost + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to enhance. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=False, + no_ssl_filter_port=port) + else: + matches = self._get_redirect_ranked_matches(target_name, port) + vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] + if not vhosts and create_if_no_match: + vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + return vhosts def _port_matches(self, test_port, matching_port): # test_port is a number, matching is a number or "" or None @@ -395,6 +471,23 @@ class NginxConfigurator(common.Installer): else: return test_port == matching_port + def _vhost_listening_on_port_no_ssl(self, vhost, port): + found_matching_port = False + if len(vhost.addrs) == 0: + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + found_matching_port = (port == self.DEFAULT_LISTEN_PORT) + else: + for addr in vhost.addrs: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: + found_matching_port = True + + if found_matching_port: + # make sure we don't have an 'ssl on' directive + return not self.parser.has_ssl_on_directive(vhost) + else: + return False + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -411,21 +504,7 @@ class NginxConfigurator(common.Installer): all_vhosts = self.parser.get_vhosts() def _vhost_matches(vhost, port): - found_matching_port = False - if len(vhost.addrs) == 0: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and addr.ssl == False: - found_matching_port = True - - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - else: - return False + return self._vhost_listening_on_port_no_ssl(vhost, port) matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] @@ -587,17 +666,31 @@ class NginxConfigurator(common.Installer): """ port = self.DEFAULT_LISTEN_PORT - vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. - vhost = self.choose_redirect_vhost(domain, port) + vhosts = self.choose_redirect_vhosts(domain, port) - if vhost is None: + if not vhosts: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) return + for vhost in vhosts: + self._enable_redirect_single(domain, vhost) + + def _enable_redirect_single(self, domain, vhost): + """Redirect all equivalent HTTP traffic to ssl_vhost. + + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. + + .. note:: This function saves the configuration + + :param str domain: domain to enable redirect for + :param `~obj.Vhost` vhost: vhost to enable redirect for + """ + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, @@ -638,7 +731,18 @@ class NginxConfigurator(common.Installer): :type chain_path: `str` or `None` """ - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._enable_ocsp_stapling_single(vhost, chain_path) + + def _enable_ocsp_stapling_single(self, vhost, chain_path): + """Include OCSP response in TLS handshake + + :param str vhost: vhost to enable OCSP response for + :param chain_path: chain file path + :type chain_path: `str` or `None` + + """ if self.version < (1, 3, 7): raise errors.PluginError("Version 1.3.7 or greater of nginx " "is needed to enable OCSP stapling") @@ -889,14 +993,23 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] + def _redirect_block_for_domain(domain): + updated_domain = domain + match_symbol = '=' + if util.is_wildcard_domain(domain): + match_symbol = '~' + updated_domain = updated_domain.replace('.', r'\.') + updated_domain = updated_domain.replace('*', '[^.]+') + updated_domain = '^' + updated_domain + '$' redirect_block = [[ - ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + ['\n ', 'if', ' ', '($host', ' ', match_symbol, ' ', '%s)' % updated_domain, ' '], [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], '\n ']], ['\n']] return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/display_ops.py b/certbot-nginx/certbot_nginx/display_ops.py new file mode 100644 index 000000000..5d6bda6b0 --- /dev/null +++ b/certbot-nginx/certbot_nginx/display_ops.py @@ -0,0 +1,44 @@ +"""Contains UI methods for Nginx operations.""" +import logging + +import zope.component + +from certbot import interfaces + +import certbot.display.util as display_util + + +logger = logging.getLogger(__name__) + + +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + :param vhosts: Available Nginx VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which server blocks would you like to modify?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c0dec061a..0b1b2bfe0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -179,13 +179,17 @@ class NginxHttp01(common.ChallengePerformer): """ try: - vhost = self.configurator.choose_redirect_vhost(achall.domain, + vhosts = self.configurator.choose_redirect_vhosts(achall.domain, '%i' % self.configurator.config.http01_port, create_if_no_match=True) except errors.MisconfigurationError: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block return self._make_server_block(achall) + # len is max 1 because Nginx doesn't authenticate wildcards + # if len were or vhosts None, we would have errored + vhost = vhosts[0] + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index e8dc8936d..3625a95b9 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,6 +193,11 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False + def __hash__(self): + return hash((self.filep, tuple(self.path), + tuple(self.addrs), tuple(self.names), + self.ssl, self.enabled)) + def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -216,3 +221,15 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True + + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.names), + https="Yes" if self.ssl else "No")) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index acb7ee282..722ba68bf 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -128,7 +128,7 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhost(self): + def test_choose_vhosts(self): localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) @@ -159,7 +159,7 @@ class NginxConfiguratorTest(util.NginxTest): '69.255.225.155'] for name in results: - vhost = self.config.choose_vhost(name) + vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) self.assertEqual(results[name], vhost.names) @@ -173,7 +173,7 @@ class NginxConfiguratorTest(util.NginxTest): for name in bad_results: self.assertRaises(errors.MisconfigurationError, - self.config.choose_vhost, name) + self.config.choose_vhosts, name) def test_ipv6only(self): # ipv6_info: (ipv6_active, ipv6only_present) @@ -702,6 +702,100 @@ class NginxConfiguratorTest(util.NginxTest): self.config.rollback_checkpoints() self.assertTrue(mock_parser_load.call_count == 3) + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=True) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_choose_vhosts_wildcard_redirect(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + mock_choose_vhosts.return_value = [vhost] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_nginx.configurator.NginxConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.com", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(vhost, mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_ocsp_after_install(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + self.config._wildcard_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertFalse(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_redirect_or_ocsp_no_install(self, mock_dialog): + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_dialog.return_value = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertTrue(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_double_redirect(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + self.config._wildcard_redirect_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "redirect") + self.assertFalse(mock_dialog.called) + + def test_choose_vhosts_wildcard_no_ssl_filter_port(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [] + self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False, + no_ssl_filter_port='80') + # Check that the dialog was called with only port 80 vhosts + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + + class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/display_ops_test.py b/certbot-nginx/certbot_nginx/tests/display_ops_test.py new file mode 100644 index 000000000..e3c6fb66b --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/display_ops_test.py @@ -0,0 +1,45 @@ +"""Test certbot_apache.display_ops.""" +import unittest + +from certbot.display import util as display_util + +from certbot.tests import util as certbot_util + +from certbot_nginx import parser + +from certbot_nginx.display_ops import select_vhost_multiple +from certbot_nginx.tests import util + + +class SelectVhostMultiTest(util.NginxTest): + """Tests for certbot_nginx.display_ops.select_vhost_multiple.""" + + def setUp(self): + super(SelectVhostMultiTest, self).setUp() + nparser = parser.NginxParser(self.config_path) + self.vhosts = nparser.get_vhosts() + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 61ee293fa..72b65911c 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -61,10 +61,10 @@ class TlsSniPerformTest(util.NginxTest): shutil.rmtree(self.work_dir) @mock.patch("certbot_nginx.configurator" - ".NginxConfigurator.choose_vhost") + ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): self.sni.add_chall(self.achalls[1]) - mock_choose.return_value = None + mock_choose.return_value = [] result = self.sni.perform() self.assertFalse(result is None) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index eca198bfe..0fd37e0cb 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,10 +55,11 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) + vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) - if vhost is not None and vhost.addrs: - addresses.append(list(vhost.addrs)) + # len is max 1 because Nginx doesn't authenticate wildcards + if vhosts and vhosts[0].addrs: + addresses.append(list(vhosts[0].addrs)) else: if ipv6: # If IPv6 is active in Nginx configuration From 8121acf2c1fed5514fd0a31a62f19dfbb92b2bb0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:54:48 -0800 Subject: [PATCH 367/631] Add user friendly wildcard error for ACMEv1 (#5636) * add WildcardUnsupportedError * Add friendly unsupported wildcard error msg * correct documentation * add version specifier --- acme/acme/client.py | 14 ++++++++++++++ acme/acme/client_test.py | 7 +++++++ acme/acme/errors.py | 3 +++ certbot/client.py | 30 +++++++++++++++++++++++------- certbot/tests/client_test.py | 2 +- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e3f6e845d..c6e897692 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -310,9 +310,17 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") + + if identifier.value.startswith("*"): + raise errors.WildcardUnsupportedError( + "Requesting an authorization for a wildcard name is" + " forbidden by this version of the ACME protocol.") + new_authz = messages.NewAuthorization(identifier=identifier) response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors @@ -333,6 +341,8 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ return self.request_challenges(messages.Identifier( typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) @@ -752,6 +762,10 @@ class BackwardsCompatibleClientV2(object): :returns: The newly created order. :rtype: OrderResource + + :raises errors.WildcardUnsupportedError: if a wildcard domain is + requested but unsupported by the ACME version + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1e4db2884..060338360 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -376,6 +376,13 @@ class ClientTest(ClientTestBase): errors.UnexpectedUpdate, self.client.request_challenges, self.identifier) + def test_request_challenges_wildcard(self): + wildcard_identifier = messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='*.example.org') + self.assertRaises( + errors.WildcardUnsupportedError, self.client.request_challenges, + wildcard_identifier) + def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() self.assertEqual( diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 991335958..97fa73614 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -115,3 +115,6 @@ class ConflictError(ClientError): self.location = location super(ConflictError, self).__init__() + +class WildcardUnsupportedError(Error): + """Error for when a wildcard is requested but is unsupported by ACME CA.""" diff --git a/certbot/client.py b/certbot/client.py index 81fc0b802..50d2262c4 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -12,6 +12,7 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util +from acme import errors as acme_errors from acme import messages import certbot @@ -258,10 +259,7 @@ class Client(object): logger.debug("CSR: %s", csr) if orderr is None: - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) - orderr = orderr.update(authorizations=authzr) - authzr = orderr.authorizations + orderr = self._get_order_and_authorizations(csr.data, best_effort=False) deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) @@ -292,9 +290,8 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) - orderr = orderr.update(authorizations=authzr) + orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) + authzr = orderr.authorizations auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] @@ -313,6 +310,25 @@ class Client(object): return cert, chain, key, csr + def _get_order_and_authorizations(self, csr_pem, best_effort): + """Request a new order and complete its authorizations. + + :param str csr_pem: A CSR in PEM format. + :param bool best_effort: True if failing to complete all + authorizations should not raise an exception + + :returns: order resource containing its completed authorizations + :rtype: acme.messages.OrderResource + + """ + try: + orderr = self.acme.new_order(csr_pem) + except acme_errors.WildcardUnsupportedError: + raise errors.Error("The currently selected ACME CA endpoint does" + " not support issuing wildcard certificates.") + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + return orderr.update(authorizations=authzr) + # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): """Obtain and enroll certificate. diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b51275d9e..34595e463 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -184,7 +184,7 @@ class ClientTest(ClientTestCommon): self.client.obtain_certificate_from_csr( test_csr, orderr=None)) - auth_handler.handle_authorizations.assert_called_with(self.eg_order) + auth_handler.handle_authorizations.assert_called_with(self.eg_order, False) # Test for no auth_handler self.client.auth_handler = None From d8a54dc444a842e02c919b5092ebd745f25339e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:55:45 -0800 Subject: [PATCH 368/631] Remove leading *. from default cert name. (#5639) --- certbot/client.py | 9 ++++++++- certbot/tests/client_test.py | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 50d2262c4..eddf93e4f 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -354,7 +354,14 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - new_name = certname if certname else domains[0] + if certname: + new_name = certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + new_name = domains[0][2:] + else: + new_name = domains[0] + if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", new_name) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 34595e463..5d01b103a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -285,7 +285,7 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): - domains = ["example.com", "www.example.com"] + domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), None) @@ -293,12 +293,14 @@ class ClientTest(ClientTestCommon): self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) self.assertTrue(self.client.obtain_and_enroll_certificate(domains, None)) + self.assertTrue(self.client.obtain_and_enroll_certificate(domains[1:], None)) self.client.config.dry_run = True self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) - self.assertTrue(mock_storage.call_count == 2) + names = [call[0][0] for call in mock_storage.call_args_list] + self.assertEqual(names, ["example_cert", "example.com", "example.com"]) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): From 8bc9cd67f0e6b445ee38342c904b8622c7f98878 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 15:08:53 -0800 Subject: [PATCH 369/631] Fix ipv6only detection (#5648) * Fix ipv6only detection * move str() to inside ipv6_info * add regression test * Update to choose_vhosts --- certbot-nginx/certbot_nginx/configurator.py | 3 +++ .../certbot_nginx/tests/configurator_test.py | 12 ++++++++++++ .../testdata/etc_nginx/sites-enabled/ipv6ssl.com | 2 ++ 3 files changed, 17 insertions(+) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e4d87744e..83e308bac 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -311,6 +311,9 @@ class NginxConfigurator(common.Installer): configuration, and existence of ipv6only directive for specified port :rtype: tuple of type (bool, bool) """ + # port should be a string, but it's easy to mess up, so let's + # make sure it is one + port = str(port) vhosts = self.parser.get_vhosts() ipv6_active = False ipv6only_present = False diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 722ba68bf..bffaef5e4 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -181,6 +181,18 @@ class NginxConfiguratorTest(util.NginxTest): # Port 443 has ipv6only=on because of ipv6ssl.com vhost self.assertEquals((True, True), self.config.ipv6_info("443")) + def test_ipv6only_detection(self): + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "ipv6.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + for addr in self.config.choose_vhosts("ipv6.com")[0].addrs: + self.assertFalse(addr.ipv6only) def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com index d8f7eff12..875a9ee1b 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -1,5 +1,7 @@ server { listen 443 ssl; listen [::]:443 ssl ipv6only=on; + listen 5001 ssl; + listen [::]:5001 ssl ipv6only=on; server_name ipv6ssl.com; } From e1878593d5608f908eebb262fe2c9c7dfcab55a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 07:27:44 -0800 Subject: [PATCH 370/631] Ensure fullchain_pem in the order is unicode/str (#5654) * Decode fullchain_pem in ACMEv1 * Convert back to bytes in Certbot * document bytes are returned --- acme/acme/client.py | 4 ++-- acme/acme/client_test.py | 4 ++-- acme/acme/crypto_util.py | 3 +++ certbot/client.py | 5 +++-- certbot/crypto_util.py | 3 ++- certbot/tests/client_test.py | 22 ++++++++++++---------- certbot/tests/crypto_util_test.py | 4 ++-- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c6e897692..d52c82a5c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -809,8 +809,8 @@ class BackwardsCompatibleClientV2(object): 'certificate, please rerun the command for a new one.') cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) - chain = crypto_util.dump_pyopenssl_chain(chain) + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode() + chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) else: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 060338360..a0c27e74f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -99,10 +99,10 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): self.chain = [wrapped, wrapped] self.cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode() single_chain = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, loaded) + OpenSSL.crypto.FILETYPE_PEM, loaded).decode() self.chain_pem = single_chain + single_chain self.fullchain_pem = self.cert_pem + self.chain_pem diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 07b55ae33..2281196eb 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -287,6 +287,9 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). + :returns: certificate chain bundle + :rtype: bytes + """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... diff --git a/certbot/client.py b/certbot/client.py index eddf93e4f..2992c0cec 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -244,7 +244,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: certificate and chain as PEM strings + :returns: certificate and chain as PEM byte strings :rtype: tuple """ @@ -263,7 +263,8 @@ class Client(object): deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) - return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + return cert.encode(), chain.encode() def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 11721cc10..37118c591 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -441,8 +441,9 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + """ cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):] return (cert, chain) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 5d01b103a..0f2c58161 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -132,7 +132,6 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] self.eg_order = mock.MagicMock( authorizations=[None], - fullchain_pem=mock.sentinel.fullchain_pem, csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): @@ -165,8 +164,7 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) @@ -199,8 +197,7 @@ class ClientTest(ClientTestCommon): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -209,7 +206,7 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( - mock.sentinel.fullchain_pem) + self.eg_order.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -218,8 +215,7 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) authzr = self._authzr_from_domains(["example.com"]) self.config.allow_subset_of_names = True @@ -237,8 +233,7 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) - mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -250,6 +245,13 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_csr.assert_not_called() self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) + def _set_mock_from_fullchain(self, mock_from_fullchain): + mock_cert = mock.Mock() + mock_cert.encode.return_value = mock.sentinel.cert + mock_chain = mock.Mock() + mock_chain.encode.return_value = mock.sentinel.chain + mock_from_fullchain.return_value = (mock_cert, mock_chain) + def _authzr_from_domains(self, domains): authzr = [] diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 00303fab3..480139378 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -377,8 +377,8 @@ class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" def test_cert_and_chain_from_fullchain(self): - cert_pem = CERT - chain_pem = CERT + SS_CERT + cert_pem = CERT.decode() + chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem from certbot.crypto_util import cert_and_chain_from_fullchain cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) From cc344bfd1e080ad8ae253e0b1073a5ba7879583d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 09:50:19 -0800 Subject: [PATCH 371/631] Break lockstep between our packages (#5655) Fixes #5490. There's a lot of possibilities discussed in #5490, but I'll try and explain what I actually did here as succinctly as I can. Unfortunately, there's a fair bit to explain. My goal was to break lockstep and give us tests to ensure the minimum specified versions are correct without taking the time now to refactor our whole test setup. To handle specifying each package's minimum acme/certbot version, I added a requirements file to each package. This won't actually be included in the shipped package (because it's not in the MANIFEST). After creating these files and modifying tools/pip_install.sh to use them, I created a separate tox env for most packages (I kept the DNS plugins together for convenience). The reason this is necessary is because we currently use a single environment for each plugin, but if we used this approach for these tests we'd hit issues due to different installed plugins requiring different versions of acme/certbot. There's a lot more discussion about this in #5490 if you're interested in this piece. I unfortunately wasted a lot of time trying to remove the boilerplate this approach causes in tox.ini, but to do this I think we need negations described at complex factor conditions which hasn't made it into a tox release yet. The biggest missing piece here is how to make sure the oldest versions that are currently pinned to master get updated. Currently, they'll stay pinned that way without manual intervention and won't be properly testing the oldest version. I think we should solve this during the larger test/repo refactoring after the release because the tests are using the correct values now and I don't see a simple way around the problem. Once this lands, I'm planning on updating the test-everything tests to do integration tests with the "oldest" versions here. * break lockstep between packages * Use per package requirements files * add local oldest requirements files * update tox.ini * work with dev0 versions * Install requirements in separate step. * don't error when we don't have requirements * install latest packages in editable mode * Update .travis.yml * Add reminder comments * move dev to requirements * request acme[dev] * Update pip_install documentation --- .travis.yml | 2 +- certbot-apache/local-oldest-requirements.txt | 2 + certbot-apache/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudflare/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudxns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-digitalocean/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsimple/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsmadeeasy/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-google/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-luadns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-nsone/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-rfc2136/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-route53/setup.py | 6 ++- certbot-nginx/local-oldest-requirements.txt | 2 + certbot-nginx/setup.py | 10 ++-- local-oldest-requirements.txt | 1 + setup.py | 4 +- tools/pip_install.sh | 34 +++++++++--- tox.ini | 53 +++++++++++++++++-- 29 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 certbot-apache/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudflare/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudxns/local-oldest-requirements.txt create mode 100644 certbot-dns-digitalocean/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsimple/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/local-oldest-requirements.txt create mode 100644 certbot-dns-google/local-oldest-requirements.txt create mode 100644 certbot-dns-luadns/local-oldest-requirements.txt create mode 100644 certbot-dns-nsone/local-oldest-requirements.txt create mode 100644 certbot-dns-rfc2136/local-oldest-requirements.txt create mode 100644 certbot-dns-route53/local-oldest-requirements.txt create mode 100644 certbot-nginx/local-oldest-requirements.txt create mode 100644 local-oldest-requirements.txt diff --git a/.travis.yml b/.travis.yml index c62664180..9ec2f724b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - python: "2.7" env: TOXENV=lint - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required services: docker - python: "3.4" diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-apache/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 336233bd4..7608c0647 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-dns-cloudflare/local-oldest-requirements.txt b/certbot-dns-cloudflare/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudflare/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index e5687a9f5..4ed8e796d 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'cloudflare>=1.5.1', 'mock', 'setuptools', diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 0ef31a90c..7f973709c 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-digitalocean/local-oldest-requirements.txt b/certbot-dns-digitalocean/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-digitalocean/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 11c2aea24..0ce91e64e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-digitalocean>=1.11', 'setuptools', diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 414a058fa..d12b26d83 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 18d773347..856eaba0f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-google/local-oldest-requirements.txt b/certbot-dns-google/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-google/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d5def1bf9..0dfff0402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', # 1.5 is the first version that supports oauth2client>=2.0 'google-api-python-client>=1.5', 'mock', diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 13fa742d5..b255691dc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 01c9579c1..68d8f6cdb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-rfc2136/local-oldest-requirements.txt b/certbot-dns-rfc2136/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-rfc2136/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 64b126595..3d6b3799b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dnspython', 'mock', 'setuptools', diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e45343f79..ad20725b5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -5,9 +5,11 @@ from setuptools import find_packages version = '0.22.0.dev0' +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'boto3', 'mock', 'setuptools', diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-nginx/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 96f8b834d..bb71cf19a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,10 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + # This plugin works with an older version of acme, but Certbot does not. + # 0.22.0 is specified here to work around + # https://github.com/pypa/pip/issues/988. + 'acme>0.21.1', + 'certbot>0.21.1', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt new file mode 100644 index 000000000..2346300a3 --- /dev/null +++ b/local-oldest-requirements.txt @@ -0,0 +1 @@ +-e acme[dev] diff --git a/setup.py b/setup.py index 9ac1a7ee7..3667a6976 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,9 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme=={0}'.format(version), + # Remember to update local-oldest-requirements.txt when changing the + # minimum acme version. + 'acme>0.21.1', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43..b385c5482 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,18 +1,30 @@ #!/bin/bash -e # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. # get the root of the Certbot repo tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" +all_constraints=$(mktemp) test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT +trap "rm -f $all_constraints $test_constraints" EXIT if [ "$CERTBOT_OLDEST" = 1 ]; then + if [ "$1" != "-e" -o "$#" -ne "2" ]; then + echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." + exit 1 + fi + pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] + requirements="$pkg_dir/local-oldest-requirements.txt" + # packages like acme don't have any local oldest requirements + if [ ! -f "$requirements" ]; then + unset requirements + fi cp "$tools_dir/oldest_constraints.txt" "$test_constraints" else repo_root=$(dirname "$tools_dir") @@ -20,7 +32,13 @@ else sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" fi +"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ + "$test_constraints" > "$all_constraints" + set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +if [ -n "$requirements" ]; then + pip install -q --constraint "$all_constraints" --requirement "$requirements" +fi +pip install -q --constraint "$all_constraints" "$@" diff --git a/tox.ini b/tox.ini index 971aa7631..049220bbb 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -all_packages = - acme[dev] \ - .[dev] \ - certbot-apache \ +dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ certbot-dns-digitalocean \ @@ -27,7 +24,12 @@ all_packages = certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ - certbot-dns-route53 \ + certbot-dns-route53 +all_packages = + acme[dev] \ + .[dev] \ + certbot-apache \ + {[base]dns_packages} \ certbot-nginx \ letshelp-certbot install_packages = @@ -70,6 +72,47 @@ setenv = passenv = {[testenv]passenv} +[testenv:py27-acme-oldest] +commands = + {[base]install_and_test} acme[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-apache-oldest] +commands = + {[base]install_and_test} certbot-apache +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-certbot-oldest] +commands = + {[base]install_and_test} .[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-dns-oldest] +commands = + {[base]install_and_test} {[base]dns_packages} +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-nginx-oldest] +commands = + {[base]install_and_test} certbot-nginx + python tests/lock_test.py +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + [testenv:py27_install] basepython = python2.7 commands = From 441625c6102126f2d63daa964ecac4073e583d0a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 5 Mar 2018 22:49:02 +0200 Subject: [PATCH 372/631] Allow Google DNS plugin to write multiple TXT record values (#5652) * Allow Google DNS plugin to write multiple TXT record values in same resourcerecord * Atomic updates * Split rrsets request --- .../certbot_dns_google/dns_google.py | 63 ++++++++++++++++++- .../certbot_dns_google/dns_google_test.py | 61 +++++++++++++++++- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index cea754c06..ab8bf20de 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -107,6 +107,15 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + add_records = record_contents[:] + + if "\""+record_content+"\"" in record_contents: + # The process was interrupted previously and validation token exists + return + + add_records.append(record_content) + data = { "kind": "dns#change", "additions": [ @@ -114,12 +123,24 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": add_records, "ttl": record_ttl, }, ], } + if record_contents: + # We need to remove old records in the same request + data["deletions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": record_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -154,6 +175,8 @@ class _GoogleClient(object): logger.warn('Error finding zone. Skipping cleanup.') return + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + data = { "kind": "dns#change", "deletions": [ @@ -161,12 +184,26 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": record_contents, "ttl": record_ttl, }, ], } + # Remove the record being deleted from the list + readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + if readd_contents: + # We need to remove old records in the same request + data["additions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": readd_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -175,6 +212,28 @@ class _GoogleClient(object): except googleapiclient_errors.Error as e: logger.warn('Encountered error deleting TXT record: %s', e) + def get_existing_txt_rrset(self, zone_id, record_name): + """ + Get existing TXT records from the RRset for the record name. + + :param str zone_id: The ID of the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + + :returns: List of TXT record values + :rtype: `list` of `string` + + """ + rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member + request = rrs_request.list(managedZone=zone_id, project=self.project_id) + response = request.execute() + # Add dot as the API returns absolute domains + record_name += "." + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return [] + def _find_managed_zone_id(self, domain): """ Find the managed zone for a given domain. diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 53f84dd6e..3291b2c3a 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -74,10 +74,15 @@ class GoogleClientTest(unittest.TestCase): mock_mz = mock.MagicMock() mock_mz.list.return_value.execute.side_effect = zone_request_side_effect + mock_rrs = mock.MagicMock() + rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""]}]} + mock_rrs.list.return_value.execute.return_value = rrsets mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) client.dns.changes = mock.MagicMock(return_value=mock_changes) + client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs) return client, mock_changes @@ -137,6 +142,30 @@ class GoogleClientTest(unittest.TestCase): managedZone=self.zone, project=PROJECT_ID) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["sample-txt-contents"] + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + self.assertTrue("sample-txt-contents" in + changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_noop(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + client.add_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + self.assertFalse(changes.create.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -172,7 +201,12 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["\"sample-txt-contents\"", + "\"example-txt-contents\""] + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) expected_body = { "kind": "dns#change", @@ -180,8 +214,17 @@ class GoogleClientTest(unittest.TestCase): { "kind": "dns#resourceRecordSet", "type": "TXT", - "name": self.record_name + ".", - "rrdatas": [self.record_content, ], + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", ], "ttl": self.record_ttl, }, ], @@ -217,6 +260,18 @@ class GoogleClientTest(unittest.TestCase): client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEquals(found, ["\"example-txt-contents\""]) + not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") + self.assertEquals(not_found, []) + def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient From fe682e779b82ab0dfd72342369df630495c26a20 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 4 Mar 2018 16:24:42 +0200 Subject: [PATCH 373/631] ACMEv2 support for Route53 plugin --- .../certbot_dns_route53/dns_route53.py | 26 +++++++-- .../certbot_dns_route53/dns_route53_test.py | 54 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 67462e369..c0e8e5495 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -85,9 +85,29 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] + def _get_validation_rrset(self, zone_id, validation_domain_name): + validation_domain_name += "." + records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) + for record in records["ResourceRecordSets"]: + if record["Name"] == validation_domain_name and record["Type"] == "TXT": + return record["ResourceRecords"] + return [] + def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) + rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + challenge = {"Value": '"{0}"'.format(validation)} + if action == "DELETE": + if len(rrecords) > 1: + # Need to update instead, as we're not deleting the rrset + action = "UPSERT" + # Remove the record being deleted from the list + rrecords = [rr for rr in rrecords if rr != challenge] + else: + if challenge not in rrecords: + rrecords.append(challenge) + response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ @@ -99,11 +119,7 @@ class Authenticator(dns_common.DNSAuthenticator): "Name": validation_domain_name, "Type": "TXT", "TTL": self.ttl, - "ResourceRecords": [ - # For some reason TXT records need to be - # manually quoted. - {"Value": '"{0}"'.format(validation)} - ], + "ResourceRecords": rrecords, } } ] diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index d5f1b2816..9aec05b6e 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,6 +178,9 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock( + return_value=[] + ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -186,6 +189,57 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_multirecord(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock() + self.client._get_validation_rrset.return_value = [ + {"Value": "\"pre-existing-value\""}, + {"Value": "\"pre-existing-value-two\""}, + ] + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + self.client._change_txt_record("DELETE", DOMAIN, "pre-existing-value") + + call_count = self.client.r53.change_resource_record_sets.call_count + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "UPSERT") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [{"Value": "\"pre-existing-value-two\""}]) + + self.assertEqual(call_count, 1) + + def test_get_validation_rrset(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": [ + {"Name": "_acme-challenge.example.org.", + "Type": "TXT", + "ResourceRecords": [ + {"Value": "\"validation-token\""}, + {"Value": "\"another-validation-token\""}, + ], + }, + {"Name": "_acme-challenge.example.org.", + "Type": "NS", + "ResourceRecords": [ + {"Value": "ns1.example.com"}, + ], + } + ]}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(len(rrset), 2) + self.assertTrue({"Value": "\"another-validation-token\""} in rrset) + + def test_get_validation_rrset_empty(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": []}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(rrset, []) + def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From 7bc45121a13537cceef4e4bf53d4738925d55511 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 18:36:03 -0800 Subject: [PATCH 374/631] Remove the need for route53:ListResourceRecordSets * add test_change_txt_record_delete --- .../certbot_dns_route53/dns_route53.py | 24 ++++----- .../certbot_dns_route53/dns_route53_test.py | 54 ++++++++----------- 2 files changed, 31 insertions(+), 47 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index c0e8e5495..08b1d03f0 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -1,4 +1,5 @@ """Certbot Route53 authenticator plugin.""" +import collections import logging import time @@ -33,6 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") + self._resource_records = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" @@ -85,28 +87,22 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] - def _get_validation_rrset(self, zone_id, validation_domain_name): - validation_domain_name += "." - records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) - for record in records["ResourceRecordSets"]: - if record["Name"] == validation_domain_name and record["Type"] == "TXT": - return record["ResourceRecords"] - return [] - def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) - rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + rrecords = self._resource_records[validation_domain_name] challenge = {"Value": '"{0}"'.format(validation)} if action == "DELETE": - if len(rrecords) > 1: + # Remove the record being deleted from the list of tracked records + rrecords.remove(challenge) + if rrecords: # Need to update instead, as we're not deleting the rrset action = "UPSERT" - # Remove the record being deleted from the list - rrecords = [rr for rr in rrecords if rr != challenge] + else: + # Create a new list containing the record to use with DELETE + rrecords = [challenge] else: - if challenge not in rrecords: - rrecords.append(challenge) + rrecords.append(challenge) response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index 9aec05b6e..7534e132c 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,9 +178,6 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() - self.client._get_validation_rrset = mock.MagicMock( - return_value=[] - ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -189,10 +186,30 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_delete(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + validation = "some-value" + validation_record = {"Value": '"{0}"'.format(validation)} + self.client._resource_records[DOMAIN] = [validation_record] + + self.client._change_txt_record("DELETE", DOMAIN, validation) + + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "DELETE") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [validation_record]) + def test_change_txt_record_multirecord(self): self.client._find_zone_id_for_domain = mock.MagicMock() self.client._get_validation_rrset = mock.MagicMock() - self.client._get_validation_rrset.return_value = [ + self.client._resource_records[DOMAIN] = [ {"Value": "\"pre-existing-value\""}, {"Value": "\"pre-existing-value-two\""}, ] @@ -211,35 +228,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(call_count, 1) - def test_get_validation_rrset(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": [ - {"Name": "_acme-challenge.example.org.", - "Type": "TXT", - "ResourceRecords": [ - {"Value": "\"validation-token\""}, - {"Value": "\"another-validation-token\""}, - ], - }, - {"Name": "_acme-challenge.example.org.", - "Type": "NS", - "ResourceRecords": [ - {"Value": "ns1.example.com"}, - ], - } - ]}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(len(rrset), 2) - self.assertTrue({"Value": "\"another-validation-token\""} in rrset) - - def test_get_validation_rrset_empty(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": []}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(rrset, []) - def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From cee9ac586ea43d355ede8eac71f5d145902169a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:20:34 -0800 Subject: [PATCH 375/631] Don't report coverage on Apache during integration tests (#5669) * ignore Apache coverage * drop min coverage to 67 --- tests/boulder-integration.sh | 2 +- tests/integration/_common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index f2b0dcf60..b5a305016 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -443,4 +443,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 63 -m +coverage report --fail-under 67 -m diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 236090a14..a8d35ed89 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -23,7 +23,7 @@ fi certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" - omit_patterns="$omit_patterns,*_test.py,*_test_*," + omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" coverage run \ --append \ From d62c56f9c91a920a07f338bbc7aa53b7329624ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:21:01 -0800 Subject: [PATCH 376/631] Remove the assumption the domain is unique in the manual plugin (#5670) * use entire achall as key * Add manual cleanup hook * use manual cleanup hook --- certbot/plugins/manual.py | 4 ++-- certbot/plugins/manual_test.py | 6 +++--- tests/boulder-integration.sh | 4 +++- tests/manual-dns-cleanup.sh | 3 +++ 4 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 tests/manual-dns-cleanup.sh diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 07371ad34..614449d34 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -189,7 +189,7 @@ when it receives a TLS ClientHello with the SNI extension set to os.environ.update(env) _, out = hooks.execute(self.conf('auth-hook')) env['CERTBOT_AUTH_OUTPUT'] = out.strip() - self.env[achall.domain] = env + self.env[achall] = env def _perform_achall_manually(self, achall): validation = achall.validation(achall.account_key) @@ -215,7 +215,7 @@ when it receives a TLS ClientHello with the SNI extension set to def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): for achall in achalls: - env = self.env.pop(achall.domain) + env = self.env.pop(achall) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ac528e81c..e5c22b377 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -93,10 +93,10 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.perform(self.achalls), [achall.response(achall.account_key) for achall in self.achalls]) self.assertEqual( - self.auth.env[self.dns_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.dns_achall]['CERTBOT_AUTH_OUTPUT'], dns_expected) self.assertEqual( - self.auth.env[self.http_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'], http_expected) # tls_sni_01 challenge must be perform()ed above before we can # get the cert_path and key_path. @@ -107,7 +107,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall), 'novalidation') self.assertEqual( - self.auth.env[self.tls_sni_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'], tls_sni_expected) @test_util.patch_get_utility() diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index b5a305016..2b92476fd 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -233,6 +233,7 @@ certname="dns.le.wtf" common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ --renew-hook 'echo deploy >> "$HOOK_TEST"' @@ -433,7 +434,8 @@ done # Test ACMEv2-only features if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi # Most CI systems set this variable to true. diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh new file mode 100755 index 000000000..0c5c56b17 --- /dev/null +++ b/tests/manual-dns-cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/sh +curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" From 6357e051f4841df8af69a8abf5a4ba3dc8578c3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 15:32:22 -0800 Subject: [PATCH 377/631] Fallback without dns.resourceRecordSets.list permission (#5678) * Add rrset list fallback * List dns.resourceRecordSets.list as required * Handle list failures differently for add and del * Quote record content * disable not-callable for iter_entry_points * List update permission --- .../certbot_dns_google/__init__.py | 2 ++ .../certbot_dns_google/dns_google.py | 29 ++++++++++++++----- .../certbot_dns_google/dns_google_test.py | 14 ++++++++- certbot/plugins/disco.py | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 7349a7696..f19266737 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -29,6 +29,8 @@ for an account with the following permissions: * ``dns.managedZones.list`` * ``dns.resourceRecordSets.create`` * ``dns.resourceRecordSets.delete`` +* ``dns.resourceRecordSets.list`` +* ``dns.resourceRecordSets.update`` Google provides instructions for `creating a service account `_ and diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index ab8bf20de..e2088b357 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -108,6 +108,8 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = [] add_records = record_contents[:] if "\""+record_content+"\"" in record_contents: @@ -176,6 +178,8 @@ class _GoogleClient(object): return record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = ["\"" + record_content + "\""] data = { "kind": "dns#change", @@ -216,23 +220,32 @@ class _GoogleClient(object): """ Get existing TXT records from the RRset for the record name. + If an error occurs while requesting the record set, it is suppressed + and None is returned. + :param str zone_id: The ID of the managed zone. :param str record_name: The record name (typically beginning with '_acme-challenge.'). - :returns: List of TXT record values - :rtype: `list` of `string` + :returns: List of TXT record values or None + :rtype: `list` of `string` or `None` """ rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member request = rrs_request.list(managedZone=zone_id, project=self.project_id) - response = request.execute() # Add dot as the API returns absolute domains record_name += "." - if response: - for rr in response["rrsets"]: - if rr["name"] == record_name and rr["type"] == "TXT": - return rr["rrdatas"] - return [] + try: + response = request.execute() + except googleapiclient_errors.Error: + logger.info("Unable to list existing records. If you're " + "requesting a wildcard certificate, this might not work.") + logger.debug("Error was:", exc_info=True) + else: + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return None def _find_managed_zone_id(self, domain): """ diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 3291b2c3a..afab847cf 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -270,7 +270,19 @@ class GoogleClientTest(unittest.TestCase): found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") self.assertEquals(found, ["\"example-txt-contents\""]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") - self.assertEquals(not_found, []) + self.assertEquals(not_found, None) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_fallback(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute + mock_execute.side_effect = API_ERROR + + rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertFalse(rrset) def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 5a7e07ec0..062c11650 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -190,6 +190,7 @@ class PluginsRegistry(collections.Mapping): def find_all(cls): """Find plugins using setuptools entry points.""" plugins = {} + # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( constants.SETUPTOOLS_PLUGINS_ENTRY_POINT), From e0ae356aa35adf22d154113e06dd01409df93bba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Mar 2018 09:10:47 -0800 Subject: [PATCH 378/631] Upgrade pipstrap to 1.5.1 (#5681) * upgrade pipstrap to 1.5.1 * build leauto --- letsencrypt-auto-source/letsencrypt-auto | 38 +++++++++------------- letsencrypt-auto-source/pieces/pipstrap.py | 38 +++++++++------------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9ff1c1386..f97dc078d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1216,7 +1216,7 @@ UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1274,7 +1274,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1287,14 +1287,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -1379,21 +1376,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index ed55b37e9..d55d5bceb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -70,14 +70,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -162,21 +159,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: From 77fdb4d7d6194989dcc775f2e0ad81b6147c2359 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Mar 2018 10:25:42 -0800 Subject: [PATCH 379/631] Release 0.22.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 104 +++++++++++------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 12 +- letsencrypt-auto | 104 +++++++++++------- letsencrypt-auto-source/certbot-auto.asc | 16 +-- letsencrypt-auto-source/letsencrypt-auto | 26 ++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++-- 22 files changed, 186 insertions(+), 130 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 071b56ab3..93458785c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7608c0647..3f64eadb7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index d3a5c23e5..343f56013 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.1" +LE_AUTO_VERSION="0.22.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -1196,24 +1199,24 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.1 \ - --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ - --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea -acme==0.21.1 \ - --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ - --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b -certbot-apache==0.21.1 \ - --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ - --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 -certbot-nginx==0.21.1 \ - --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ - --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 +certbot==0.22.0 \ + --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ + --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 +acme==0.22.0 \ + --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ + --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 +certbot-apache==0.22.0 \ + --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ + --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd +certbot-nginx==0.22.0 \ + --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ + --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1237,6 +1240,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1270,14 +1274,14 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) @@ -1285,18 +1289,14 @@ maybe_argparse = ( PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1317,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1332,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1346,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1353,11 +1373,13 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: @@ -1428,6 +1450,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 861921ef7..c4073ed39 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4ed8e796d..780ea6db6 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 7f973709c..8c6e64af5 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0ce91e64e..7dbf9dab5 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index d12b26d83..c8593b7f7 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 856eaba0f..ec2cfd3ad 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 0dfff0402..900e9e4b2 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b255691dc..36b80bf19 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 68d8f6cdb..a5bfe256b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 3d6b3799b..442a1e4af 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index ad20725b5..b85c13dcd 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index bb71cf19a..e2715adda 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0.dev0' +version = '0.22.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 2869d29b0..97fdb75d7 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.22.0.dev0' +__version__ = '0.22.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index abebdb9c9..65f623d79 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.21.1 (certbot; + "". (default: CertbotACMEClient/0.22.0 (certbot; darwin 10.13.3) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags + (SUBCOMMAND; flags: FLAGS) Py/3.6.4). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -199,8 +199,8 @@ testing: --test-cert, --staging Use the staging server to obtain or revoke test - (invalid) certificates; equivalent to --server https - ://acme-staging.api.letsencrypt.org/directory + (invalid) certificates; equivalent to --server + https://acme-staging.api.letsencrypt.org/directory (default: False) --debug Show tracebacks in case of errors, and allow certbot- auto execution on experimental platforms (default: @@ -308,8 +308,8 @@ renew: of renewed certificate domains (for example, "example.com www.example.com" (default: None) --disable-hook-validation - Ordinarily the commands specified for --pre-hook - /--post-hook/--deploy-hook will be checked for + Ordinarily the commands specified for --pre- + hook/--post-hook/--deploy-hook will be checked for validity, to see if the programs being run are in the $PATH, so that mistakes can be caught early, even when the hooks aren't being run just yet. The validation is diff --git a/letsencrypt-auto b/letsencrypt-auto index d3a5c23e5..343f56013 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.1" +LE_AUTO_VERSION="0.22.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -1196,24 +1199,24 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.1 \ - --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ - --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea -acme==0.21.1 \ - --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ - --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b -certbot-apache==0.21.1 \ - --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ - --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 -certbot-nginx==0.21.1 \ - --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ - --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 +certbot==0.22.0 \ + --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ + --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 +acme==0.22.0 \ + --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ + --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 +certbot-apache==0.22.0 \ + --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ + --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd +certbot-nginx==0.22.0 \ + --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ + --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1237,6 +1240,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1270,14 +1274,14 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) @@ -1285,18 +1289,14 @@ maybe_argparse = ( PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1317,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1332,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1346,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1353,11 +1373,13 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: @@ -1428,6 +1450,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index f28fd9893..e9dd75a11 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X -dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 -JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc -a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 -0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR -zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR -F/j/631OC6yLRasr+xcL1gc+SCryfA== -=tkZT +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlqgLj0ACgkQTRfJlc2X +dfIAtAf/YwRvn17fdJOSXr08LP/qDPefz8AFefUJdKoOa+ikWIOWZTh5hskGtXO0 +e894FNcZqbg6NQu/KUBQAz/nBDRz8IOaWN30MFq5B4V2A3In5rn59PNaCDSKSBbC +auyU24gYkBxbDPjMpuode7yCsvHxTsB5sLNmHByMyMTBmQaiT5odAjr7PztTP52S +s/29/WOCJAYzBBFFJ9d0QD0drVSIcDM5JCuUK2vXgPuPVD4f3GankgP1nnAJ5ADV +acJp3cQ3OsofeE/HTw0qq7TiL0dGYf8yhRFovFve7tX+oujMIRALQJW6K9Qi7KTv +777V6xHuphrA+1qIrg2H8czOBDclFQ== +=Ngvl -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f97dc078d..343f56013 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.0.dev0" +LE_AUTO_VERSION="0.22.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.1 \ - --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ - --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea -acme==0.21.1 \ - --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ - --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b -certbot-apache==0.21.1 \ - --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ - --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 -certbot-nginx==0.21.1 \ - --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ - --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 +certbot==0.22.0 \ + --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ + --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 +acme==0.22.0 \ + --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ + --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 +certbot-apache==0.22.0 \ + --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ + --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd +certbot-nginx==0.22.0 \ + --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ + --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 8dd6837754301180039bdcd2192ec2ffa9911f1e..21cbf1a09de3849414c6c163dced1ead3fb2e131 100644 GIT binary patch literal 256 zcmV+b0ssC1UnH~cd*`0q_j<H6AHDK;n~Qf+SKW+Xy3TJcAQX-?GAmynNPos)KcguU5#wAO5CBQ$`2#CmQb9#87kxckZun8=50Gy6T(2&LYe?9T(@a* Date: Wed, 7 Mar 2018 10:26:08 -0800 Subject: [PATCH 380/631] Bump version to 0.23.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 93458785c..5660cf424 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3f64eadb7..f00b6d95d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c4073ed39..17abe65ec 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 780ea6db6..956e37f79 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 8c6e64af5..df7dcc59a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 7dbf9dab5..f136c7161 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index c8593b7f7..a327edf93 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ec2cfd3ad..9ff317ee1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 900e9e4b2..6c25ed452 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 36b80bf19..7b8ffd84f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a5bfe256b..53b091065 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 442a1e4af..2cbc29e6d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index b85c13dcd..3f21c4dc5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index e2715adda..25023b307 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.22.0' +version = '0.23.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 97fdb75d7..ebc8d5343 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.22.0' +__version__ = '0.23.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 343f56013..0ba318140 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.0" +LE_AUTO_VERSION="0.23.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From f4bac423fb794c14e426defecc492306ea53cbc4 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 7 Mar 2018 15:09:47 -0800 Subject: [PATCH 381/631] fix(acme): client._revoke sends default content_type (#5687) --- acme/acme/client.py | 3 +-- acme/acme/client_test.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d52c82a5c..9e2478afe 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -227,8 +227,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response = self._post(url, messages.Revocation( certificate=cert, - reason=rsn), - content_type=None) + reason=rsn)) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index a0c27e74f..00b9e19dd 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -635,8 +635,7 @@ class ClientTest(ClientTestBase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None, - acme_version=1) + self.directory[messages.Revocation], mock.ANY, acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -776,8 +775,7 @@ class ClientV2Test(ClientTestBase): def test_revoke(self): self.client.revoke(messages_test.CERT, self.rsn) self.net.post.assert_called_once_with( - self.directory["revokeCert"], mock.ANY, content_type=None, - acme_version=2) + self.directory["revokeCert"], mock.ANY, acme_version=2) class MockJSONDeSerializable(jose.JSONDeSerializable): From cc18da926ed0c64ffd9564bcbf8cc701f6506360 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:09:31 -0800 Subject: [PATCH 382/631] Quiet pylint (#5689) --- certbot-dns-google/certbot_dns_google/dns_google_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index afab847cf..72b8be8af 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -278,6 +278,7 @@ class GoogleClientTest(unittest.TestCase): def test_get_existing_fallback(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=no-member mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute mock_execute.side_effect = API_ERROR From cc24b4e40af5841c8dfddeecd9bde7d1acce62e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:12:33 -0800 Subject: [PATCH 383/631] Fix --allow-subset-of-names (#5690) * Remove aauthzr instance variable * If domain begins with fail, fail the challenge. * test --allow-subset-of-names * Fix renewal and add extra check * test after hook checks --- certbot/auth_handler.py | 80 ++++++++++++++++-------------- certbot/tests/auth_handler_test.py | 54 ++++++++++---------- tests/boulder-integration.sh | 13 +++++ tests/manual-dns-auth.sh | 12 +++-- tests/manual-dns-cleanup.sh | 11 ++-- 5 files changed, 99 insertions(+), 71 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 67d36c8cc..51cdf09ee 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -34,8 +34,6 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar aauthzrs: ACME Authorization Resources and their active challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -45,7 +43,6 @@ class AuthHandler(object): self.acme = acme self.account = account - self.aauthzrs = [] self.pref_challs = pref_challs def handle_authorizations(self, orderr, best_effort=False): @@ -63,29 +60,29 @@ class AuthHandler(object): authorizations """ - for authzr in orderr.authorizations: - self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) + aauthzrs = [AnnotatedAuthzr(authzr, []) + for authzr in orderr.authorizations] - self._choose_challenges() + self._choose_challenges(aauthzrs) config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self._has_challenges(): - resp = self._solve_challenges() + while self._has_challenges(aauthzrs): + resp = self._solve_challenges(aauthzrs) logger.info("Waiting for verification...") if config.debug_challenges: notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(aauthzrs, resp, best_effort) # Just make sure all decisions are complete. - self.verify_authzr_complete() + self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + retVal = [aauthzr.authzr for aauthzr in aauthzrs if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: @@ -94,10 +91,10 @@ class AuthHandler(object): return retVal - def _choose_challenges(self): + def _choose_challenges(self, aauthzrs): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -113,15 +110,15 @@ class AuthHandler(object): aauthzr.authzr, path) aauthzr.achalls.extend(aauthzr_achalls) - def _has_challenges(self): + def _has_challenges(self, aauthzrs): """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in self.aauthzrs) + return any(aauthzr.achalls for aauthzr in aauthzrs) - def _solve_challenges(self): + def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" resp = [] - all_achalls = self._get_all_achalls() - with error_handler.ErrorHandler(self._cleanup_challenges): + all_achalls = self._get_all_achalls(aauthzrs) + with error_handler.ErrorHandler(self._cleanup_challenges, all_achalls): try: if all_achalls: resp = self.auth.perform(all_achalls) @@ -134,15 +131,15 @@ class AuthHandler(object): return resp - def _get_all_achalls(self): + def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" all_achalls = [] - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) return all_achalls - def _respond(self, resp, best_effort): + def _respond(self, aauthzrs, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -150,24 +147,27 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(resp, chall_update) + active_achalls = self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(aauthzrs, chall_update, best_effort) finally: - self._cleanup_challenges(active_achalls) + self._cleanup_challenges(aauthzrs, active_achalls) - def _send_responses(self, resps, chall_update): + def _send_responses(self, aauthzrs, resps, chall_update): """Send responses and make sure errors are handled. + :param aauthzrs: authorizations and the selected annotated challenges + to try and perform + :type aauthzrs: `list` of `AnnotatedAuthzr` :param dict chall_update: parameter that is updated to hold aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] resps_iter = iter(resps) - for i, aauthzr in enumerate(self.aauthzrs): + for i, aauthzr in enumerate(aauthzrs): for achall in aauthzr.achalls: # This line needs to be outside of the if block below to # ensure failed challenges are cleaned up correctly @@ -184,8 +184,8 @@ class AuthHandler(object): return active_achalls - def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + def _poll_challenges(self, aauthzrs, chall_update, + best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" indices_to_check = set(chall_update.keys()) comp_indices = set() @@ -197,7 +197,7 @@ class AuthHandler(object): all_failed_achalls = set() for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - index, chall_update[index]) + aauthzrs, index, chall_update[index]) if len(comp_achalls) == len(chall_update[index]): comp_indices.add(index) @@ -210,7 +210,7 @@ class AuthHandler(object): comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - self.aauthzrs[index].authzr.body.identifier.value) + aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -223,14 +223,14 @@ class AuthHandler(object): comp_indices.clear() rounds += 1 - def _handle_check(self, index, achalls): + def _handle_check(self, aauthzrs, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - original_aauthzr = self.aauthzrs[index] + original_aauthzr = aauthzrs[index] updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] @@ -287,7 +287,7 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, achall_list=None): + def _cleanup_challenges(self, aauthzrs, achall_list=None): """Cleanup challenges. If achall_list is not provided, cleanup all achallenges. @@ -296,26 +296,30 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self._get_all_achalls() + achalls = self._get_all_achalls(aauthzrs) else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: if achall in aauthzr.achalls: aauthzr.achalls.remove(achall) break - def verify_authzr_complete(self): + def verify_authzr_complete(self, aauthzrs): """Verifies that all authorizations have been decided. + :param aauthzrs: authorizations and their selected annotated + challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` + :returns: Whether all authzr are complete :rtype: bool """ - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index b6af3d0f5..54e284d9e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -101,7 +101,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -132,7 +132,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 3) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -158,7 +158,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -187,7 +187,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # Check poll call self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) self.assertTrue(0 in list(six.iterkeys(chall_update))) self.assertEqual(len(chall_update[0]), 1) @@ -278,8 +278,8 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): - for i, aauthzr in enumerate(self.handler.aauthzrs): + def _validate_all(self, aauthzrs, unused_1, unused_2): + for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, @@ -287,7 +287,7 @@ class HandleAuthorizationsTest(unittest.TestCase): [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) - self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) + aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -304,19 +304,21 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.aauthzrs = [ + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[0], + [acme_util.HTTP01, acme_util.TLSSNI01], + [messages.STATUS_PENDING] * 2, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[1], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[2], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) + ] self.chall_update = {} - for i, aauthzr in enumerate(self.handler.aauthzrs): + for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] @@ -324,17 +326,17 @@ class PollChallengesTest(unittest.TestCase): @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.chall_update, False) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.chall_update, True) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @@ -343,7 +345,7 @@ class PollChallengesTest(unittest.TestCase): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) @mock.patch("certbot.auth_handler.time") def test_unable_to_find_challenge_status(self, unused_mock_time): @@ -353,11 +355,11 @@ class PollChallengesTest(unittest.TestCase): challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) def test_verify_authzr_failure(self): - self.assertRaises( - errors.AuthorizationError, self.handler.verify_authzr_complete) + self.assertRaises(errors.AuthorizationError, + self.handler.verify_authzr_complete, self.aauthzrs) def _mock_poll_solve_one_valid(self, authzr): # Pending here because my dummy script won't change the full status. diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2b92476fd..9748befa3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -327,6 +327,19 @@ CheckDirHooks 1 common renew --cert-name le2.wtf CheckDirHooks 1 +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ diff --git a/tests/manual-dns-auth.sh b/tests/manual-dns-auth.sh index 9b9a1a5eb..febecf455 100755 --- a/tests/manual-dns-auth.sh +++ b/tests/manual-dns-auth.sh @@ -1,4 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/set-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ - \"value\": \"$CERTBOT_VALIDATION\"}" +#!/bin/bash + +# If domain begins with fail, fail the challenge by not completing it. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/set-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ + \"value\": \"$CERTBOT_VALIDATION\"}" +fi diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh index 0c5c56b17..1c09e892c 100755 --- a/tests/manual-dns-cleanup.sh +++ b/tests/manual-dns-cleanup.sh @@ -1,3 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/clear-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +#!/bin/bash + +# If domain begins with fail, we didn't complete the challenge so there is +# nothing to clean up. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +fi From 2e6d65d9ecb5f2416413597d74c3599b470a5bd4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 17:24:30 -0800 Subject: [PATCH 384/631] Add readthedocs requirements files (#5696) * Add readthedocs requirements files. * Only install docs extras for plugin. --- .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-google/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-luadns/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-nsone/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-rfc2136/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-route53/readthedocs.org.requirements.txt | 12 ++++++++++++ 10 files changed, 120 insertions(+) create mode 100644 certbot-dns-cloudflare/readthedocs.org.requirements.txt create mode 100644 certbot-dns-cloudxns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-digitalocean/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsimple/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt create mode 100644 certbot-dns-google/readthedocs.org.requirements.txt create mode 100644 certbot-dns-luadns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-nsone/readthedocs.org.requirements.txt create mode 100644 certbot-dns-rfc2136/readthedocs.org.requirements.txt create mode 100644 certbot-dns-route53/readthedocs.org.requirements.txt diff --git a/certbot-dns-cloudflare/readthedocs.org.requirements.txt b/certbot-dns-cloudflare/readthedocs.org.requirements.txt new file mode 100644 index 000000000..b18901111 --- /dev/null +++ b/certbot-dns-cloudflare/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudflare[docs] diff --git a/certbot-dns-cloudxns/readthedocs.org.requirements.txt b/certbot-dns-cloudxns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..ae2ff8165 --- /dev/null +++ b/certbot-dns-cloudxns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudxns[docs] diff --git a/certbot-dns-digitalocean/readthedocs.org.requirements.txt b/certbot-dns-digitalocean/readthedocs.org.requirements.txt new file mode 100644 index 000000000..08d973ab3 --- /dev/null +++ b/certbot-dns-digitalocean/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-digitalocean[docs] diff --git a/certbot-dns-dnsimple/readthedocs.org.requirements.txt b/certbot-dns-dnsimple/readthedocs.org.requirements.txt new file mode 100644 index 000000000..fef73916c --- /dev/null +++ b/certbot-dns-dnsimple/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsimple[docs] diff --git a/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt new file mode 100644 index 000000000..8f8c6c731 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsmadeeasy[docs] diff --git a/certbot-dns-google/readthedocs.org.requirements.txt b/certbot-dns-google/readthedocs.org.requirements.txt new file mode 100644 index 000000000..6ea393f86 --- /dev/null +++ b/certbot-dns-google/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-google[docs] diff --git a/certbot-dns-luadns/readthedocs.org.requirements.txt b/certbot-dns-luadns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..acb51e4ef --- /dev/null +++ b/certbot-dns-luadns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-luadns[docs] diff --git a/certbot-dns-nsone/readthedocs.org.requirements.txt b/certbot-dns-nsone/readthedocs.org.requirements.txt new file mode 100644 index 000000000..dbdee4480 --- /dev/null +++ b/certbot-dns-nsone/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-nsone[docs] diff --git a/certbot-dns-rfc2136/readthedocs.org.requirements.txt b/certbot-dns-rfc2136/readthedocs.org.requirements.txt new file mode 100644 index 000000000..df89018ce --- /dev/null +++ b/certbot-dns-rfc2136/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-rfc2136[docs] diff --git a/certbot-dns-route53/readthedocs.org.requirements.txt b/certbot-dns-route53/readthedocs.org.requirements.txt new file mode 100644 index 000000000..660a90d0e --- /dev/null +++ b/certbot-dns-route53/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-route53[docs] From f13fdccf04f7bcbe9a5fa73d449fcb04abf86a56 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Mar 2018 10:51:45 -0700 Subject: [PATCH 385/631] document resps param (#5695) --- certbot/auth_handler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 51cdf09ee..4b8160ef9 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -161,6 +161,13 @@ class AuthHandler(object): :param aauthzrs: authorizations and the selected annotated challenges to try and perform :type aauthzrs: `list` of `AnnotatedAuthzr` + :param resps: challenge responses from the authenticator where + each response at index i corresponds to the annotated + challenge at index i in the list returned by + :func:`_get_all_achalls` + :type resps: `collections.abc.Iterable` of + :class:`~acme.challenges.ChallengeResponse` or `False` or + `None` :param dict chall_update: parameter that is updated to hold aauthzr index to list of outstanding solved annotated challenges From 64d647774e75c2f1ae2fcc9b7ba3855aed1178fc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Mar 2018 10:57:46 -0700 Subject: [PATCH 386/631] Update the changelog to reflect 0.22.0 (#5691) --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1369b0907..2ac87b0f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,62 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.22.0 - 2018-03-07 + +### Added + +* Support for obtaining wildcard certificates and a newer version of the ACME + protocol such as the one implemented by Let's Encrypt's upcoming ACMEv2 + endpoint was added to Certbot and its ACME library. Certbot still works with + older ACME versions and will automatically change the version of the protocol + used based on the version the ACME CA implements. +* The Apache and Nginx plugins are now able to automatically install a wildcard + certificate to multiple virtual hosts that you select from your server + configuration. +* The `certbot install` command now accepts the `--cert-name` flag for + selecting a certificate. +* `acme.client.BackwardsCompatibleClientV2` was added to Certbot's ACME library + which automatically handles most of the differences between new and old ACME + versions. `acme.client.ClientV2` is also available for people who only want + to support one version of the protocol or want to handle the differences + between versions themselves. +* certbot-auto now supports the flag --install-only which has the script + install Certbot and its dependencies and exit without invoking Certbot. +* Support for issuing a single certificate for a wildcard and base domain was + added to our Google Cloud DNS plugin. To do this, we now require your API + credentials have additional permissions, however, your credentials will + already have these permissions unless you defined a custom role with fewer + permissions than the standard DNS administrator role provided by Google. + These permissions are also only needed for the case described above so it + will continue to work for existing users. For more information about the + permissions changes, see the documentation in the plugin. + +### Changed + +* We have broken lockstep between our ACME library, Certbot, and its plugins. + This means that the different components do not need to be the same version + to work together like they did previously. This makes packaging easier + because not every piece of Certbot needs to be repackaged to ship a change to + a subset of its components. +* Support for Python 2.6 and Python 3.3 has been removed from ACME, Certbot, + Certbot's plugins, and certbot-auto. If you are using certbot-auto on a RHEL + 6 based system, it will walk you through the process of installing Certbot + with Python 3 and refuse to upgrade to a newer version of Certbot until you + have done so. +* Certbot's components now work with older versions of setuptools to simplify + packaging for EPEL 7. + +### Fixed + +* Issues caused by Certbot's Nginx plugin adding multiple ipv6only directives + has been resolved. +* A problem where Certbot's Apache plugin would add redundant include + directives for the TLS configuration managed by Certbot has been fixed. +* Certbot's webroot plugin now properly deletes any directories it creates. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/48?closed=1 + ## 0.21.1 - 2018-01-25 ### Fixed From d310ad18c716f64bd295ce951494b1bc0cc4122d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Mar 2018 17:10:23 -0700 Subject: [PATCH 387/631] Put API link at the bottom of DNS plugin docs (#5699) * Put link to API at the bottom for future docs. * Put API link at the bottom of existing docs. --- certbot-dns-cloudflare/docs/index.rst | 6 +++--- certbot-dns-cloudxns/docs/index.rst | 6 +++--- certbot-dns-digitalocean/docs/index.rst | 6 +++--- certbot-dns-dnsimple/docs/index.rst | 6 +++--- certbot-dns-dnsmadeeasy/docs/index.rst | 6 +++--- certbot-dns-google/docs/index.rst | 6 +++--- certbot-dns-luadns/docs/index.rst | 6 +++--- certbot-dns-nsone/docs/index.rst | 6 +++--- certbot-dns-rfc2136/docs/index.rst | 6 +++--- certbot-dns-route53/docs/index.rst | 6 +++--- tools/sphinx-quickstart.sh | 2 +- 11 files changed, 31 insertions(+), 31 deletions(-) diff --git a/certbot-dns-cloudflare/docs/index.rst b/certbot-dns-cloudflare/docs/index.rst index e75106a01..f2d59baea 100644 --- a/certbot-dns-cloudflare/docs/index.rst +++ b/certbot-dns-cloudflare/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudflare's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_cloudflare + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_cloudflare - :members: - Indices and tables ================== diff --git a/certbot-dns-cloudxns/docs/index.rst b/certbot-dns-cloudxns/docs/index.rst index 41ea250cd..83c6ca18d 100644 --- a/certbot-dns-cloudxns/docs/index.rst +++ b/certbot-dns-cloudxns/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudxns's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_cloudxns + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_cloudxns - :members: - Indices and tables diff --git a/certbot-dns-digitalocean/docs/index.rst b/certbot-dns-digitalocean/docs/index.rst index 9f66382ee..5e30f2d2c 100644 --- a/certbot-dns-digitalocean/docs/index.rst +++ b/certbot-dns-digitalocean/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-digitalocean's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_digitalocean + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_digitalocean - :members: - Indices and tables diff --git a/certbot-dns-dnsimple/docs/index.rst b/certbot-dns-dnsimple/docs/index.rst index 4ff1e59eb..a565ba919 100644 --- a/certbot-dns-dnsimple/docs/index.rst +++ b/certbot-dns-dnsimple/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-dnsimple's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_dnsimple + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_dnsimple - :members: - Indices and tables diff --git a/certbot-dns-dnsmadeeasy/docs/index.rst b/certbot-dns-dnsmadeeasy/docs/index.rst index 2e9aef36b..dddf0e745 100644 --- a/certbot-dns-dnsmadeeasy/docs/index.rst +++ b/certbot-dns-dnsmadeeasy/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-dnsmadeeasy's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_dnsmadeeasy + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_dnsmadeeasy - :members: - Indices and tables diff --git a/certbot-dns-google/docs/index.rst b/certbot-dns-google/docs/index.rst index a8a322f97..6bb82c76a 100644 --- a/certbot-dns-google/docs/index.rst +++ b/certbot-dns-google/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-google's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_google + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_google - :members: - Indices and tables diff --git a/certbot-dns-luadns/docs/index.rst b/certbot-dns-luadns/docs/index.rst index 589e925c0..d8cbdeb3f 100644 --- a/certbot-dns-luadns/docs/index.rst +++ b/certbot-dns-luadns/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-luadns's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_luadns + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_luadns - :members: - Indices and tables diff --git a/certbot-dns-nsone/docs/index.rst b/certbot-dns-nsone/docs/index.rst index 6abba81ec..bc204a6ff 100644 --- a/certbot-dns-nsone/docs/index.rst +++ b/certbot-dns-nsone/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-nsone's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_nsone + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_nsone - :members: - Indices and tables diff --git a/certbot-dns-rfc2136/docs/index.rst b/certbot-dns-rfc2136/docs/index.rst index 71705cb7f..c2d4daafe 100644 --- a/certbot-dns-rfc2136/docs/index.rst +++ b/certbot-dns-rfc2136/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-rfc2136's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_rfc2136 + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_rfc2136 - :members: - Indices and tables diff --git a/certbot-dns-route53/docs/index.rst b/certbot-dns-route53/docs/index.rst index bacf73150..fe5adfad5 100644 --- a/certbot-dns-route53/docs/index.rst +++ b/certbot-dns-route53/docs/index.rst @@ -10,14 +10,14 @@ Welcome to certbot-dns-route53's documentation! :maxdepth: 2 :caption: Contents: +.. automodule:: certbot_dns_route53 + :members: + .. toctree:: :maxdepth: 1 api -.. automodule:: certbot_dns_route53 - :members: - Indices and tables diff --git a/tools/sphinx-quickstart.sh b/tools/sphinx-quickstart.sh index d67c45b6f..72dc9e200 100755 --- a/tools/sphinx-quickstart.sh +++ b/tools/sphinx-quickstart.sh @@ -25,7 +25,7 @@ API Documentation :glob: api/**" > api.rst -sed -i -e "s| :caption: Contents:| :caption: Contents:\n\n.. toctree::\n :maxdepth: 1\n\n api\n\n.. automodule:: ${PROJECT//-/_}\n :members:|" index.rst +sed -i -e "s| :caption: Contents:| :caption: Contents:\n\n.. automodule:: ${PROJECT//-/_}\n :members:\n\n.. toctree::\n :maxdepth: 1\n\n api|" index.rst echo "Suggested next steps: * Add API docs to: $PROJECT/docs/api/ From 1d0e3b1bfa5afaa861adc1a4157e6a94d34320a2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 13 Mar 2018 07:08:01 -0700 Subject: [PATCH 388/631] Add documentation about DNS plugins and Docker (#5710) * make binding port optional * Add DNS docker docs * add basic DNS plugin docs * Add link to DNS plugin docs from Docker docs * Shrink table size --- docs/install.rst | 22 ++++++++++++++++++---- docs/using.rst | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index aec885b62..07af41fbd 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -115,13 +115,17 @@ these make much sense to you, you should definitely use the certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. -If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a certficate for resolves -to, `install Docker`_, then issue the following command: +If you're still not convinced and have decided to use this method, from +the server that the domain you're requesting a certficate for resolves +to, `install Docker`_, then issue a command like the one found below. If +you are using Certbot with the :ref:`Standalone` plugin, you will need +to make the port it uses accessible from outside of the container by +including something like ``-p 80:80`` or ``-p 443:443`` on the command +line before ``certbot/certbot``. .. code-block:: shell - sudo docker run -it --rm -p 443:443 -p 80:80 --name certbot \ + sudo docker run -it --rm --name certbot \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ certbot/certbot certonly @@ -131,6 +135,16 @@ Running Certbot with the ``certonly`` command will obtain a certificate and plac within Docker, you must install the certificate manually according to the procedure recommended by the provider of your webserver. +There are also Docker images for each of Certbot's DNS plugins available +at https://hub.docker.com/u/certbot which automate doing domain +validation over DNS for popular providers. To use one, just replace +``certbot/certbot`` in the command above with the name of the image you +want to use. For example, to use Certbot's plugin for Amazon Route 53, +you'd use ``certbot/dns-route53``. You may also need to add flags to +Certbot and/or mount additional directories to provide access to your +DNS API credentials. See the :ref:`DNS plugin documentation +` for more info. + For more information about the layout of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. diff --git a/docs/using.rst b/docs/using.rst index e8f84e2d7..f26ec2563 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -54,12 +54,19 @@ standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. | Requires port 80 or 443 to be available. This is useful on tls-sni-01_ (443) | systems with no webserver, or when direct integration with | the local webserver is not supported or not desired. +|dns_plugs| Y N | This category of plugins automates obtaining a certificate by dns-01_ (53) + | modifying DNS records to prove you have control over a + | domain. Doing domain validation in this way is + | the only way to obtain wildcard certificates from Let's + | Encrypt. manual_ Y N | Helps you obtain a certificate by giving you instructions to http-01_ (80), | perform domain validation yourself. Additionally allows you dns-01_ (53) or | to specify scripts to automate the validation task in a tls-sni-01_ (443) | customized way. =========== ==== ==== =============================================================== ============================= +.. |dns_plugs| replace:: :ref:`DNS plugins ` + Under the hood, plugins use one of several ACME protocol challenges_ to prove you control a domain. The options are http-01_ (which uses port 80), tls-sni-01_ (port 443) and dns-01_ (requiring configuration of a DNS server on @@ -141,6 +148,8 @@ the ``--nginx`` flag on the commandline. certbot --nginx +.. _standalone: + Standalone ---------- @@ -164,6 +173,33 @@ the Internet on the specified port using each requested domain name. .. note:: The ``--standalone-supported-challenges`` option has been deprecated since ``certbot`` version 0.9.0. +.. _dns_plugins: + +DNS Plugins +----------- + +If you'd like to obtain a wildcard certificate from Let's Encrypt or run +``certbot`` on a machine other than your target webserver, you can use one of +Certbot's DNS plugins. + +These plugins are still in the process of being packaged +by many distributions and cannot currently be installed with ``certbot-auto``. +If, however, you are comfortable installing the certificates yourself, +you can run these plugins with :ref:`Docker `. + +Once installed, you can find documentation on how to use each plugin at: + +* `certbot-dns-cloudflare `_ +* `certbot-dns-cloudxns `_ +* `certbot-dns-digitalocean `_ +* `certbot-dns-dnsimple `_ +* `certbot-dns-dnsmadeeasy `_ +* `certbot-dns-google `_ +* `certbot-dns-luadns `_ +* `certbot-dns-nsone `_ +* `certbot-dns-rfc2136 `_ +* `certbot-dns-route53 `_ + Manual ------ From 9ea14d2e2b9709ac260bcbfd0a39bf4587e0e2f7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Mar 2018 08:48:40 -0700 Subject: [PATCH 389/631] Add docs about --server (#5713) * Add docs about --server * address review comments * mention server in Docker docs * correct server URL * Use prod ACMEv2 example --- docs/install.rst | 7 +++++-- docs/using.rst | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 07af41fbd..67889d8f7 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -142,8 +142,11 @@ validation over DNS for popular providers. To use one, just replace want to use. For example, to use Certbot's plugin for Amazon Route 53, you'd use ``certbot/dns-route53``. You may also need to add flags to Certbot and/or mount additional directories to provide access to your -DNS API credentials. See the :ref:`DNS plugin documentation -` for more info. +DNS API credentials as specified in the :ref:`DNS plugin documentation +`. If you would like to obtain a wildcard certificate from +Let's Encrypt's ACMEv2 server, you'll need to include ``--server +https://acme-v02.api.letsencrypt.org/directory`` on the command line as +well. For more information about the layout of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. diff --git a/docs/using.rst b/docs/using.rst index f26ec2563..a40532998 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -837,6 +837,27 @@ Example usage for DNS-01 (Cloudflare API v4) (for example purposes only, do not .. _lock-files: +Changing the ACME Server +======================== + +By default, Certbot uses Let's Encrypt's initial production server at +https://acme-v01.api.letsencrypt.org/. You can tell Certbot to use a +different CA by providing ``--server`` on the command line or in a +:ref:`configuration file ` with the URL of the server's +ACME directory. For example, if you would like to use Let's Encrypt's +new ACMEv2 server, you would add ``--server +https://acme-v02.api.letsencrypt.org/directory`` to the command line. +Certbot will automatically select which version of the ACME protocol to +use based on the contents served at the provided URL. + +If you use ``--server`` to specify an ACME CA that implements a newer +version of the spec, you may be able to obtain a certificate for a +wildcard domain. Some CAs (such as Let's Encrypt) require that domain +validation for wildcard domains must be done through modifications to +DNS records which means that the dns-01_ challenge type must be used. To +see a list of Certbot plugins that support this challenge type and how +to use them, see plugins_. + Lock Files ========== From e405aaa4c1474c76c2270885267f5fab1f38e21c Mon Sep 17 00:00:00 2001 From: cclauss Date: Wed, 14 Mar 2018 17:37:29 +0100 Subject: [PATCH 390/631] Fix print() and xrange() for Python 3 (#5590) --- .../certbot_compatibility_test/test_driver.py | 2 ++ .../certbot_compatibility_test/validator.py | 1 + certbot-compatibility-test/nginx/roundtrip.py | 2 +- letsencrypt-auto-source/tests/auto_test.py | 1 + tools/simple_http_server.py | 2 +- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 71a0ba574..2c6c917b3 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -10,6 +10,8 @@ import sys import OpenSSL +from six.moves import xrange # pylint: disable=import-error,redefined-builtin + from acme import challenges from acme import crypto_util from acme import messages diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 0fd6efab5..791fe0da2 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -5,6 +5,7 @@ import requests import zope.interface import six +from six.moves import xrange # pylint: disable=import-error,redefined-builtin from acme import crypto_util from acme import errors as acme_errors diff --git a/certbot-compatibility-test/nginx/roundtrip.py b/certbot-compatibility-test/nginx/roundtrip.py index 852221df5..85d283c78 100644 --- a/certbot-compatibility-test/nginx/roundtrip.py +++ b/certbot-compatibility-test/nginx/roundtrip.py @@ -8,7 +8,7 @@ from certbot_nginx import nginxparser def roundtrip(stuff): success = True for t in stuff: - print t + print(t) if not os.path.isfile(t): continue with open(t, "r") as f: diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..c5109e208 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -18,6 +18,7 @@ from threading import Thread from unittest import TestCase from pytest import mark +from six.moves import xrange # pylint: disable=redefined-builtin @mark.skip diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py index 26bf231b7..14ac9a3d3 100755 --- a/tools/simple_http_server.py +++ b/tools/simple_http_server.py @@ -14,7 +14,7 @@ def serve_forever(port=0): """ server = HTTPServer(('', port), SimpleHTTPRequestHandler) - print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address) + print('Serving HTTP on {0} port {1} ...'.format(*server.server_address)) sys.stdout.flush() server.serve_forever() From 065e923bc9b75f1a7e59164da0541e5f0540e1b4 Mon Sep 17 00:00:00 2001 From: Spencer Eick Date: Wed, 14 Mar 2018 15:59:13 -0400 Subject: [PATCH 391/631] Improve "cannot find cert of key directive" error (#5525) (#5679) - Fix code to log separate error messages when either SSLCertificateFile or SSLCertificateKeyFile - directives are not found. - Update the section in install.rst where the relevant error is referenced. - Edit a docstring where 'cert' previously referred to certificate. - Edit test_deploy_cert_invalid_vhost in the test suite to cover changes. Fixes #5525. --- certbot-apache/certbot_apache/configurator.py | 22 ++++++++------ .../certbot_apache/tests/configurator_test.py | 30 +++++++++++++++++-- docs/install.rst | 10 +++---- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 6377bb114..8b996c675 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -285,8 +285,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. - Currently tries to find the last directives to deploy the cert in - the VHost associated with the given domain. If it can't find the + Currently tries to find the last directives to deploy the certificate + in the VHost associated with the given domain. If it can't find the directives, it searches the "included" confs. The function verifies that it has located the three directives and finally modifies them to point to the correct destination. After the certificate is @@ -424,14 +424,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): path["chain_path"] = self.parser.find_dir( "SSLCertificateChainFile", None, vhost.path) - if not path["cert_path"] or not path["cert_key"]: - # Throw some can't find all of the directives error" + # Handle errors when certificate/key directives cannot be found + if not path["cert_path"]: logger.warning( - "Cannot find a cert or key directive in %s. " + "Cannot find an SSLCertificateFile directive in %s. " "VirtualHost was not modified", vhost.path) - # Presumably break here so that the virtualhost is not modified raise errors.PluginError( - "Unable to find cert and/or key directives") + "Unable to find an SSLCertificateFile directive") + elif not path["cert_key"]: + logger.warning( + "Cannot find an SSLCertificateKeyFile directive for " + "certificate in %s. VirtualHost was not modified", vhost.path) + raise errors.PluginError( + "Unable to find an SSLCertificateKeyFile directive for " + "certificate") logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) @@ -2117,5 +2123,3 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) - - diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index c9bf9a63f..7b1e4fa86 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -441,13 +441,37 @@ class MultipleVhostsTest(util.ApacheTest): self.vh_truth[1].path)) def test_deploy_cert_invalid_vhost(self): + """For test cases where the `ApacheConfigurator` class' `_deploy_cert` + method is called with an invalid vhost parameter. Currently this tests + that a PluginError is appropriately raised when important directives + are missing in an SSL module.""" self.config.parser.modules.add("ssl_module") - mock_find = mock.MagicMock() - mock_find.return_value = [] - self.config.parser.find_dir = mock_find + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") + + def side_effect(*args): + """Mocks case where an SSLCertificateFile directive can be found + but an SSLCertificateKeyFile directive is missing.""" + if "SSLCertificateFile" in args: + return ["example/cert.pem"] + else: + return [] + + mock_find_dir = mock.MagicMock(return_value=[]) + mock_find_dir.side_effect = side_effect + + self.config.parser.find_dir = mock_find_dir # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] + + self.assertRaises( + errors.PluginError, self.config.deploy_cert, "random.demo", + "example/cert.pem", "example/key.pem", "example/cert_chain.pem") + + # Remove side_effect to mock case where both SSLCertificateFile + # and SSLCertificateKeyFile directives are missing + self.config.parser.find_dir.side_effect = None self.assertRaises( errors.PluginError, self.config.deploy_cert, "random.demo", "example/cert.pem", "example/key.pem", "example/cert_chain.pem") diff --git a/docs/install.rst b/docs/install.rst index 67889d8f7..45b0b7785 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -204,10 +204,11 @@ want to use the Apache plugin, it has to be installed separately: emerge -av app-crypt/certbot emerge -av app-crypt/certbot-apache -When using the Apache plugin, you will run into a "cannot find a cert or key -directive" error if you're sporting the default Gentoo ``httpd.conf``. -You can fix this by commenting out two lines in ``/etc/apache2/httpd.conf`` -as follows: +When using the Apache plugin, you will run into a "cannot find an +SSLCertificateFile directive" or "cannot find an SSLCertificateKeyFile +directive for certificate" error if you're sporting the default Gentoo +``httpd.conf``. You can fix this by commenting out two lines in +``/etc/apache2/httpd.conf`` as follows: Change @@ -257,4 +258,3 @@ whole process is described in the :doc:`contributing`. e.g. ``sudo python setup.py install``, ``sudo pip install``, ``sudo ./venv/bin/...``. These modes of operation might corrupt your operating system and are **not supported** by the Certbot team! - From b3e73bd2ab07c09c881693b8326cf75c7b5a119f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Mar 2018 17:38:37 -0700 Subject: [PATCH 392/631] removes blank line from chain.pem (#5730) --- certbot/crypto_util.py | 2 +- certbot/tests/crypto_util_test.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 37118c591..bd4e7fcfc 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -445,5 +445,5 @@ def cert_and_chain_from_fullchain(fullchain_pem): """ cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() - chain = fullchain_pem[len(cert):] + chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 480139378..2fe0e3d30 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -380,10 +380,12 @@ class CertAndChainFromFullchainTest(unittest.TestCase): cert_pem = CERT.decode() chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem + spacey_fullchain_pem = cert_pem + u'\n' + chain_pem from certbot.crypto_util import cert_and_chain_from_fullchain - cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) - self.assertEqual(cert_out, cert_pem) - self.assertEqual(chain_out, chain_pem) + for fullchain in (fullchain_pem, spacey_fullchain_pem): + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) if __name__ == '__main__': From 5ecb68f2ed41474d65f70d309d2bd05c61fd6faf Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 16 Mar 2018 15:24:55 -0700 Subject: [PATCH 393/631] Update instances of acme-staging url to acme-staging-v02 (#5734) * update instances of acme-staging url to acme-staging-v02 * keep example client as v1 * keep deactivate script as v1 --- certbot/constants.py | 2 +- certbot/tests/storage_test.py | 2 +- certbot/tests/testdata/sample-renewal.conf | 2 +- docs/contributing.rst | 10 +++++----- examples/dev-cli.ini | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/certbot/constants.py b/certbot/constants.py index a6878824b..9dfc00c6b 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -107,7 +107,7 @@ CLI_DEFAULTS = dict( dns_route53=False ) -STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" +STAGING_URI = "https://acme-staging-v02.api.letsencrypt.org/directory" # The set of reasons for revoking a certificate is defined in RFC 5280 in # section 5.3.1. The reasons that users are allowed to submit are restricted to diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c0970e72..09c752ebe 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -726,7 +726,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] self.assertEqual(self.test_rc.is_test_cert, False) - rp["server"] = "https://acme-staging.api.letsencrypt.org/directory" + rp["server"] = "https://acme-staging-v02.api.letsencrypt.org/directory" self.assertEqual(self.test_rc.is_test_cert, True) rp["server"] = "https://staging.someotherca.com/directory" self.assertEqual(self.test_rc.is_test_cert, True) diff --git a/certbot/tests/testdata/sample-renewal.conf b/certbot/tests/testdata/sample-renewal.conf index 52b3ec45c..04f9ae8ca 100644 --- a/certbot/tests/testdata/sample-renewal.conf +++ b/certbot/tests/testdata/sample-renewal.conf @@ -61,7 +61,7 @@ chain_path = /home/ubuntu/letsencrypt/chain.pem break_my_certs = False standalone = True manual = False -server = https://acme-staging.api.letsencrypt.org/directory +server = https://acme-staging-v02.api.letsencrypt.org/directory standalone_supported_challenges = "tls-sni-01,http-01" webroot = False os_packages_only = False diff --git a/docs/contributing.rst b/docs/contributing.rst index 654528e3d..45cd2e9f2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -43,7 +43,7 @@ each shell where you're working: .. code-block:: shell source ./venv/bin/activate - export SERVER=https://acme-staging.api.letsencrypt.org/directory + export SERVER=https://acme-staging-v02.api.letsencrypt.org/directory source tests/integration/_common.sh After that, your shell will be using the virtual environment, your copy of @@ -443,10 +443,10 @@ For squeeze you will need to: FreeBSD ------- -Packages can be installed on FreeBSD using ``pkg``, -or any other port-management tool (``portupgrade``, ``portmanager``, etc.) -from the pre-built package or can be built and installed from ports. -Either way will ensure proper installation of all the dependencies required +Packages can be installed on FreeBSD using ``pkg``, +or any other port-management tool (``portupgrade``, ``portmanager``, etc.) +from the pre-built package or can be built and installed from ports. +Either way will ensure proper installation of all the dependencies required for the package. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see diff --git a/examples/dev-cli.ini b/examples/dev-cli.ini index c02038ca1..a405a0aef 100644 --- a/examples/dev-cli.ini +++ b/examples/dev-cli.ini @@ -1,5 +1,5 @@ # Always use the staging/testing server - avoids rate limiting -server = https://acme-staging.api.letsencrypt.org/directory +server = https://acme-staging-v02.api.letsencrypt.org/directory # This is an example configuration file for developers config-dir = /tmp/le/conf From 79d90d67452b708eb69e66a057b3b40add7af8e4 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 16 Mar 2018 15:27:39 -0700 Subject: [PATCH 394/631] feat(nginx plugin): add HSTS enhancement (#5463) * feat(nginx plugin): add HSTS enhancement * chore(nginx): factor out block-splitting code from redirect & hsts enhancements! * chore(nginx): merge fixes * address comments * fix linter: remove a space * fix(config): remove SSL directives in HTTP block after block split, and remove_directive removes 'Managed by certbot' comment * chore(nginx-hsts): Move added SSL directives to a constant on Configurator class * fix(nginx-hsts): rebase on wildcard cert changes --- certbot-nginx/certbot_nginx/configurator.py | 94 +++++++++++++++---- certbot-nginx/certbot_nginx/constants.py | 4 + certbot-nginx/certbot_nginx/obj.py | 26 ++++- certbot-nginx/certbot_nginx/parser.py | 7 ++ .../certbot_nginx/tests/configurator_test.py | 50 +++++++++- certbot-nginx/certbot_nginx/tests/obj_test.py | 15 +++ .../certbot_nginx/tests/parser_test.py | 26 +++++ certbot/constants.py | 4 +- 8 files changed, 204 insertions(+), 22 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 83e308bac..41ca52d13 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -61,6 +61,9 @@ class NginxConfigurator(common.Installer): DEFAULT_LISTEN_PORT = '80' + # SSL directives that Certbot can add when installing a new certificate. + SSL_DIRECTIVES = ['ssl_certificate', 'ssl_certificate_key', 'ssl_dhparam'] + @classmethod def add_parser_arguments(cls, add): add("server-root", default=constants.CLI_DEFAULTS["server_root"], @@ -105,6 +108,7 @@ class NginxConfigurator(common.Installer): self.parser = None self.version = version self._enhance_func = {"redirect": self._enable_redirect, + "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} self.reverter.recovery_routine() @@ -621,7 +625,7 @@ class NginxConfigurator(common.Installer): ################################## def supported_enhancements(self): # pylint: disable=no-self-use """Returns currently supported enhancements.""" - return ['redirect', 'staple-ocsp'] + return ['redirect', 'ensure-http-header', 'staple-ocsp'] def enhance(self, domain, enhancement, options=None): """Enhance configuration. @@ -647,6 +651,40 @@ class NginxConfigurator(common.Installer): test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) + def _set_http_header(self, domain, header_substring): + """Enables header identified by header_substring on domain. + + If the vhost is listening plaintextishly, separates out the relevant + directives into a new server block, and only add header directive to + HTTPS block. + + :param str domain: the domain to enable header for. + :param str header_substring: String to uniquely identify a header. + e.g. Strict-Transport-Security, Upgrade-Insecure-Requests + :returns: Success + :raises .errors.PluginError: If no viable HTTPS host can be created or + set with header header_substring. + """ + vhosts = self.choose_vhosts(domain) + if not vhosts: + raise errors.PluginError( + "Unable to find corresponding HTTPS host for enhancement.") + for vhost in vhosts: + if vhost.has_header(header_substring): + raise errors.PluginEnhancementAlreadyPresent( + "Existing %s header" % (header_substring)) + + # if there is no separate SSL block, break the block into two and + # choose the SSL block. + if vhost.ssl and any([not addr.ssl for addr in vhost.addrs]): + _, vhost = self._split_block(vhost) + + header_directives = [ + ['\n ', 'add_header', ' ', header_substring, ' '] + + constants.HEADER_ARGS[header_substring], + ['\n']] + self.parser.add_server_directives(vhost, header_directives, replace=False) + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ @@ -655,6 +693,39 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives( vhost, redirect_block, replace=False, insert_at_top=True) + def _split_block(self, vhost, only_directives=None): + """Splits this "virtual host" (i.e. this nginx server block) into + separate HTTP and HTTPS blocks. + + :param vhost: The server block to break up into two. + :param list only_directives: If this exists, only duplicate these directives + when splitting the block. + :type vhost: :class:`~certbot_nginx.obj.VirtualHost` + :returns: tuple (http_vhost, https_vhost) + :rtype: tuple of type :class:`~certbot_nginx.obj.VirtualHost` + """ + http_vhost = self.parser.duplicate_vhost(vhost, only_directives=only_directives) + + def _ssl_match_func(directive): + return 'ssl' in directive + + def _ssl_config_match_func(directive): + return self.mod_ssl_conf in directive + + def _no_ssl_match_func(directive): + return 'ssl' not in directive + + # remove all ssl addresses and related directives from the new block + for directive in self.SSL_DIRECTIVES: + self.parser.remove_server_directives(http_vhost, directive) + self.parser.remove_server_directives(http_vhost, 'listen', match_func=_ssl_match_func) + self.parser.remove_server_directives(http_vhost, 'include', + match_func=_ssl_config_match_func) + + # remove all non-ssl addresses from the existing block + self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + return http_vhost, vhost + def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -694,28 +765,15 @@ class NginxConfigurator(common.Installer): :param `~obj.Vhost` vhost: vhost to enable redirect for """ - new_vhost = None + http_vhost = None if vhost.ssl: - new_vhost = self.parser.duplicate_vhost(vhost, - only_directives=['listen', 'server_name']) - - def _ssl_match_func(directive): - return 'ssl' in directive - - def _no_ssl_match_func(directive): - return 'ssl' not in directive - - # remove all ssl addresses from the new block - self.parser.remove_server_directives(new_vhost, 'listen', match_func=_ssl_match_func) - - # remove all non-ssl addresses from the existing block - self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + http_vhost, _ = self._split_block(vhost, ['listen', 'server_name']) # Add this at the bottom to get the right order of directives return_404_directive = [['\n ', 'return', ' ', '404']] - self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + self.parser.add_server_directives(http_vhost, return_404_directive, replace=False) - vhost = new_vhost + vhost = http_vhost if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 2e72b8686..3f263fea3 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -44,3 +44,7 @@ def os_constant(key): :return: value of constant for active os """ return CLI_DEFAULTS[key] + +HSTS_ARGS = ['\"max-age=31536000\"', ' ', 'always'] + +HEADER_ARGS = {'Strict-Transport-Security': HSTS_ARGS} diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 3625a95b9..ea5c6e2f8 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -5,7 +5,7 @@ import six from certbot.plugins import common -REDIRECT_DIRECTIVES = ['return', 'rewrite'] +ADD_HEADER_DIRECTIVE = 'add_header' class Addr(common.Addr): r"""Represents an Nginx address, i.e. what comes after the 'listen' @@ -198,6 +198,14 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods tuple(self.addrs), tuple(self.names), self.ssl, self.enabled)) + def has_header(self, header_name): + """Determine if this server block has a particular header set. + :param str header_name: The name of the header to check for, e.g. + 'Strict-Transport-Security' + """ + found = _find_directive(self.raw, ADD_HEADER_DIRECTIVE, header_name) + return found is not None + def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -233,3 +241,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods addrs=", ".join(str(addr) for addr in self.addrs), names=", ".join(self.names), https="Yes" if self.ssl else "No")) + +def _find_directive(directives, directive_name, match_content=None): + """Find a directive of type directive_name in directives. If match_content is given, + Searches for `match_content` in the directive arguments. + """ + if not directives or isinstance(directives, six.string_types) or len(directives) == 0: + return None + + # If match_content is None, just match on directive type. Otherwise, match on + # both directive type -and- the content! + if directives[0] == directive_name and \ + (match_content is None or match_content in directives): + return directives + + matches = (_find_directive(line, directive_name, match_content) for line in directives) + return next((m for m in matches if m is not None), None) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index fbd6c0ade..e329307c0 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -377,6 +377,7 @@ class NginxParser(object): del directive[directive.index('default_server')] return new_vhost + def _parse_ssl_options(ssl_options): if ssl_options is not None: try: @@ -667,6 +668,9 @@ def _add_directive(block, directive, replace, insert_at_top): elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) +def _is_certbot_comment(directive): + return '#' in directive and COMMENT in directive + def _remove_directives(directive_name, match_func, block): """Removes directives of name directive_name from a config block if match_func matches. """ @@ -674,6 +678,9 @@ def _remove_directives(directive_name, match_func, block): location = _find_location(block, directive_name, match_func=match_func) if location is None: return + # if the directive was made by us, remove the comment following + if location + 1 < len(block) and _is_certbot_comment(block[location + 1]): + del block[location + 1] del block[location] def _apply_global_addr_ssl(addr_to_ssl, parsed_server): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index bffaef5e4..9489b534a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -94,7 +94,7 @@ class NginxConfiguratorTest(util.NginxTest): "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) def test_supported_enhancements(self): - self.assertEqual(['redirect', 'staple-ocsp'], + self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], self.config.supported_enhancements()) def test_enhance(self): @@ -510,6 +510,54 @@ class NginxConfiguratorTest(util.NginxTest): ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) + def test_split_for_headers(self): + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.deploy_cert( + "example.org", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.enhance("www.example.com", "ensure-http-header", "Strict-Transport-Security") + generated_conf = self.config.parser.parsed[example_conf] + self.assertEqual( + [[['server'], [ + ['server_name', '.example.com'], + ['server_name', 'example.*'], [], + ['listen', '5001', 'ssl'], ['#', ' managed by Certbot'], + ['ssl_certificate', 'example/fullchain.pem'], ['#', ' managed by Certbot'], + ['ssl_certificate_key', 'example/key.pem'], ['#', ' managed by Certbot'], + ['include', self.config.mod_ssl_conf], ['#', ' managed by Certbot'], + ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], + [], [], + ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always'], + ['#', ' managed by Certbot'], + [], []]], + [['server'], [ + ['listen', '69.50.225.155:9000'], + ['listen', '127.0.0.1'], + ['server_name', '.example.com'], + ['server_name', 'example.*'], + [], [], []]]], + generated_conf) + + def test_http_header_hsts(self): + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.enhance("www.example.com", "ensure-http-header", + "Strict-Transport-Security") + expected = ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always'] + generated_conf = self.config.parser.parsed[example_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + + def test_http_header_hsts_twice(self): + self.config.enhance("www.example.com", "ensure-http-header", + "Strict-Transport-Security") + self.assertRaises( + errors.PluginEnhancementAlreadyPresent, + self.config.enhance, "www.example.com", + "ensure-http-header", "Strict-Transport-Security") + + @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index b30338b5b..929e7cdf0 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -143,6 +143,15 @@ class VirtualHostTest(unittest.TestCase): "filp", set([Addr.fromstring("localhost")]), False, False, set(['localhost']), raw4, []) + raw_has_hsts = [ + ['listen', '69.50.225.155:9000'], + ['server_name', 'return.com'], + ['add_header', 'always', 'set', 'Strict-Transport-Security', '\"max-age=31536000\"'], + ] + self.vhost_has_hsts = VirtualHost( + "filep", + set([Addr.fromstring("localhost")]), False, False, + set(['localhost']), raw_has_hsts, []) def test_eq(self): from certbot_nginx.obj import Addr @@ -162,6 +171,12 @@ class VirtualHostTest(unittest.TestCase): 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) + def test_has_header(self): + self.assertTrue(self.vhost_has_hsts.has_header('Strict-Transport-Security')) + self.assertFalse(self.vhost_has_hsts.has_header('Bogus-Header')) + self.assertFalse(self.vhost1.has_header('Strict-Transport-Security')) + self.assertFalse(self.vhost1.has_header('Bogus-Header')) + def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index e21acb8ea..5fce6f25a 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -191,6 +191,32 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ['server_name', '*.www.foo.com', '*.www.example.com']] self.assertTrue(nparser.has_ssl_on_directive(mock_vhost)) + + def test_remove_server_directives(self): + nparser = parser.NginxParser(self.config_path) + mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), + None, None, None, + set(['localhost', + r'~^(www\.)?(example|bar)\.']), + None, [10, 1, 9]) + example_com = nparser.abs_path('sites-enabled/example.com') + names = set(['.example.com', 'example.*']) + mock_vhost.filep = example_com + mock_vhost.names = names + mock_vhost.path = [0] + nparser.add_server_directives(mock_vhost, + [['foo', 'bar'], ['ssl_certificate', + '/etc/ssl/cert2.pem']], + replace=False) + nparser.remove_server_directives(mock_vhost, 'foo') + nparser.remove_server_directives(mock_vhost, 'ssl_certificate') + self.assertEqual(nparser.parsed[example_com], + [[['server'], [['listen', '69.50.225.155:9000'], + ['listen', '127.0.0.1'], + ['server_name', '.example.com'], + ['server_name', 'example.*'], + []]]]) + def test_add_server_directives(self): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), diff --git a/certbot/constants.py b/certbot/constants.py index 9dfc00c6b..0d0ee8d3f 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -135,13 +135,13 @@ RENEWER_DEFAULTS = dict( """Defaults for renewer script.""" -ENHANCEMENTS = ["redirect", "http-header", "ocsp-stapling", "spdy"] +ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling", "spdy"] """List of possible :class:`certbot.interfaces.IInstaller` enhancements. List of expected options parameters: - redirect: None -- http-header: TODO +- ensure-http-header: name of header (i.e. Strict-Transport-Security) - ocsp-stapling: certificate chain file path - spdy: TODO From ba6bdb50998bd55aeef7972a5c839560e02142f3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Mar 2018 17:45:46 -0700 Subject: [PATCH 395/631] Fix acme.client.Client.__init__ (#5747) * fixes #5738 * add test to prevent regressions --- acme/acme/client.py | 5 +++-- acme/acme/client_test.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9e2478afe..19615b087 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -259,11 +259,12 @@ class Client(ClientBase): """ # pylint: disable=too-many-arguments self.key = key - self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + if net is None: + net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if isinstance(directory, six.string_types): directory = messages.Directory.from_json( - self.net.get(directory).json()) + net.get(directory).json()) super(Client, self).__init__(directory=directory, net=net, acme_version=1) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 00b9e19dd..be08c2919 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -299,6 +299,16 @@ class ClientTest(ClientTestBase): directory=uri, key=KEY, alg=jose.RS256, net=self.net) self.net.get.assert_called_once_with(uri) + @mock.patch('acme.client.ClientNetwork') + def test_init_without_net(self, mock_net): + mock_net.return_value = mock.sentinel.net + alg = jose.RS256 + from acme.client import Client + self.client = Client( + directory=self.directory, key=KEY, alg=alg) + mock_net.called_once_with(KEY, alg=alg, verify_ssl=True) + self.assertEqual(self.client.net, mock.sentinel.net) + def test_register(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member From d4834da0f4bd26d1e989080805a2f1d7e442f10e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Mar 2018 17:48:46 -0700 Subject: [PATCH 396/631] fix docker link --- docs/contributing.rst | 4 ++-- docs/install.rst | 2 ++ docs/using.rst | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 654528e3d..124cb398c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -24,7 +24,7 @@ running: If you're on macOS, we recommend you skip the rest of this section and instead run Certbot in Docker. You can find instructions for how to do this :ref:`here -`. If you're running on Linux, you can run the following commands to +`. If you're running on Linux, you can run the following commands to install dependencies and set up a virtual environment where you can run Certbot. You will need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -377,7 +377,7 @@ This should generate documentation in the ``docs/_build/html`` directory. -.. _docker: +.. _docker-dev: Running the client with Docker ============================== diff --git a/docs/install.rst b/docs/install.rst index 45b0b7785..d47264545 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -94,6 +94,8 @@ Disable and remove the swapfile once the virtual environment is constructed:: user@webserver:~$ sudo swapoff /tmp/swapfile user@webserver:~$ sudo rm /tmp/swapfile +.. _docker-user: + Running with Docker ------------------- diff --git a/docs/using.rst b/docs/using.rst index a40532998..319651af0 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -185,7 +185,7 @@ Certbot's DNS plugins. These plugins are still in the process of being packaged by many distributions and cannot currently be installed with ``certbot-auto``. If, however, you are comfortable installing the certificates yourself, -you can run these plugins with :ref:`Docker `. +you can run these plugins with :ref:`Docker `. Once installed, you can find documentation on how to use each plugin at: From a26a78e84e0e93f5cf2122c18de7b6cad526f441 Mon Sep 17 00:00:00 2001 From: Edelita Valdez Date: Sat, 17 Mar 2018 19:23:53 -0700 Subject: [PATCH 397/631] Add note to developer guide docs about installing docs extras. (#4946) --- docs/contributing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index d8985b8d2..7178e2bad 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -376,6 +376,9 @@ commands: This should generate documentation in the ``docs/_build/html`` directory. +.. note:: If you skipped the "Getting Started" instructions above, + run ``pip install -e .[docs]`` to install Certbot's docs extras modules. + .. _docker-dev: From 41ed6367b4eb82b9a0d5886853aaed1dfd130241 Mon Sep 17 00:00:00 2001 From: Gopal Adhikari Date: Mon, 19 Mar 2018 14:08:45 -0400 Subject: [PATCH 398/631] Fix typo: damain -> domain (#5756) Fix typo: damain -> domain in certbot/util.py:607 --- certbot/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/util.py b/certbot/util.py index f7ce6a3bc..b3973d96b 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -604,7 +604,7 @@ def enforce_domain_sanity(domain): def is_wildcard_domain(domain): """"Is domain a wildcard domain? - :param damain: domain to check + :param domain: domain to check :type domain: `bytes` or `str` or `unicode` :returns: True if domain is a wildcard, otherwise, False From 41ce1088812dc482d822f24aed2e7b42188a7bbb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 19 Mar 2018 16:51:01 -0700 Subject: [PATCH 399/631] Fix cleanup_challenges call (#5761) * fixes cleanup_challenges * add test to prevent regressions --- certbot/auth_handler.py | 15 +++++++-------- certbot/tests/auth_handler_test.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 4b8160ef9..68389d1f8 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -118,7 +118,7 @@ class AuthHandler(object): """Get Responses for challenges from authenticators.""" resp = [] all_achalls = self._get_all_achalls(aauthzrs) - with error_handler.ErrorHandler(self._cleanup_challenges, all_achalls): + with error_handler.ErrorHandler(self._cleanup_challenges, aauthzrs, all_achalls): try: if all_achalls: resp = self.auth.perform(all_achalls) @@ -294,19 +294,18 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, aauthzrs, achall_list=None): + def _cleanup_challenges(self, aauthzrs, achalls): """Cleanup challenges. - If achall_list is not provided, cleanup all achallenges. + :param aauthzrs: authorizations and their selected annotated + challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` + :param achalls: annotated challenges to cleanup + :type achalls: `list` of :class:`certbot.achallenges.AnnotatedChallenge` """ logger.info("Cleaning up challenges") - if achall_list is None: - achalls = self._get_all_achalls(aauthzrs) - else: - achalls = achall_list - if achalls: self.auth.cleanup(achalls) for achall in achalls: diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 54e284d9e..a4ac9eb73 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -278,6 +278,17 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_perform_error(self): + self.mock_auth.perform.side_effect = errors.AuthorizationError + + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=True) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def _validate_all(self, aauthzrs, unused_1, unused_2): for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr From c0dc31fd8896618e32bba0d6628c1b0620827e43 Mon Sep 17 00:00:00 2001 From: noci2012 Date: Tue, 20 Mar 2018 21:29:24 +0100 Subject: [PATCH 400/631] Allow _acme-challenge as a zone (#5707) * Allow _acme-challenge as a zone Like described here: https://github.com/lukas2511/dehydrated/wiki/example-dns-01-nsupdate-script Not using this patch may be an issue if the parent zone has been (where a wildcard certificate has been requested.) signed by DNSSEC. Please consider this also for inclusion before dns-01 will be allowed for wildcards. * Update dns_rfc2136.py forgot one domain_name reference * Update dns_rfc2136.py moved domain up & added assignment. * Update dns_rfc2136_test.py tests adjusted to new calls. * Update dns_rfc2136_test.py Forgot on DOMAIN... * Update dns_rfc2136_test.py * Update dns_rfc2136.py pydoc updates. * Update dns_rfc2136.py --- .../certbot_dns_rfc2136/dns_rfc2136.py | 26 +++++++++---------- .../certbot_dns_rfc2136/dns_rfc2136_test.py | 16 ++++++------ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py index 85a3bf9bb..127773469 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py @@ -70,11 +70,11 @@ class Authenticator(dns_common.DNSAuthenticator): self._validate_algorithm ) - def _perform(self, domain, validation_name, validation): - self._get_rfc2136_client().add_txt_record(domain, validation_name, validation, self.ttl) + def _perform(self, _domain, validation_name, validation): + self._get_rfc2136_client().add_txt_record(validation_name, validation, self.ttl) - def _cleanup(self, domain, validation_name, validation): - self._get_rfc2136_client().del_txt_record(domain, validation_name, validation) + def _cleanup(self, _domain, validation_name, validation): + self._get_rfc2136_client().del_txt_record(validation_name, validation) def _get_rfc2136_client(self): return _RFC2136Client(self.credentials.conf('server'), @@ -95,18 +95,17 @@ class _RFC2136Client(object): }) self.algorithm = key_algorithm - def add_txt_record(self, domain_name, record_name, record_content, record_ttl): + def add_txt_record(self, record_name, record_content, record_ttl): """ Add a TXT record using the supplied information. - :param str domain: The domain to use to find the closest SOA. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). :param int record_ttl: The record TTL (number of seconds that the record may be cached). :raises certbot.errors.PluginError: if an error occurs communicating with the DNS server """ - domain = self._find_domain(domain_name) + domain = self._find_domain(record_name) n = dns.name.from_text(record_name) o = dns.name.from_text(domain) @@ -131,18 +130,17 @@ class _RFC2136Client(object): raise errors.PluginError('Received response from server: {0}' .format(dns.rcode.to_text(rcode))) - def del_txt_record(self, domain_name, record_name, record_content): + def del_txt_record(self, record_name, record_content): """ Delete a TXT record using the supplied information. - :param str domain: The domain to use to find the closest SOA. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). :param int record_ttl: The record TTL (number of seconds that the record may be cached). :raises certbot.errors.PluginError: if an error occurs communicating with the DNS server """ - domain = self._find_domain(domain_name) + domain = self._find_domain(record_name) n = dns.name.from_text(record_name) o = dns.name.from_text(domain) @@ -167,17 +165,17 @@ class _RFC2136Client(object): raise errors.PluginError('Received response from server: {0}' .format(dns.rcode.to_text(rcode))) - def _find_domain(self, domain_name): + def _find_domain(self, record_name): """ Find the closest domain with an SOA record for a given domain name. - :param str domain_name: The domain name for which to find the closest SOA record. + :param str record_name: The record name for which to find the closest SOA record. :returns: The domain, if found. :rtype: str :raises certbot.errors.PluginError: if no SOA record can be found. """ - domain_name_guesses = dns_common.base_domain_name_guesses(domain_name) + domain_name_guesses = dns_common.base_domain_name_guesses(record_name) # Loop through until we find an authoritative SOA record for guess in domain_name_guesses: @@ -185,7 +183,7 @@ class _RFC2136Client(object): return guess raise errors.PluginError('Unable to determine base domain for {0} using names: {1}.' - .format(domain_name, domain_name_guesses)) + .format(record_name, domain_name_guesses)) def _query_soa(self, domain_name): """ diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 54fdb8575..8a5166330 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -41,7 +41,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def test_perform(self): self.auth.perform([self.achall]) - expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, mock.ANY)] + expected = [mock.call.add_txt_record('_acme-challenge.'+DOMAIN, mock.ANY, mock.ANY)] self.assertEqual(expected, self.mock_client.mock_calls) def test_cleanup(self): @@ -49,7 +49,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic self.auth._attempt_cleanup = True self.auth.cleanup([self.achall]) - expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + expected = [mock.call.del_txt_record('_acme-challenge.'+DOMAIN, mock.ANY)] self.assertEqual(expected, self.mock_client.mock_calls) def test_invalid_algorithm_raises(self): @@ -82,7 +82,7 @@ class RFC2136ClientTest(unittest.TestCase): # _find_domain | pylint: disable=protected-access self.rfc2136_client._find_domain = mock.MagicMock(return_value="example.com") - self.rfc2136_client.add_txt_record(DOMAIN, "bar", "baz", 42) + self.rfc2136_client.add_txt_record("bar", "baz", 42) query_mock.assert_called_with(mock.ANY, SERVER) self.assertTrue("bar. 42 IN TXT \"baz\"" in str(query_mock.call_args[0][0])) @@ -96,7 +96,7 @@ class RFC2136ClientTest(unittest.TestCase): self.assertRaises( errors.PluginError, self.rfc2136_client.add_txt_record, - DOMAIN, "bar", "baz", 42) + "bar", "baz", 42) @mock.patch("dns.query.tcp") def test_add_txt_record_server_error(self, query_mock): @@ -107,7 +107,7 @@ class RFC2136ClientTest(unittest.TestCase): self.assertRaises( errors.PluginError, self.rfc2136_client.add_txt_record, - DOMAIN, "bar", "baz", 42) + "bar", "baz", 42) @mock.patch("dns.query.tcp") def test_del_txt_record(self, query_mock): @@ -115,7 +115,7 @@ class RFC2136ClientTest(unittest.TestCase): # _find_domain | pylint: disable=protected-access self.rfc2136_client._find_domain = mock.MagicMock(return_value="example.com") - self.rfc2136_client.del_txt_record(DOMAIN, "bar", "baz") + self.rfc2136_client.del_txt_record("bar", "baz") query_mock.assert_called_with(mock.ANY, SERVER) self.assertTrue("bar. 0 NONE TXT \"baz\"" in str(query_mock.call_args[0][0])) @@ -129,7 +129,7 @@ class RFC2136ClientTest(unittest.TestCase): self.assertRaises( errors.PluginError, self.rfc2136_client.del_txt_record, - DOMAIN, "bar", "baz") + "bar", "baz") @mock.patch("dns.query.tcp") def test_del_txt_record_server_error(self, query_mock): @@ -140,7 +140,7 @@ class RFC2136ClientTest(unittest.TestCase): self.assertRaises( errors.PluginError, self.rfc2136_client.del_txt_record, - DOMAIN, "bar", "baz") + "bar", "baz") def test_find_domain(self): # _query_soa | pylint: disable=protected-access From f01aa1295f0b35771b9937c3ee51e5966776a927 Mon Sep 17 00:00:00 2001 From: Edelita Valdez Date: Tue, 20 Mar 2018 23:40:44 -0700 Subject: [PATCH 401/631] Add quotes to command for docs extras. --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7178e2bad..2098f7cdf 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -377,7 +377,7 @@ This should generate documentation in the ``docs/_build/html`` directory. .. note:: If you skipped the "Getting Started" instructions above, - run ``pip install -e .[docs]`` to install Certbot's docs extras modules. + run ``pip install -e ".[docs]"`` to install Certbot's docs extras modules. .. _docker-dev: From cbd827382e3ab311b3ecfef91cebff51e90c794a Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Wed, 21 Mar 2018 11:17:06 -0400 Subject: [PATCH 402/631] Documentation on cron renewal (#5460) --- docs/using.rst | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 319651af0..83e27bf34 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -552,6 +552,12 @@ can run on a regular basis, like every week or every day). In that case, you are likely to want to use the ``-q`` or ``--quiet`` quiet flag to silence all output except errors. +.. seealso:: Many of the certbot clients obtained through a + distribution come with automatic renewal out of the box, + such as Debian and Ubuntu versions installed through `apt`, + CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ + for more details. + If you are manually renewing all of your certificates, the ``--force-renewal`` flag may be helpful; it causes the expiration time of the certificate(s) to be ignored when considering renewal, and attempts to @@ -647,6 +653,31 @@ The following commands could be used to specify where these files are located:: sed -i 's,/etc/letsencrypt/live/example.com,/home/user/me/certbot,g' /etc/letsencrypt/renewal/example.com.conf certbot update_symlinks +Automated Renewals +------------------ + +Many Linux distributions provide automated renewal when you use the +packages installed through their system package manager. The +following table is an *incomplete* list of distributions which do so, +as well as their methods for doing so. + +If you are not sure whether or not your system has this already +automated, refer to your distribution's documentation, or check your +system's crontab (typically in `/etc/crontab/` and `/etc/cron.*/*` and +systemd timers (`systemctl list-timers`). + +.. csv-table:: Distributions with Automated Renewal + :header: "Distribution Name", "Distribution Version", "Automation Method" + + "CentOS", "EPEL 7", "systemd" + "Debian", "jessie", "cron, systemd" + "Debian", "stretch", "cron, systemd" + "Debian", "testing/sid", "cron, systemd" + "Fedora", "26", "systemd" + "Fedora", "27", "systemd" + "RHEL", "EPEL 7", "systemd" + "Ubuntu", "17.10", "cron, systemd" + "Ubuntu", "certbot PPA", "cron, systemd" .. _where-certs: @@ -888,7 +919,7 @@ Certbot accepts a global configuration file that applies its options to all invo of Certbot. Certificate specific configuration choices should be set in the ``.conf`` files that can be found in ``/etc/letsencrypt/renewal``. -By default no cli.ini file is created, after creating one +By default no cli.ini file is created, after creating one it is possible to specify the location of this configuration file with ``certbot-auto --config cli.ini`` (or shorter ``-c cli.ini``). An example configuration file is shown below: @@ -924,6 +955,12 @@ the oldest one to make room for new logs. The number of subsequent logs can be changed by passing the desired number to the command line flag ``--max-log-backups``. +.. note:: Some distributions, including Debian and Ubuntu, disable + certbot's internal log rotation in favor of a more traditional + logrotate script. If you are using a distribution's packages and + want to alter the log rotation, check `/etc/logrotate.d/` for a + certbot rotation script. + .. _command-line: Certbot command-line options From fe8e0c98c564a3b2c09d08d31dc6cf0d6b03f3ba Mon Sep 17 00:00:00 2001 From: Sebastiaan Lokhorst Date: Wed, 21 Mar 2018 19:18:39 +0100 Subject: [PATCH 403/631] Update docs for Apache plugin (#5776) The supported OSs are now listed in another file. The table also contradicted the text below. --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 83e27bf34..7a25a5cc2 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -45,7 +45,7 @@ a combination_ of distinct authenticator and installer plugins. Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= apache_ Y Y | Automates obtaining and installing a certificate with Apache tls-sni-01_ (443) - | 2.4 on Debian-based distributions with ``libaugeas0`` 1.0+. + | 2.4 on OSes with ``libaugeas0`` 1.0+. webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. tls-sni-01_ (443) @@ -87,7 +87,7 @@ Apache The Apache plugin currently requires an OS with augeas version 1.0; currently `it supports -`_ +`_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. This automates both obtaining *and* installing certificates on an Apache webserver. To specify this plugin on the command line, simply include From 3f291e51c6df96389c8aa3fae05847066a331c7f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Mar 2018 11:21:09 -0700 Subject: [PATCH 404/631] Update certbot auto to reflect 0.22 point releases (#5768) * Release 0.22.1 (cherry picked from commit 05c75e34e274bd76b72113d9c832198c065caf86) * Bump version to 0.23.0 (cherry picked from commit 6fd3a577910f18f6c931d50f575466edeed93557) * Release 0.22.2 (cherry picked from commit ea445ed11ee3895e98f92debee541772455fe35b) * Bump version to 0.23.0 (cherry picked from commit cbe87d451c66931a084f4e513d899aae085a37d3) --- certbot-auto | 26 +++++++++---------- docs/cli-help.txt | 12 ++++----- letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++++------ letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | 4 ++- .../pieces/certbot-requirements.txt | 24 ++++++++--------- 7 files changed, 67 insertions(+), 65 deletions(-) diff --git a/certbot-auto b/certbot-auto index 343f56013..8c9745a6f 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.0" +LE_AUTO_VERSION="0.22.2" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.0 \ - --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ - --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 -acme==0.22.0 \ - --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ - --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 -certbot-apache==0.22.0 \ - --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ - --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd -certbot-nginx==0.22.0 \ - --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ - --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c +certbot==0.22.2 \ + --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ + --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef +acme==0.22.2 \ + --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ + --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 +certbot-apache==0.22.2 \ + --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ + --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 +certbot-nginx==0.22.2 \ + --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ + --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 65f623d79..399adc194 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.22.0 (certbot; + "". (default: CertbotACMEClient/0.22.2 (certbot; darwin 10.13.3) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/3.6.4). The flags + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -199,8 +199,8 @@ testing: --test-cert, --staging Use the staging server to obtain or revoke test - (invalid) certificates; equivalent to --server - https://acme-staging.api.letsencrypt.org/directory + (invalid) certificates; equivalent to --server https + ://acme-staging-v02.api.letsencrypt.org/directory (default: False) --debug Show tracebacks in case of errors, and allow certbot- auto execution on experimental platforms (default: @@ -308,8 +308,8 @@ renew: of renewed certificate domains (for example, "example.com www.example.com" (default: None) --disable-hook-validation - Ordinarily the commands specified for --pre- - hook/--post-hook/--deploy-hook will be checked for + Ordinarily the commands specified for --pre-hook + /--post-hook/--deploy-hook will be checked for validity, to see if the programs being run are in the $PATH, so that mistakes can be caught early, even when the hooks aren't being run just yet. The validation is diff --git a/letsencrypt-auto b/letsencrypt-auto index 343f56013..8c9745a6f 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.0" +LE_AUTO_VERSION="0.22.2" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.0 \ - --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ - --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 -acme==0.22.0 \ - --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ - --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 -certbot-apache==0.22.0 \ - --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ - --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd -certbot-nginx==0.22.0 \ - --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ - --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c +certbot==0.22.2 \ + --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ + --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef +acme==0.22.2 \ + --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ + --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 +certbot-apache==0.22.2 \ + --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ + --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 +certbot-nginx==0.22.2 \ + --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ + --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index e9dd75a11..3e1c4791c 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlqgLj0ACgkQTRfJlc2X -dfIAtAf/YwRvn17fdJOSXr08LP/qDPefz8AFefUJdKoOa+ikWIOWZTh5hskGtXO0 -e894FNcZqbg6NQu/KUBQAz/nBDRz8IOaWN30MFq5B4V2A3In5rn59PNaCDSKSBbC -auyU24gYkBxbDPjMpuode7yCsvHxTsB5sLNmHByMyMTBmQaiT5odAjr7PztTP52S -s/29/WOCJAYzBBFFJ9d0QD0drVSIcDM5JCuUK2vXgPuPVD4f3GankgP1nnAJ5ADV -acJp3cQ3OsofeE/HTw0qq7TiL0dGYf8yhRFovFve7tX+oujMIRALQJW6K9Qi7KTv -777V6xHuphrA+1qIrg2H8czOBDclFQ== -=Ngvl +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlqwWJwACgkQTRfJlc2X +dfIzmwgAghmc3W63/qpCtJdezYeGLJdu03LvKoWYc7dTNYj2+0P5qmAAgCvKNY34 +qYzXA1jfCOgILSzRNE5WY+rbgjcmxxsxH+luYm6Ik0909MaMQ0D3h+5cRFs/tTtd +5cX0gxL3RQQTBwpnwbAZibe7lhjs9pXBiob2ek67hVr+xEwem69BQMlOhtYJbOs1 +osccoKc4NqaKbrfgOjjtMaL8YoRPO9vJHS9rRr6hxRZlPsmvusAHAiCbIrbX4XKE +CgxJFnuHK+amtfRoZg/xCqIK3Z94yZXPezywsri/YvDteOIs+DZ2qG/StfUrNYFX +WYfFFFyld0xwQtb4Oi9u4mx4sPg7lw== +=jZDE -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ba318140..07b313528 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.0 \ - --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ - --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 -acme==0.22.0 \ - --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ - --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 -certbot-apache==0.22.0 \ - --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ - --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd -certbot-nginx==0.22.0 \ - --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ - --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c +certbot==0.22.2 \ + --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ + --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef +acme==0.22.2 \ + --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ + --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 +certbot-apache==0.22.2 \ + --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ + --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 +certbot-nginx==0.22.2 \ + --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ + --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 21cbf1a09..088d5efa6 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1 +1,3 @@ -_$³î{çžÜ÷zÆÖéúÚQ)6("Rmnåf$B|cs°;D†û¤…ë¶V,Š?ÎË8æ¨j„ê™Íàã|¨»GÍýâfÃb"ߘ§,²î*ÓÉã‹¿ì%‚ƒ™]¦6#Ÿ¢•nœ|\`+.Xû,œ)#àŸÝQB‚j@¼0GþXéz5Œ Ž42+_HÆ`?£.ÏòÚÐXÛçÝô›0ÿè/„ì<ÎtúpÜN)’¤ÃICñQõÇ&i 8Y¸Ü‚&¤R·6µ^í]l²JÜ¡¦Ê#¹–Pœ»*ãþnúæmN6B3…oæc{^r%(âÊdvœûƒ<)ï® \ No newline at end of file +ˆc÷œNdC…f’òï*2‰;û=uTM,žù]K†= jy!Ð5M÷=2D’}/Ú°3¨Û.>¢¶±ˆ:K‚†"Ótaæöç,ÙGe®ÖC‹ùÌÞr‹Fö¬¥q™î©8d³ mßq›S˜ +oGQìžÍÃY ”íé4&FöåË8çWâÕÑxUmè|¸ãè¦0wÙ”~£Ìþg†R…T)…¬Eö=· ¤Ú‡7£(ŠÕ¿,›(\e‘fÃÅ°…®·å‰*¤ô! nÀã (ª‡ƒ¦)ªª6òrk:noô¨¶Ûàªô×Fœ»]òx%ϸ÷ +c)w2~³Ÿ¯ ìËÚxC \ No newline at end of file diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index ff7b80ec0..865c3f6bc 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.22.0 \ - --hash=sha256:ebfeaf9737dc440a9f263099487523ab4c8d8da9def31a71327439d9186e00fa \ - --hash=sha256:ee307dd8f194bd710a3326aa4bacf95d358877498c0b9aa187eff0dc211dcbb3 -acme==0.22.0 \ - --hash=sha256:37e6d8e4eb7dd18edac96de209f451300e04074f14be7fce713db6931a0e4a20 \ - --hash=sha256:4a2cd52db32e914b68d8446c8e788f507c20edebbd1c36d4f3eda7b47c555fe8 -certbot-apache==0.22.0 \ - --hash=sha256:e91f6ec8203b636fa44f01017646fca68406224ee327fd56017103b78bc65539 \ - --hash=sha256:8fbab1a358ec131996d1c00f7d0ed18ee3624f8469cab3962dfd8ba40ca3e7cd -certbot-nginx==0.22.0 \ - --hash=sha256:d67210cf73cf44e8aeff04f6f228d8bde74444703ce3ccd929a450685b58c30b \ - --hash=sha256:b2b26bf9112062b02518407704cad09f7136322163d529a2dde3b6e1578ecb8c +certbot==0.22.2 \ + --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ + --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef +acme==0.22.2 \ + --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ + --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 +certbot-apache==0.22.2 \ + --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ + --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 +certbot-nginx==0.22.2 \ + --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ + --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b From afb6260c346962571113d7f26df8c6939539d2fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Mar 2018 11:21:35 -0700 Subject: [PATCH 405/631] update changelog for 0.22.1 and 0.22.2 (#5770) --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac87b0f9..1906858dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,54 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.22.2 - 2018-03-19 + +### Fixed + +* A type error introduced in 0.22.1 that would occur during challenge cleanup + when a Certbot plugin raises an exception while trying to complete the + challenge was fixed. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* certbot + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/53?closed=1 + +## 0.22.1 - 2018-03-19 + +### Changed + +* The ACME server used with Certbot's --dry-run and --staging flags is now + Let's Encrypt's ACMEv2 staging server which allows people to also test ACMEv2 + features with these flags. + +### Fixed + +* The HTTP Content-Type header is now set to the correct value during + certificate revocation with new versions of the ACME protocol. +* When using Certbot with Let's Encrypt's ACMEv2 server, it would add a blank + line to the top of chain.pem and between the certificates in fullchain.pem + for each lineage. These blank lines have been removed. +* Resolved a bug that caused Certbot's --allow-subset-of-names flag not to + work. +* Fixed a regression in acme.client.Client that caused the class to not work + when it was initialized without a ClientNetwork which is done by some of the + other projects using our ACME library. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* acme +* certbot + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/51?closed=1 + ## 0.22.0 - 2018-03-07 ### Added From bca0aa48c2d37142bf4ab06e3b5b79e68ffbd224 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 21 Mar 2018 15:41:33 -0700 Subject: [PATCH 406/631] logging: log timestamps as local timezone instead of UTC (#5607) * logging: log timestamps as local timezone instead of UTC * test(logging): expect localtime instead of gmtime * linter fix in logging --- certbot/log.py | 2 -- certbot/tests/log_test.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/certbot/log.py b/certbot/log.py index e0d2e8f11..face93cb3 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -19,7 +19,6 @@ import logging.handlers import os import sys import tempfile -import time import traceback from acme import messages @@ -148,7 +147,6 @@ def setup_log_file_handler(config, logfile, fmt): handler.doRollover() # TODO: creates empty letsencrypt.log.1 file handler.setLevel(logging.DEBUG) handler_formatter = logging.Formatter(fmt=fmt) - handler_formatter.converter = time.gmtime # don't use localtime handler.setFormatter(handler_formatter) return handler, log_file_path diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 3b0e1c5f6..549d2c5e1 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -156,7 +156,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): handler.close() self.assertEqual(handler.level, logging.DEBUG) - self.assertEqual(handler.formatter.converter, time.gmtime) + self.assertEqual(handler.formatter.converter, time.localtime) expected_path = os.path.join(self.config.logs_dir, log_file) self.assertEqual(log_path, expected_path) From 8e9a4447ff5598c303515f183698574318499598 Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Fri, 23 Mar 2018 06:24:53 +1100 Subject: [PATCH 407/631] make pip_install.sh compatible with POSIX sh(1) again (#5622) --- tools/pip_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pip_install.sh b/tools/pip_install.sh index b385c5482..78e2afa17 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/sh -e # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set # to 1, a combination of tools/oldest_constraints.txt, # tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the From 693cb1d162ae37aa184aa022803a1dec9403bcad Mon Sep 17 00:00:00 2001 From: Alokin Software Pvt Ltd Date: Fri, 23 Mar 2018 06:20:05 +0530 Subject: [PATCH 408/631] Support Openresty in the NGINX plugin (#5467) * fixes #4919 openresty_support * making the regex more general * reformatting warning to pass lint * Fix string formatting in logging function * Fix LE_AUTO_VERSION --- certbot-nginx/certbot_nginx/configurator.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 41ca52d13..4b039417d 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -895,7 +895,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError( "Unable to run %s -V" % self.conf('ctl')) - version_regex = re.compile(r"nginx/([0-9\.]*)", re.IGNORECASE) + version_regex = re.compile(r"nginx version: ([^/]+)/([0-9\.]*)", re.IGNORECASE) version_matches = version_regex.findall(text) sni_regex = re.compile(r"TLS SNI support enabled", re.IGNORECASE) @@ -912,7 +912,12 @@ class NginxConfigurator(common.Installer): if not sni_matches: raise errors.PluginError("Nginx build doesn't support SNI") - nginx_version = tuple([int(i) for i in version_matches[0].split(".")]) + product_name, product_version = version_matches[0] + if product_name is not 'nginx': + logger.warning("NGINX derivative %s is not officially supported by" + " certbot", product_name) + + nginx_version = tuple([int(i) for i in product_version.split(".")]) # nginx < 0.8.48 uses machine hostname as default server_name instead of # the empty string From 8d0d42a739893fba38cefa33fc60f8d70fbf60dc Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 23 Mar 2018 16:30:13 -0700 Subject: [PATCH 409/631] Refactor _add_directive into separate functions (#5786) * Refactor _add_directive to separate functions * UnspacedList isn't idempotent * refactor parser in add_server_directives and update_or_add_server_directives * update parser tests * remove replace=False and add to update_or_add for replace=True in configurator * remove replace=False and add to update_or_add for replace=True in http01 * update documentation --- certbot-nginx/certbot_nginx/configurator.py | 18 +-- certbot-nginx/certbot_nginx/http_01.py | 4 +- certbot-nginx/certbot_nginx/parser.py | 109 +++++++++++------- .../certbot_nginx/tests/configurator_test.py | 9 +- .../certbot_nginx/tests/parser_test.py | 29 ++--- 5 files changed, 92 insertions(+), 77 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 4b039417d..1073bff4f 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -192,8 +192,8 @@ class NginxConfigurator(common.Installer): cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] - self.parser.add_server_directives(vhost, - cert_directives, replace=True) + self.parser.update_or_add_server_directives(vhost, + cert_directives) logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % @@ -344,7 +344,7 @@ class NginxConfigurator(common.Installer): for name in vhost.names: name_block[0].append(' ') name_block[0].append(name) - self.parser.add_server_directives(vhost, name_block, replace=True) + self.parser.update_or_add_server_directives(vhost, name_block) def _get_default_vhost(self, port): vhost_list = self.parser.get_vhosts() @@ -584,7 +584,7 @@ class NginxConfigurator(common.Installer): # have it continue to do so. if len(vhost.addrs) == 0: listen_block = [['\n ', 'listen', ' ', self.DEFAULT_LISTEN_PORT]] - self.parser.add_server_directives(vhost, listen_block, replace=False) + self.parser.add_server_directives(vhost, listen_block) if vhost.ipv6_enabled(): ipv6_block = ['\n ', @@ -618,7 +618,7 @@ class NginxConfigurator(common.Installer): ]) self.parser.add_server_directives( - vhost, ssl_block, replace=False) + vhost, ssl_block) ################################## # enhancement methods (IInstaller) @@ -683,7 +683,7 @@ class NginxConfigurator(common.Installer): ['\n ', 'add_header', ' ', header_substring, ' '] + constants.HEADER_ARGS[header_substring], ['\n']] - self.parser.add_server_directives(vhost, header_directives, replace=False) + self.parser.add_server_directives(vhost, header_directives) def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost @@ -691,7 +691,7 @@ class NginxConfigurator(common.Installer): redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False, insert_at_top=True) + vhost, redirect_block, insert_at_top=True) def _split_block(self, vhost, only_directives=None): """Splits this "virtual host" (i.e. this nginx server block) into @@ -771,7 +771,7 @@ class NginxConfigurator(common.Installer): # Add this at the bottom to get the right order of directives return_404_directive = [['\n ', 'return', ' ', '404']] - self.parser.add_server_directives(http_vhost, return_404_directive, replace=False) + self.parser.add_server_directives(http_vhost, return_404_directive) vhost = http_vhost @@ -821,7 +821,7 @@ class NginxConfigurator(common.Installer): try: self.parser.add_server_directives(vhost, - stapling_directives, replace=False) + stapling_directives) except errors.MisconfigurationError as error: logger.debug(error) raise errors.PluginError("An error occurred while enabling OCSP " diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 0b1b2bfe0..93c7bfc90 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -199,9 +199,9 @@ class NginxHttp01(common.ChallengePerformer): ['return', ' ', '200', ' ', validation]]]] self.configurator.parser.add_server_directives(vhost, - location_directive, replace=False) + location_directive) rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', ' ', '$1', ' ', 'break']] self.configurator.parser.add_server_directives(vhost, - rewrite_directive, replace=False, insert_at_top=True) + rewrite_directive, insert_at_top=True) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index e329307c0..3dc70f19b 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -276,15 +276,13 @@ class NginxParser(object): return False - def add_server_directives(self, vhost, directives, replace, insert_at_top=False): - """Add or replace directives in the server block identified by vhost. + def add_server_directives(self, vhost, directives, insert_at_top=False): + """Add directives to the server block identified by vhost. This method modifies vhost to be fully consistent with the new directives. - ..note :: If replace is True and the directive already exists, the first - instance will be replaced. Otherwise, the directive is added. - ..note :: If replace is False nothing gets added if an identical - block exists already. + ..note :: It's an error to try and add a nonrepeatable directive that already + exists in the config block with a conflicting value. ..todo :: Doesn't match server blocks whose server_name directives are split across multiple conf files. @@ -292,13 +290,34 @@ class NginxParser(object): :param :class:`~certbot_nginx.obj.VirtualHost` vhost: The vhost whose information we use to match on :param list directives: The directives to add - :param bool replace: Whether to only replace existing directives :param bool insert_at_top: True if the directives need to be inserted at the top of the server block instead of the bottom """ self._modify_server_directives(vhost, - functools.partial(_add_directives, directives, replace, insert_at_top)) + functools.partial(_add_directives, directives, insert_at_top)) + + def update_or_add_server_directives(self, vhost, directives, insert_at_top=False): + """Add or replace directives in the server block identified by vhost. + + This method modifies vhost to be fully consistent with the new directives. + + ..note :: When a directive with the same name already exists in the + config block, the first instance will be replaced. Otherwise, the directive + will be appended/prepended to the config block as in add_server_directives. + + ..todo :: Doesn't match server blocks whose server_name directives are + split across multiple conf files. + + :param :class:`~certbot_nginx.obj.VirtualHost` vhost: The vhost + whose information we use to match on + :param list directives: The directives to add + :param bool insert_at_top: True if the directives need to be inserted at the top + of the server block instead of the bottom + + """ + self._modify_server_directives(vhost, + functools.partial(_update_or_add_directives, directives, insert_at_top)) def remove_server_directives(self, vhost, directive_name, match_func=None): """Remove all directives of type directive_name. @@ -524,26 +543,17 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(directives, replace, insert_at_top, block): - """Adds or replaces directives in a config block. - - When replace=False, it's an error to try and add a nonrepeatable directive that already - exists in the config block with a conflicting value. - - When replace=True and a directive with the same name already exists in the - config block, the first instance will be replaced. Otherwise, the directive - will be added to the config block. - - ..todo :: Find directives that are in included files. - - :param list directives: The new directives. - :param bool replace: Described above. - :param bool insert_at_top: Described above. - :param list block: The block to replace in - - """ +def _add_directives(directives, insert_at_top, block): + """Adds directives to a config block.""" for directive in directives: - _add_directive(block, directive, replace, insert_at_top) + _add_directive(block, directive, insert_at_top) + if block and '\n' not in block[-1]: # could be " \n " or ["\n"] ! + block.append(nginxparser.UnspacedList('\n')) + +def _update_or_add_directives(directives, insert_at_top, block): + """Adds or replaces directives in a config block.""" + for directive in directives: + _update_or_add_directive(block, directive, insert_at_top) if block and '\n' not in block[-1]: # could be " \n " or ["\n"] ! block.append(nginxparser.UnspacedList('\n')) @@ -601,28 +611,20 @@ def _find_location(block, directive_name, match_func=None): return next((index for index, line in enumerate(block) \ if line and line[0] == directive_name and (match_func is None or match_func(line))), None) -def _add_directive(block, directive, replace, insert_at_top): - """Adds or replaces a single directive in a config block. +def _is_whitespace_or_comment(directive): + """Is this directive either a whitespace or comment directive?""" + return len(directive) == 0 or directive[0] == '#' - See _add_directives for more documentation. - - """ - directive = nginxparser.UnspacedList(directive) - def is_whitespace_or_comment(directive): - """Is this directive either a whitespace or comment directive?""" - return len(directive) == 0 or directive[0] == '#' - if is_whitespace_or_comment(directive): +def _add_directive(block, directive, insert_at_top): + if not isinstance(directive, nginxparser.UnspacedList): + directive = nginxparser.UnspacedList(directive) + if _is_whitespace_or_comment(directive): # whitespace or comment block.append(directive) return location = _find_location(block, directive[0]) - if replace: - if location is not None: - block[location] = directive - comment_directive(block, location) - return # Append or prepend directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. @@ -647,7 +649,7 @@ def _add_directive(block, directive, replace, insert_at_top): for included_directive in included_directives: included_dir_loc = _find_location(block, included_directive[0]) included_dir_name = included_directive[0] - if not is_whitespace_or_comment(included_directive) \ + if not _is_whitespace_or_comment(included_directive) \ and not can_append(included_dir_loc, included_dir_name): if block[included_dir_loc] != included_directive: raise errors.MisconfigurationError(err_fmt.format(included_directive, @@ -668,6 +670,27 @@ def _add_directive(block, directive, replace, insert_at_top): elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) +def _update_directive(block, directive, location): + block[location] = directive + comment_directive(block, location) + +def _update_or_add_directive(block, directive, insert_at_top): + if not isinstance(directive, nginxparser.UnspacedList): + directive = nginxparser.UnspacedList(directive) + if _is_whitespace_or_comment(directive): + # whitespace or comment + block.append(directive) + return + + location = _find_location(block, directive[0]) + + # we can update directive + if location is not None: + _update_directive(block, directive, location) + return + + _add_directive(block, directive, insert_at_top) + def _is_certbot_comment(directive): return '#' in directive and COMMENT in directive diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 9489b534a..34abf2f0d 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -113,8 +113,7 @@ class NginxConfiguratorTest(util.NginxTest): None, [0]) self.config.parser.add_server_directives( mock_vhost, - [['listen', ' ', '5001', ' ', 'ssl']], - replace=False) + [['listen', ' ', '5001', ' ', 'ssl']]) self.config.save() # pylint: disable=protected-access @@ -206,9 +205,9 @@ class NginxConfiguratorTest(util.NginxTest): "example/chain.pem", None) - @mock.patch('certbot_nginx.parser.NginxParser.add_server_directives') - def test_deploy_cert_raise_on_add_error(self, mock_add_server_directives): - mock_add_server_directives.side_effect = errors.MisconfigurationError() + @mock.patch('certbot_nginx.parser.NginxParser.update_or_add_server_directives') + def test_deploy_cert_raise_on_add_error(self, mock_update_or_add_server_directives): + mock_update_or_add_server_directives.side_effect = errors.MisconfigurationError() self.assertRaises( errors.PluginError, self.config.deploy_cert, diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 5fce6f25a..9db251d59 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -206,8 +206,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods mock_vhost.path = [0] nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert2.pem']], - replace=False) + '/etc/ssl/cert2.pem']]) nparser.remove_server_directives(mock_vhost, 'foo') nparser.remove_server_directives(mock_vhost, 'ssl_certificate') self.assertEqual(nparser.parsed[example_com], @@ -226,8 +225,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods None, [10, 1, 9]) nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ', - '/etc/ssl/cert.pem']], - replace=False) + '/etc/ssl/cert.pem']]) ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) self.assertEqual(1, len(re.findall(ssl_re, dump))) @@ -239,10 +237,8 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods mock_vhost.path = [0] nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert2.pem']], - replace=False) - nparser.add_server_directives(mock_vhost, [['foo', 'bar']], - replace=False) + '/etc/ssl/cert2.pem']]) + nparser.add_server_directives(mock_vhost, [['foo', 'bar']]) from certbot_nginx.parser import COMMENT self.assertEqual(nparser.parsed[example_com], [[['server'], [['listen', '69.50.225.155:9000'], @@ -264,8 +260,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods nparser.add_server_directives, mock_vhost, [['foo', 'bar'], - ['ssl_certificate', '/etc/ssl/cert2.pem']], - replace=False) + ['ssl_certificate', '/etc/ssl/cert2.pem']]) def test_comment_is_repeatable(self): nparser = parser.NginxParser(self.config_path) @@ -275,12 +270,10 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods set(['.example.com', 'example.*']), None, [0]) nparser.add_server_directives(mock_vhost, - [['\n ', '#', ' ', 'what a nice comment']], - replace=False) + [['\n ', '#', ' ', 'what a nice comment']]) nparser.add_server_directives(mock_vhost, [['\n ', 'include', ' ', - nparser.abs_path('comment_in_file.conf')]], - replace=False) + nparser.abs_path('comment_in_file.conf')]]) from certbot_nginx.parser import COMMENT self.assertEqual(nparser.parsed[example_com], [[['server'], [['listen', '69.50.225.155:9000'], @@ -299,8 +292,8 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods target = set(['.example.com', 'example.*']) filep = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, target, None, [0]) - nparser.add_server_directives( - mock_vhost, [['server_name', 'foobar.com']], replace=True) + nparser.update_or_add_server_directives( + mock_vhost, [['server_name', 'foobar.com']]) from certbot_nginx.parser import COMMENT self.assertEqual( nparser.parsed[filep], @@ -310,8 +303,8 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ['server_name', 'example.*'], [] ]]]) mock_vhost.names = set(['foobar.com', 'example.*']) - nparser.add_server_directives( - mock_vhost, [['ssl_certificate', 'cert.pem']], replace=True) + nparser.update_or_add_server_directives( + mock_vhost, [['ssl_certificate', 'cert.pem']]) self.assertEqual( nparser.parsed[filep], [[['server'], [['listen', '69.50.225.155:9000'], From e9707ebc26008a1422e80fcafed8e2bc0dc471cf Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 26 Mar 2018 14:56:31 -0700 Subject: [PATCH 410/631] Allow 'default' along with 'default_server' in Nginx (#5788) * test default detection * Allow 'default' along with 'default_server' in Nginx * Test that default gets written out as default_server in canonical string * remove superfulous parens --- certbot-nginx/certbot_nginx/obj.py | 2 ++ certbot-nginx/certbot_nginx/parser.py | 8 +++++--- certbot-nginx/certbot_nginx/tests/obj_test.py | 7 ++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index ea5c6e2f8..eb94c138e 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -88,6 +88,8 @@ class Addr(common.Addr): ssl = True elif nextpart == 'default_server': default = True + elif nextpart == 'default': + default = True elif nextpart == "ipv6only=on": ipv6only = True diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 3dc70f19b..85a239c85 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -391,9 +391,11 @@ class NginxParser(object): for addr in new_vhost.addrs: addr.default = False for directive in enclosing_block[new_vhost.path[-1]][1]: - if (len(directive) > 0 and directive[0] == 'listen' - and 'default_server' in directive): - del directive[directive.index('default_server')] + if len(directive) > 0 and directive[0] == 'listen': + if 'default_server' in directive: + del directive[directive.index('default_server')] + if 'default' in directive: + del directive[directive.index('default')] return new_vhost diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 929e7cdf0..9e5853c4a 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -14,6 +14,7 @@ class AddrTest(unittest.TestCase): self.addr5 = Addr.fromstring("myhost") self.addr6 = Addr.fromstring("80 default_server spdy") self.addr7 = Addr.fromstring("unix:/var/run/nginx.sock") + self.addr8 = Addr.fromstring("*:80 default ssl") def test_fromstring(self): self.assertEqual(self.addr1.get_addr(), "192.168.1.1") @@ -46,6 +47,8 @@ class AddrTest(unittest.TestCase): self.assertFalse(self.addr6.ssl) self.assertTrue(self.addr6.default) + self.assertTrue(self.addr8.default) + self.assertEqual(None, self.addr7) def test_str(self): @@ -55,6 +58,7 @@ class AddrTest(unittest.TestCase): self.assertEqual(str(self.addr4), "*:80 default_server ssl") self.assertEqual(str(self.addr5), "myhost") self.assertEqual(str(self.addr6), "80 default_server") + self.assertEqual(str(self.addr8), "*:80 default_server ssl") def test_to_string(self): self.assertEqual(self.addr1.to_string(), "192.168.1.1") @@ -77,7 +81,8 @@ class AddrTest(unittest.TestCase): from certbot_nginx.obj import Addr any_addresses = ("0.0.0.0:80 default_server ssl", "80 default_server ssl", - "*:80 default_server ssl") + "*:80 default_server ssl", + "80 default ssl") for first, second in itertools.combinations(any_addresses, 2): self.assertEqual(Addr.fromstring(first), Addr.fromstring(second)) From 8cdb213a613ade9a5b7a58f393a391a65ec097bc Mon Sep 17 00:00:00 2001 From: Andrew Starr-Bochicchio Date: Mon, 26 Mar 2018 19:12:55 -0400 Subject: [PATCH 411/631] Google DNS: Mock API discovery to run tests without internet connection. (#5791) * Google DNS: Mock API discovery to run tests without internet connection. * Allow test to pass when run from main cerbot package. --- .../certbot_dns_google/dns_google.py | 9 +- .../certbot_dns_google/dns_google_test.py | 10 +- .../testdata/discovery.json | 1401 +++++++++++++++++ 3 files changed, 1417 insertions(+), 3 deletions(-) create mode 100644 certbot-dns-google/certbot_dns_google/testdata/discovery.json diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index e2088b357..c204cb0ca 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -81,7 +81,7 @@ class _GoogleClient(object): Encapsulates all communication with the Google Cloud DNS API. """ - def __init__(self, account_json=None): + def __init__(self, account_json=None, dns_api=None): scopes = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite'] if account_json is not None: @@ -92,7 +92,12 @@ class _GoogleClient(object): credentials = None self.project_id = self.get_project_id() - self.dns = discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False) + if not dns_api: + self.dns = discovery.build('dns', 'v1', + credentials=credentials, + cache_discovery=False) + else: + self.dns = dns_api def add_txt_record(self, domain, record_name, record_content, record_ttl): """ diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 72b8be8af..b6f6e08b6 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -4,7 +4,9 @@ import os import unittest import mock +from googleapiclient import discovery from googleapiclient.errors import Error +from googleapiclient.http import HttpMock from httplib2 import ServerNotFoundError from certbot import errors @@ -68,7 +70,13 @@ class GoogleClientTest(unittest.TestCase): def _setUp_client_with_mock(self, zone_request_side_effect): from certbot_dns_google.dns_google import _GoogleClient - client = _GoogleClient(ACCOUNT_JSON_PATH) + pwd = os.path.dirname(__file__) + rel_path = 'testdata/discovery.json' + discovery_file = os.path.join(pwd, rel_path) + http_mock = HttpMock(discovery_file, {'status': '200'}) + dns_api = discovery.build('dns', 'v1', http=http_mock) + + client = _GoogleClient(ACCOUNT_JSON_PATH, dns_api) # Setup mock_mz = mock.MagicMock() diff --git a/certbot-dns-google/certbot_dns_google/testdata/discovery.json b/certbot-dns-google/certbot_dns_google/testdata/discovery.json new file mode 100644 index 000000000..79a406645 --- /dev/null +++ b/certbot-dns-google/certbot_dns_google/testdata/discovery.json @@ -0,0 +1,1401 @@ +{ + "kind": "discovery#restDescription", + "etag": "\"-iA1DTNe4s-I6JZXPt1t1Ypy8IU/gSzgHqX4Zwypnde2YApimTf_qmE\"", + "discoveryVersion": "v1", + "id": "dns:v1", + "name": "dns", + "version": "v1", + "revision": "20180314", + "title": "Google Cloud DNS API", + "description": "Configures and serves authoritative DNS records.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://developers.google.com/cloud-dns", + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/dns/v1/projects/", + "basePath": "/dns/v1/projects/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "dns/v1/projects/", + "batchPath": "batch/dns/v1", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/cloud-platform.read-only": { + "description": "View your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readonly": { + "description": "View your DNS records hosted by Google Cloud DNS" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readwrite": { + "description": "View and manage your DNS records hosted by Google Cloud DNS" + } + } + } + }, + "schemas": { + "Change": { + "id": "Change", + "type": "object", + "description": "An atomic update to a collection of ResourceRecordSets.", + "properties": { + "additions": { + "type": "array", + "description": "Which ResourceRecordSets to add?", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "deletions": { + "type": "array", + "description": "Which ResourceRecordSets to remove? Must match existing data exactly.", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)." + }, + "isServing": { + "type": "boolean", + "description": "If the DNS queries for the zone will be served." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#change\".", + "default": "dns#change" + }, + "startTime": { + "type": "string", + "description": "The time that this operation was started by the server (output only). This is in RFC3339 text format." + }, + "status": { + "type": "string", + "description": "Status of the operation (output only).", + "enum": [ + "done", + "pending" + ], + "enumDescriptions": [ + "", + "" + ] + } + } + }, + "ChangesListResponse": { + "id": "ChangesListResponse", + "type": "object", + "description": "The response to a request to enumerate Changes to a ResourceRecordSets collection.", + "properties": { + "changes": { + "type": "array", + "description": "The requested changes.", + "items": { + "$ref": "Change" + } + }, + "header": { + "$ref": "ResponseHeader" + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#changesListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a \"snapshot\" of collections larger than the maximum page size." + } + } + }, + "DnsKey": { + "id": "DnsKey", + "type": "object", + "description": "A DNSSEC key pair.", + "properties": { + "algorithm": { + "type": "string", + "description": "String mnemonic specifying the DNSSEC algorithm of this key. Immutable after creation time.", + "enum": [ + "ecdsap256sha256", + "ecdsap384sha384", + "rsasha1", + "rsasha256", + "rsasha512" + ], + "enumDescriptions": [ + "", + "", + "", + "", + "" + ] + }, + "creationTime": { + "type": "string", + "description": "The time that this resource was created in the control plane. This is in RFC3339 text format. Output only." + }, + "description": { + "type": "string", + "description": "A mutable string of at most 1024 characters associated with this resource for the user's convenience. Has no effect on the resource's function." + }, + "digests": { + "type": "array", + "description": "Cryptographic hashes of the DNSKEY resource record associated with this DnsKey. These digests are needed to construct a DS record that points at this DNS key. Output only.", + "items": { + "$ref": "DnsKeyDigest" + } + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)." + }, + "isActive": { + "type": "boolean", + "description": "Active keys will be used to sign subsequent changes to the ManagedZone. Inactive keys will still be present as DNSKEY Resource Records for the use of resolvers validating existing signatures." + }, + "keyLength": { + "type": "integer", + "description": "Length of the key in bits. Specified at creation time then immutable.", + "format": "uint32" + }, + "keyTag": { + "type": "integer", + "description": "The key tag is a non-cryptographic hash of the a DNSKEY resource record associated with this DnsKey. The key tag can be used to identify a DNSKEY more quickly (but it is not a unique identifier). In particular, the key tag is used in a parent zone's DS record to point at the DNSKEY in this child ManagedZone. The key tag is a number in the range [0, 65535] and the algorithm to calculate it is specified in RFC4034 Appendix B. Output only.", + "format": "int32" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#dnsKey\".", + "default": "dns#dnsKey" + }, + "publicKey": { + "type": "string", + "description": "Base64 encoded public half of this key. Output only." + }, + "type": { + "type": "string", + "description": "One of \"KEY_SIGNING\" or \"ZONE_SIGNING\". Keys of type KEY_SIGNING have the Secure Entry Point flag set and, when active, will be used to sign only resource record sets of type DNSKEY. Otherwise, the Secure Entry Point flag will be cleared and this key will be used to sign only resource record sets of other types. Immutable after creation time.", + "enum": [ + "keySigning", + "zoneSigning" + ], + "enumDescriptions": [ + "", + "" + ] + } + } + }, + "DnsKeyDigest": { + "id": "DnsKeyDigest", + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "The base-16 encoded bytes of this digest. Suitable for use in a DS resource record." + }, + "type": { + "type": "string", + "description": "Specifies the algorithm used to calculate this digest.", + "enum": [ + "sha1", + "sha256", + "sha384" + ], + "enumDescriptions": [ + "", + "", + "" + ] + } + } + }, + "DnsKeySpec": { + "id": "DnsKeySpec", + "type": "object", + "description": "Parameters for DnsKey key generation. Used for generating initial keys for a new ManagedZone and as default when adding a new DnsKey.", + "properties": { + "algorithm": { + "type": "string", + "description": "String mnemonic specifying the DNSSEC algorithm of this key.", + "enum": [ + "ecdsap256sha256", + "ecdsap384sha384", + "rsasha1", + "rsasha256", + "rsasha512" + ], + "enumDescriptions": [ + "", + "", + "", + "", + "" + ] + }, + "keyLength": { + "type": "integer", + "description": "Length of the keys in bits.", + "format": "uint32" + }, + "keyType": { + "type": "string", + "description": "One of \"KEY_SIGNING\" or \"ZONE_SIGNING\". Keys of type KEY_SIGNING have the Secure Entry Point flag set and, when active, will be used to sign only resource record sets of type DNSKEY. Otherwise, the Secure Entry Point flag will be cleared and this key will be used to sign only resource record sets of other types.", + "enum": [ + "keySigning", + "zoneSigning" + ], + "enumDescriptions": [ + "", + "" + ] + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#dnsKeySpec\".", + "default": "dns#dnsKeySpec" + } + } + }, + "DnsKeysListResponse": { + "id": "DnsKeysListResponse", + "type": "object", + "description": "The response to a request to enumerate DnsKeys in a ManagedZone.", + "properties": { + "dnsKeys": { + "type": "array", + "description": "The requested resources.", + "items": { + "$ref": "DnsKey" + } + }, + "header": { + "$ref": "ResponseHeader" + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#dnsKeysListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a \"snapshot\" of collections larger than the maximum page size." + } + } + }, + "ManagedZone": { + "id": "ManagedZone", + "type": "object", + "description": "A zone is a subtree of the DNS namespace under one administrative responsibility. A ManagedZone is a resource that represents a DNS zone hosted by the Cloud DNS service.", + "properties": { + "creationTime": { + "type": "string", + "description": "The time that this resource was created on the server. This is in RFC3339 text format. Output only." + }, + "description": { + "type": "string", + "description": "A mutable string of at most 1024 characters associated with this resource for the user's convenience. Has no effect on the managed zone's function." + }, + "dnsName": { + "type": "string", + "description": "The DNS name of this managed zone, for instance \"example.com.\"." + }, + "dnssecConfig": { + "$ref": "ManagedZoneDnsSecConfig", + "description": "DNSSEC configuration." + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)", + "format": "uint64" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#managedZone\".", + "default": "dns#managedZone" + }, + "labels": { + "type": "object", + "description": "User labels.", + "additionalProperties": { + "type": "string" + } + }, + "name": { + "type": "string", + "description": "User assigned name for this resource. Must be unique within the project. The name must be 1-63 characters long, must begin with a letter, end with a letter or digit, and only contain lowercase letters, digits or dashes." + }, + "nameServerSet": { + "type": "string", + "description": "Optionally specifies the NameServerSet for this ManagedZone. A NameServerSet is a set of DNS name servers that all host the same ManagedZones. Most users will leave this field unset." + }, + "nameServers": { + "type": "array", + "description": "Delegate your managed_zone to these virtual name servers; defined by the server (output only)", + "items": { + "type": "string" + } + } + } + }, + "ManagedZoneDnsSecConfig": { + "id": "ManagedZoneDnsSecConfig", + "type": "object", + "properties": { + "defaultKeySpecs": { + "type": "array", + "description": "Specifies parameters that will be used for generating initial DnsKeys for this ManagedZone. Output only while state is not OFF.", + "items": { + "$ref": "DnsKeySpec" + } + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#managedZoneDnsSecConfig\".", + "default": "dns#managedZoneDnsSecConfig" + }, + "nonExistence": { + "type": "string", + "description": "Specifies the mechanism used to provide authenticated denial-of-existence responses. Output only while state is not OFF.", + "enum": [ + "nsec", + "nsec3" + ], + "enumDescriptions": [ + "", + "" + ] + }, + "state": { + "type": "string", + "description": "Specifies whether DNSSEC is enabled, and what mode it is in.", + "enum": [ + "off", + "on", + "transfer" + ], + "enumDescriptions": [ + "", + "", + "" + ] + } + } + }, + "ManagedZoneOperationsListResponse": { + "id": "ManagedZoneOperationsListResponse", + "type": "object", + "properties": { + "header": { + "$ref": "ResponseHeader" + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#managedZoneOperationsListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your page token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + }, + "operations": { + "type": "array", + "description": "The operation resources.", + "items": { + "$ref": "Operation" + } + } + } + }, + "ManagedZonesListResponse": { + "id": "ManagedZonesListResponse", + "type": "object", + "properties": { + "header": { + "$ref": "ResponseHeader" + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#managedZonesListResponse" + }, + "managedZones": { + "type": "array", + "description": "The managed zone resources.", + "items": { + "$ref": "ManagedZone" + } + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your page token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + } + } + }, + "Operation": { + "id": "Operation", + "type": "object", + "description": "An operation represents a successful mutation performed on a Cloud DNS resource. Operations provide: - An audit log of server resource mutations. - A way to recover/retry API calls in the case where the response is never received by the caller. Use the caller specified client_operation_id.", + "properties": { + "dnsKeyContext": { + "$ref": "OperationDnsKeyContext", + "description": "Only populated if the operation targeted a DnsKey (output only)." + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource. This is the client_operation_id if the client specified it when the mutation was initiated, otherwise, it is generated by the server. The name must be 1-63 characters long and match the regular expression [-a-z0-9]? (output only)" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#operation\".", + "default": "dns#operation" + }, + "startTime": { + "type": "string", + "description": "The time that this operation was started by the server. This is in RFC3339 text format (output only)." + }, + "status": { + "type": "string", + "description": "Status of the operation. Can be one of the following: \"PENDING\" or \"DONE\" (output only).", + "enum": [ + "done", + "pending" + ], + "enumDescriptions": [ + "", + "" + ] + }, + "type": { + "type": "string", + "description": "Type of the operation. Operations include insert, update, and delete (output only)." + }, + "user": { + "type": "string", + "description": "User who requested the operation, for example: user@example.com. cloud-dns-system for operations automatically done by the system. (output only)" + }, + "zoneContext": { + "$ref": "OperationManagedZoneContext", + "description": "Only populated if the operation targeted a ManagedZone (output only)." + } + } + }, + "OperationDnsKeyContext": { + "id": "OperationDnsKeyContext", + "type": "object", + "properties": { + "newValue": { + "$ref": "DnsKey", + "description": "The post-operation DnsKey resource." + }, + "oldValue": { + "$ref": "DnsKey", + "description": "The pre-operation DnsKey resource." + } + } + }, + "OperationManagedZoneContext": { + "id": "OperationManagedZoneContext", + "type": "object", + "properties": { + "newValue": { + "$ref": "ManagedZone", + "description": "The post-operation ManagedZone resource." + }, + "oldValue": { + "$ref": "ManagedZone", + "description": "The pre-operation ManagedZone resource." + } + } + }, + "Project": { + "id": "Project", + "type": "object", + "description": "A project resource. The project is a top level container for resources including Cloud DNS ManagedZones. Projects can be created only in the APIs console.", + "properties": { + "id": { + "type": "string", + "description": "User assigned unique identifier for the resource (output only)." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#project\".", + "default": "dns#project" + }, + "number": { + "type": "string", + "description": "Unique numeric identifier for the resource; defined by the server (output only).", + "format": "uint64" + }, + "quota": { + "$ref": "Quota", + "description": "Quotas assigned to this project (output only)." + } + } + }, + "Quota": { + "id": "Quota", + "type": "object", + "description": "Limits associated with a Project.", + "properties": { + "dnsKeysPerManagedZone": { + "type": "integer", + "description": "Maximum allowed number of DnsKeys per ManagedZone.", + "format": "int32" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#quota\".", + "default": "dns#quota" + }, + "managedZones": { + "type": "integer", + "description": "Maximum allowed number of managed zones in the project.", + "format": "int32" + }, + "resourceRecordsPerRrset": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecords per ResourceRecordSet.", + "format": "int32" + }, + "rrsetAdditionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to add per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetDeletionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to delete per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetsPerManagedZone": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets per zone in the project.", + "format": "int32" + }, + "totalRrdataSizePerChange": { + "type": "integer", + "description": "Maximum allowed size for total rrdata in one ChangesCreateRequest in bytes.", + "format": "int32" + }, + "whitelistedKeySpecs": { + "type": "array", + "description": "DNSSEC algorithm and key length types that can be used for DnsKeys.", + "items": { + "$ref": "DnsKeySpec" + } + } + } + }, + "ResourceRecordSet": { + "id": "ResourceRecordSet", + "type": "object", + "description": "A unit of data that will be returned by the DNS servers.", + "properties": { + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#resourceRecordSet\".", + "default": "dns#resourceRecordSet" + }, + "name": { + "type": "string", + "description": "For example, www.example.com." + }, + "rrdatas": { + "type": "array", + "description": "As defined in RFC 1035 (section 5) and RFC 1034 (section 3.6.1).", + "items": { + "type": "string" + } + }, + "signatureRrdatas": { + "type": "array", + "description": "As defined in RFC 4034 (section 3.2).", + "items": { + "type": "string" + } + }, + "ttl": { + "type": "integer", + "description": "Number of seconds that this ResourceRecordSet can be cached by resolvers.", + "format": "int32" + }, + "type": { + "type": "string", + "description": "The identifier of a supported record type, for example, A, AAAA, MX, TXT, and so on." + } + } + }, + "ResourceRecordSetsListResponse": { + "id": "ResourceRecordSetsListResponse", + "type": "object", + "properties": { + "header": { + "$ref": "ResponseHeader" + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#resourceRecordSetsListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + }, + "rrsets": { + "type": "array", + "description": "The resource record set resources.", + "items": { + "$ref": "ResourceRecordSet" + } + } + } + }, + "ResponseHeader": { + "id": "ResponseHeader", + "type": "object", + "description": "Elements common to every response.", + "properties": { + "operationId": { + "type": "string", + "description": "For mutating operation requests that completed successfully. This is the client_operation_id if the client specified it, otherwise it is generated by the server (output only)." + } + } + } + }, + "resources": { + "changes": { + "methods": { + "create": { + "id": "dns.changes.create", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "POST", + "description": "Atomically update the ResourceRecordSet collection.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "request": { + "$ref": "Change" + }, + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.changes.get", + "path": "{project}/managedZones/{managedZone}/changes/{changeId}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Change.", + "parameters": { + "changeId": { + "type": "string", + "description": "The identifier of the requested change, from a previous ResourceRecordSetsChangeResponse.", + "required": true, + "location": "path" + }, + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone", + "changeId" + ], + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.changes.list", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "GET", + "description": "Enumerate Changes to a ResourceRecordSet collection.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "sortBy": { + "type": "string", + "description": "Sorting criterion. The only supported value is change sequence.", + "default": "changeSequence", + "enum": [ + "changeSequence" + ], + "enumDescriptions": [ + "" + ], + "location": "query" + }, + "sortOrder": { + "type": "string", + "description": "Sorting order direction: 'ascending' or 'descending'.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ChangesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "dnsKeys": { + "methods": { + "get": { + "id": "dns.dnsKeys.get", + "path": "{project}/managedZones/{managedZone}/dnsKeys/{dnsKeyId}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing DnsKey.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "digestType": { + "type": "string", + "description": "An optional comma-separated list of digest types to compute and display for key signing keys. If omitted, the recommended digest type will be computed and displayed.", + "location": "query" + }, + "dnsKeyId": { + "type": "string", + "description": "The identifier of the requested DnsKey.", + "required": true, + "location": "path" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone", + "dnsKeyId" + ], + "response": { + "$ref": "DnsKey" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.dnsKeys.list", + "path": "{project}/managedZones/{managedZone}/dnsKeys", + "httpMethod": "GET", + "description": "Enumerate DnsKeys to a ResourceRecordSet collection.", + "parameters": { + "digestType": { + "type": "string", + "description": "An optional comma-separated list of digest types to compute and display for key signing keys. If omitted, the recommended digest type will be computed and displayed.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "DnsKeysListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "managedZoneOperations": { + "methods": { + "get": { + "id": "dns.managedZoneOperations.get", + "path": "{project}/managedZones/{managedZone}/operations/{operation}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Operation.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request.", + "required": true, + "location": "path" + }, + "operation": { + "type": "string", + "description": "Identifies the operation addressed by this request.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone", + "operation" + ], + "response": { + "$ref": "Operation" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.managedZoneOperations.list", + "path": "{project}/managedZones/{managedZone}/operations", + "httpMethod": "GET", + "description": "Enumerate Operations for the given ManagedZone.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "sortBy": { + "type": "string", + "description": "Sorting criterion. The only supported values are START_TIME and ID.", + "default": "startTime", + "enum": [ + "id", + "startTime" + ], + "enumDescriptions": [ + "", + "" + ], + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ManagedZoneOperationsListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "managedZones": { + "methods": { + "create": { + "id": "dns.managedZones.create", + "path": "{project}/managedZones", + "httpMethod": "POST", + "description": "Create a new ManagedZone.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "request": { + "$ref": "ManagedZone" + }, + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "delete": { + "id": "dns.managedZones.delete", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "DELETE", + "description": "Delete a previously created ManagedZone.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.managedZones.get", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing ManagedZone.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.managedZones.list", + "path": "{project}/managedZones", + "httpMethod": "GET", + "description": "Enumerate ManagedZones that have been created but not yet deleted.", + "parameters": { + "dnsName": { + "type": "string", + "description": "Restricts the list to return only zones with this domain name.", + "location": "query" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "ManagedZonesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "patch": { + "id": "dns.managedZones.patch", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "PATCH", + "description": "Update an existing ManagedZone. This method supports patch semantics.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "request": { + "$ref": "ManagedZone" + }, + "response": { + "$ref": "Operation" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "update": { + "id": "dns.managedZones.update", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "PUT", + "description": "Update an existing ManagedZone.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "request": { + "$ref": "ManagedZone" + }, + "response": { + "$ref": "Operation" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "projects": { + "methods": { + "get": { + "id": "dns.projects.get", + "path": "{project}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Project.", + "parameters": { + "clientOperationId": { + "type": "string", + "description": "For mutating operation requests only. An optional identifier specified by the client. Must be unique for operation resources in the Operations collection.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "Project" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "resourceRecordSets": { + "methods": { + "list": { + "id": "dns.resourceRecordSets.list", + "path": "{project}/managedZones/{managedZone}/rrsets", + "httpMethod": "GET", + "description": "Enumerate ResourceRecordSets that have been created but not yet deleted.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "name": { + "type": "string", + "description": "Restricts the list to return only records with this fully qualified domain name.", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "type": { + "type": "string", + "description": "Restricts the list to return only records of this type. If present, the \"name\" parameter must also be present.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ResourceRecordSetsListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + } + } +} From 804fd4b78a2bcf7d962fce6953103daa62ed420c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 26 Mar 2018 16:28:30 -0700 Subject: [PATCH 412/631] factor out location_directive_for_achall (#5794) --- certbot-nginx/certbot_nginx/http_01.py | 27 +++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 93c7bfc90..d08a3b1cb 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -159,16 +159,22 @@ class NginxHttp01(common.ChallengePerformer): document_root = os.path.join( self.configurator.config.work_dir, "http_01_nonexistent") + block.extend([['server_name', ' ', achall.domain], + ['root', ' ', document_root], + self._location_directive_for_achall(achall) + ]) + # TODO: do we want to return something else if they otherwise access this block? + return [['server'], block] + + def _location_directive_for_achall(self, achall): validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) - block.extend([['server_name', ' ', achall.domain], - ['root', ' ', document_root], - [['location', ' ', '=', ' ', validation_path], - [['default_type', ' ', 'text/plain'], - ['return', ' ', '200', ' ', validation]]]]) - # TODO: do we want to return something else if they otherwise access this block? - return [['server'], block] + location_directive = [['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]] + return location_directive + def _make_or_mod_server_block(self, achall): """Modifies a server block to respond to a challenge. @@ -191,12 +197,7 @@ class NginxHttp01(common.ChallengePerformer): vhost = vhosts[0] # Modify existing server block - validation = achall.validation(achall.account_key) - validation_path = self._get_validation_path(achall) - - location_directive = [[['location', ' ', '=', ' ', validation_path], - [['default_type', ' ', 'text/plain'], - ['return', ' ', '200', ' ', validation]]]] + location_directive = [self._location_directive_for_achall(achall)] self.configurator.parser.add_server_directives(vhost, location_directive) From af2cce4ca8c55c37a18197b565b67bdbf5c83879 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 26 Mar 2018 17:09:02 -0700 Subject: [PATCH 413/631] fix(auth_handler): cleanup is always called (#5779) * fix(auth_handler): cleanup is always called * test(auth_handler): tests for various error cases --- certbot/auth_handler.py | 42 ++++++++++++++--------------- certbot/error_handler.py | 19 +++++++++++-- certbot/tests/auth_handler_test.py | 26 ++++++++++++++++++ certbot/tests/error_handler_test.py | 30 ++++++++++++++++++++- 4 files changed, 92 insertions(+), 25 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 68389d1f8..9d7c75f57 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -69,14 +69,15 @@ class AuthHandler(object): # While there are still challenges remaining... while self._has_challenges(aauthzrs): - resp = self._solve_challenges(aauthzrs) - logger.info("Waiting for verification...") - if config.debug_challenges: - notify('Challenges loaded. Press continue to submit to CA. ' - 'Pass "-v" for more info about challenges.', pause=True) + with error_handler.ExitHandler(self._cleanup_challenges, aauthzrs): + resp = self._solve_challenges(aauthzrs) + logger.info("Waiting for verification...") + if config.debug_challenges: + notify('Challenges loaded. Press continue to submit to CA. ' + 'Pass "-v" for more info about challenges.', pause=True) - # Send all Responses - this modifies achalls - self._respond(aauthzrs, resp, best_effort) + # Send all Responses - this modifies achalls + self._respond(aauthzrs, resp, best_effort) # Just make sure all decisions are complete. self.verify_authzr_complete(aauthzrs) @@ -118,14 +119,13 @@ class AuthHandler(object): """Get Responses for challenges from authenticators.""" resp = [] all_achalls = self._get_all_achalls(aauthzrs) - with error_handler.ErrorHandler(self._cleanup_challenges, aauthzrs, all_achalls): - try: - if all_achalls: - resp = self.auth.perform(all_achalls) - except errors.AuthorizationError: - logger.critical("Failure in setting up challenges.") - logger.info("Attempting to clean up outstanding challenges...") - raise + try: + if all_achalls: + resp = self.auth.perform(all_achalls) + except errors.AuthorizationError: + logger.critical("Failure in setting up challenges.") + logger.info("Attempting to clean up outstanding challenges...") + raise assert len(resp) == len(all_achalls) @@ -147,13 +147,10 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(aauthzrs, resp, chall_update) + self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... - try: - self._poll_challenges(aauthzrs, chall_update, best_effort) - finally: - self._cleanup_challenges(aauthzrs, active_achalls) + self._poll_challenges(aauthzrs, chall_update, best_effort) def _send_responses(self, aauthzrs, resps, chall_update): """Send responses and make sure errors are handled. @@ -294,7 +291,7 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, aauthzrs, achalls): + def _cleanup_challenges(self, aauthzrs, achalls=None): """Cleanup challenges. :param aauthzrs: authorizations and their selected annotated @@ -305,7 +302,8 @@ class AuthHandler(object): """ logger.info("Cleaning up challenges") - + if achalls is None: + achalls = self._get_all_achalls(aauthzrs) if achalls: self.auth.cleanup(achalls) for achall in achalls: diff --git a/certbot/error_handler.py b/certbot/error_handler.py index 842243f70..e2737711e 100644 --- a/certbot/error_handler.py +++ b/certbot/error_handler.py @@ -24,7 +24,6 @@ if os.name != "nt": if signal.getsignal(signal_code) != signal.SIG_IGN: _SIGNALS.append(signal_code) - class ErrorHandler(object): """Context manager for running code that must be cleaned up on failure. @@ -55,6 +54,7 @@ class ErrorHandler(object): """ def __init__(self, func=None, *args, **kwargs): + self.call_on_regular_exit = False self.body_executed = False self.funcs = [] self.prev_handlers = {} @@ -70,8 +70,11 @@ class ErrorHandler(object): self.body_executed = True retval = False # SystemExit is ignored to properly handle forks that don't exec - if exec_type in (None, SystemExit): + if exec_type is SystemExit: return retval + elif exec_type is None: + if not self.call_on_regular_exit: + return retval elif exec_type is errors.SignalExit: logger.debug("Encountered signals: %s", self.received_signals) retval = True @@ -136,3 +139,15 @@ class ErrorHandler(object): for signum in self.received_signals: logger.debug("Calling signal %s", signum) os.kill(os.getpid(), signum) + +class ExitHandler(ErrorHandler): + """Context manager for running code that must be cleaned up. + + Subclass of ErrorHandler, with the same usage and parameters. + In addition to cleaning up on all signals, also cleans up on + regular exit. + """ + def __init__(self, func=None, *args, **kwargs): + ErrorHandler.__init__(self, func, *args, **kwargs) + self.call_on_regular_exit = True + diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index a4ac9eb73..9a8a13498 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -289,6 +289,32 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + @mock.patch("certbot.auth_handler.AuthHandler._respond") + def test_respond_error(self, mock_respond): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + mock_respond.side_effect = errors.AuthorizationError + + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") + @mock.patch("certbot.auth_handler.AuthHandler.verify_authzr_complete") + def test_incomplete_authzr_error(self, mock_verify, mock_poll): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + mock_verify.side_effect = errors.AuthorizationError + mock_poll.side_effect = self._validate_all + + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def _validate_all(self, aauthzrs, unused_1, unused_2): for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 60dcf5e99..d4c48c242 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -36,7 +36,7 @@ def send_signal(signum): class ErrorHandlerTest(unittest.TestCase): - """Tests for certbot.error_handler.""" + """Tests for certbot.error_handler.ErrorHandler.""" def setUp(self): from certbot import error_handler @@ -47,6 +47,7 @@ class ErrorHandlerTest(unittest.TestCase): self.handler = error_handler.ErrorHandler(self.init_func, *self.init_args, **self.init_kwargs) + # pylint: disable=protected-access self.signals = error_handler._SIGNALS @@ -113,6 +114,33 @@ class ErrorHandlerTest(unittest.TestCase): pass self.assertFalse(self.init_func.called) + def test_regular_exit(self): + func = mock.MagicMock() + self.handler.register(func) + with self.handler: + pass + self.init_func.assert_not_called() + func.assert_not_called() + + +class ExitHandlerTest(ErrorHandlerTest): + """Tests for certbot.error_handler.ExitHandler.""" + + def setUp(self): + from certbot import error_handler + super(ExitHandlerTest, self).setUp() + self.handler = error_handler.ExitHandler(self.init_func, + *self.init_args, + **self.init_kwargs) + + def test_regular_exit(self): + func = mock.MagicMock() + self.handler.register(func) + with self.handler: + pass + self.init_func.assert_called_once_with(*self.init_args, + **self.init_kwargs) + func.assert_called_once_with() if __name__ == "__main__": unittest.main() # pragma: no cover From 4d082e22e61be36cebc4b22e8c7f507347b72e67 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 27 Mar 2018 15:11:39 -0700 Subject: [PATCH 414/631] Remove ipv6only=on from duplicated vhosts (#5793) * rename delete_default to remove_singleton_listen_params * update docstring * add documentation to obj.py * add test for remove duplicate ipv6only * Remove ipv6only=on from duplicated vhosts * add test to make sure ipv6only=on is not erroneously removed --- certbot-nginx/certbot_nginx/configurator.py | 3 ++- certbot-nginx/certbot_nginx/obj.py | 2 ++ certbot-nginx/certbot_nginx/parser.py | 12 ++++++---- .../certbot_nginx/tests/parser_test.py | 24 ++++++++++++++++++- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 1073bff4f..13fe493fc 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -332,7 +332,8 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: default_vhost = self._get_default_vhost(port) - self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) + self.new_vhost = self.parser.duplicate_vhost(default_vhost, + remove_singleton_listen_params=True) self.new_vhost.names = set() self._add_server_name_to_vhost(self.new_vhost, domain) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index eb94c138e..8868fcfad 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -29,6 +29,8 @@ class Addr(common.Addr): :param str port: port number or "\*" or "" :param bool ssl: Whether the directive includes 'ssl' :param bool default: Whether the directive includes 'default_server' + :param bool default: Whether this is an IPv6 address + :param bool ipv6only: Whether the directive includes 'ipv6only=on' """ UNSPECIFIED_IPV4_ADDRESSES = ('', '*', '0.0.0.0') diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 85a239c85..23eacf70a 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -354,13 +354,14 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, delete_default=False, only_directives=None): + def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, + only_directives=None): """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx.obj.VirtualHost` vhost_template: The vhost whose information we copy - :param bool delete_default: If we should remove default_server - from listen directives in the block. + :param bool remove_singleton_listen_params: If we should remove parameters + from listen directives in the block that can only be used once per address :param list only_directives: If it exists, only duplicate the named directives. Only looks at first level of depth; does not expand includes. @@ -387,15 +388,18 @@ class NginxParser(object): enclosing_block.append(raw_in_parsed) new_vhost.path[-1] = len(enclosing_block) - 1 - if delete_default: + if remove_singleton_listen_params: for addr in new_vhost.addrs: addr.default = False + addr.ipv6only = False for directive in enclosing_block[new_vhost.path[-1]][1]: if len(directive) > 0 and directive[0] == 'listen': if 'default_server' in directive: del directive[directive.index('default_server')] if 'default' in directive: del directive[directive.index('default')] + if 'ipv6only=on' in directive: + del directive[directive.index('ipv6only=on')] return new_vhost diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 9db251d59..1e9703185 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -430,7 +430,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods vhosts = nparser.get_vhosts() default = [x for x in vhosts if 'default' in x.filep][0] - new_vhost = nparser.duplicate_vhost(default, delete_default=True) + new_vhost = nparser.duplicate_vhost(default, remove_singleton_listen_params=True) nparser.filedump(ext='') # check properties of new vhost @@ -448,6 +448,28 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods self.assertEqual(len(default.raw), len(new_vhost_parsed.raw)) self.assertTrue(next(iter(default.addrs)).super_eq(next(iter(new_vhost_parsed.addrs)))) + def test_duplicate_vhost_remove_ipv6only(self): + nparser = parser.NginxParser(self.config_path) + + vhosts = nparser.get_vhosts() + ipv6ssl = [x for x in vhosts if 'ipv6ssl' in x.filep][0] + new_vhost = nparser.duplicate_vhost(ipv6ssl, remove_singleton_listen_params=True) + nparser.filedump(ext='') + + for addr in new_vhost.addrs: + self.assertFalse(addr.ipv6only) + + identical_vhost = nparser.duplicate_vhost(ipv6ssl, remove_singleton_listen_params=False) + nparser.filedump(ext='') + + called = False + for addr in identical_vhost.addrs: + if addr.ipv6: + self.assertTrue(addr.ipv6only) + called = True + self.assertTrue(called) + + if __name__ == "__main__": unittest.main() # pragma: no cover From 669312d248bc11bc981472f1e4d0e0dfe0c9e4a7 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 27 Mar 2018 15:25:34 -0700 Subject: [PATCH 415/631] We don't try to add location blocks through a mechanism that checks REPEATABLE_DIRECTIVES, and it wouldn't work as an accurate check even if we did, so just remove it (#5787) --- certbot-nginx/certbot_nginx/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 23eacf70a..577e783fc 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -565,7 +565,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] From a779e06d472b71de17ca0c63de6c6913a95fa55a Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 27 Mar 2018 17:33:48 -0700 Subject: [PATCH 416/631] Add integration tests for nginx plugin (#5441) * Add a rewrite directive for the .well-known location so we don't hit existing rewrites * add comment * Add (nonexistent) document root so we don't use the default value * Add integration tests for nginx plugin * add a sleep 5 to test on travis * put sleep 5 in the right spot * test return status of grep respecting -e and note that we're actually not posix compliant * redelete newline --- .../tests/boulder-integration.conf.sh | 35 +++++++++++++++++-- certbot-nginx/tests/boulder-integration.sh | 34 ++++++++++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index c38180698..4374f9094 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -49,9 +49,9 @@ http { server { # IPv4. - listen 5002; + listen 5002 $default_server; # IPv6. - listen [::]:5002 default ipv6only=on; + listen [::]:5002 $default_server; server_name nginx.wtf nginx2.wtf; root $root/webroot; @@ -62,5 +62,36 @@ http { try_files \$uri \$uri/ /index.html; } } + + server { + listen 5002; + listen [::]:5002; + server_name nginx3.wtf; + + root $root/webroot; + + location /.well-known/ { + return 404; + } + + return 301 https://\$host\$request_uri; + } + + server { + listen 8082; + listen [::]:8082; + server_name nginx4.wtf nginx5.wtf; + } + + server { + listen 5002; + listen [::]:5002; + listen 5001 ssl; + listen [::]:5001 ssl; + if (\$scheme != "https") { + return 301 https://\$host\$request_uri; + } + server_name nginx6.wtf nginx7.wtf; + } } EOF diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index f236fb103..d6bd767ce 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -1,4 +1,4 @@ -#!/bin/sh -xe +#!/bin/bash -xe # prerequisite: apt-get install --no-install-recommends nginx-light openssl . ./tests/integration/_common.sh @@ -6,13 +6,15 @@ export PATH="/usr/sbin:$PATH" # /usr/sbin/nginx nginx_root="$root/nginx" mkdir $nginx_root -original=$(root="$nginx_root" ./certbot-nginx/tests/boulder-integration.conf.sh) -nginx_conf="$nginx_root/nginx.conf" -echo "$original" > $nginx_conf +reload_nginx () { + original=$(root="$nginx_root" ./certbot-nginx/tests/boulder-integration.conf.sh) + nginx_conf="$nginx_root/nginx.conf" + echo "$original" > $nginx_conf -killall nginx || true -nginx -c $nginx_root/nginx.conf + killall nginx || true + nginx -c $nginx_root/nginx.conf +} certbot_test_nginx () { certbot_test \ @@ -32,10 +34,30 @@ test_deployment_and_rollback() { diff -q <(echo "$original") $nginx_conf } +export default_server="default_server" +reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf +# Overlapping location block and server-block-level return 301 +certbot_test_nginx --domains nginx3.wtf --preferred-challenges http +test_deployment_and_rollback nginx3.wtf +# No matching server block; default_server exists +certbot_test_nginx --domains nginx4.wtf --preferred-challenges http +test_deployment_and_rollback nginx4.wtf +# No matching server block; default_server does not exist +export default_server="" +reload_nginx +if nginx -c $nginx_root/nginx.conf -T 2>/dev/null | grep "default_server"; then + echo "Failed to remove default_server" + exit 1 +fi +certbot_test_nginx --domains nginx5.wtf --preferred-challenges http +test_deployment_and_rollback nginx5.wtf +# Mutiple domains, mix of matching and not +certbot_test_nginx --domains nginx6.wtf,nginx7.wtf --preferred-challenges http +test_deployment_and_rollback nginx6.wtf # note: not reached if anything above fails, hence "killall" at the # top From 336950c0b906c4830a15536ad2e8216efa0d08d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 28 Mar 2018 08:37:00 -0700 Subject: [PATCH 417/631] Update oldest tests to test against 0.22.0 versions (#5800) --- certbot-nginx/local-oldest-requirements.txt | 4 ++-- local-oldest-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 65f5a758e..4b88f0288 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.22.0 +certbot[dev]==0.22.0 diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 2346300a3..37bef2083 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ --e acme[dev] +acme[dev]==0.22.0 From 7630550ac47f2191069bd7482d0c0473be8b4551 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 29 Mar 2018 14:15:59 -0700 Subject: [PATCH 418/631] Revert "Update oldest tests to test against 0.22.0 versions (#5800)" (#5809) This reverts commit 336950c0b906c4830a15536ad2e8216efa0d08d1. --- certbot-nginx/local-oldest-requirements.txt | 4 ++-- local-oldest-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 4b88f0288..65f5a758e 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.22.0 -certbot[dev]==0.22.0 +-e acme[dev] +-e .[dev] diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 37bef2083..2346300a3 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ -acme[dev]==0.22.0 +-e acme[dev] From 5ff7f2211edde280a50087d3a593557d6b90405d Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 29 Mar 2018 15:34:38 -0700 Subject: [PATCH 419/631] Explicitly add six as a dependency in letsencrypt-auto-source dockerfiles (#5808) * update documentation * explicitly add six as a dependency in letsencrypt-auto-source dockerfiles * pin six version --- letsencrypt-auto-source/Dockerfile.centos6 | 2 +- letsencrypt-auto-source/Dockerfile.precise | 2 +- letsencrypt-auto-source/Dockerfile.trusty | 2 +- letsencrypt-auto-source/Dockerfile.wheezy | 2 +- letsencrypt-auto-source/tests/__init__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 47eb48f50..92fec168b 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -11,7 +11,7 @@ RUN yum install -y python-pip sudo COPY ./pieces/pipstrap.py /opt RUN /opt/pipstrap.py # Pin pytest version for increased stability -RUN pip install pytest==3.2.5 +RUN pip install pytest==3.2.5 six==1.10.0 # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.precise index 71d572315..39a167c14 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.precise @@ -15,7 +15,7 @@ RUN apt-get update && \ COPY ./pieces/pipstrap.py /opt RUN /opt/pipstrap.py # Pin pytest version for increased stability -RUN pip install pytest==3.2.5 +RUN pip install pytest==3.2.5 six==1.10.0 # Let that user sudo: RUN sed -i.bkp -e \ diff --git a/letsencrypt-auto-source/Dockerfile.trusty b/letsencrypt-auto-source/Dockerfile.trusty index e0aacd118..3de88f9af 100644 --- a/letsencrypt-auto-source/Dockerfile.trusty +++ b/letsencrypt-auto-source/Dockerfile.trusty @@ -19,7 +19,7 @@ RUN apt-get update && \ COPY ./pieces/pipstrap.py /opt RUN /opt/pipstrap.py # Pin pytest version for increased stability -RUN pip install pytest==3.2.5 +RUN pip install pytest==3.2.5 six==1.10.0 RUN mkdir -p /home/lea/certbot diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.wheezy index 56948d22a..f4f3fea15 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.wheezy @@ -14,7 +14,7 @@ RUN apt-get update && \ COPY ./pieces/pipstrap.py /opt RUN /opt/pipstrap.py # Pin pytest version for increased stability -RUN pip install pytest==3.2.5 +RUN pip install pytest==3.2.5 six==1.10.0 # Let that user sudo: RUN sed -i.bkp -e \ diff --git a/letsencrypt-auto-source/tests/__init__.py b/letsencrypt-auto-source/tests/__init__.py index 45db90444..8a1613aa5 100644 --- a/letsencrypt-auto-source/tests/__init__.py +++ b/letsencrypt-auto-source/tests/__init__.py @@ -2,6 +2,6 @@ Run these locally by saying... :: - ./build.py && docker build -t lea . && docker run --rm -t -i lea + ./build.py && docker build -t lea . -f Dockerfile. && docker run --rm -t -i lea """ From 8231b1a19cc62262f099dc17609d4d1041e2e5da Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 29 Mar 2018 17:09:21 -0700 Subject: [PATCH 420/631] Pin Lexicon version to 2.2.1 (#5803) --- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index df7dcc59a..3493638a0 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -11,7 +11,7 @@ version = '0.23.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index a327edf93..f2887d371 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -11,7 +11,7 @@ version = '0.23.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 9ff317ee1..a3ee12cf0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -11,7 +11,7 @@ version = '0.23.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 7b8ffd84f..f872d7093 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -11,7 +11,7 @@ version = '0.23.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 53b091065..102ed48c2 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -11,7 +11,7 @@ version = '0.23.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 47440241d..d02204215 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.1.14 +dns-lexicon==2.2.1 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 From 4d706ac77e3eabb3134d0ea75ab76fd58b412bb6 Mon Sep 17 00:00:00 2001 From: Joshua Bowman Date: Fri, 30 Mar 2018 17:16:48 -0700 Subject: [PATCH 421/631] Update default to ACMEv2 server (#5722) --- certbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/constants.py b/certbot/constants.py index 0d0ee8d3f..9da5415d4 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -84,7 +84,7 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", - server="https://acme-v01.api.letsencrypt.org/directory", + server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers configurator=None, From 8fd3f6c64cb9fc456753c105664606fa89d0a0c4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Apr 2018 11:44:13 -0700 Subject: [PATCH 422/631] fixes #5380 (#5812) --- docs/api/constants.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/api/constants.rst b/docs/api/constants.rst index e225056a2..99ecc240a 100644 --- a/docs/api/constants.rst +++ b/docs/api/constants.rst @@ -3,3 +3,7 @@ .. automodule:: certbot.constants :members: + :exclude-members: SSL_DHPARAMS_SRC + +.. autodata:: SSL_DHPARAMS_SRC + :annotation: = '/path/to/certbot/ssl-dhparams.pem' From f5ad08047bbdf1ba230a6c7933eb7f923fe07938 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 3 Apr 2018 22:04:57 +0300 Subject: [PATCH 423/631] Fix comparison to check values (#5815) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 13fe493fc..3ba8bcb06 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -914,7 +914,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError("Nginx build doesn't support SNI") product_name, product_version = version_matches[0] - if product_name is not 'nginx': + if product_name != 'nginx': logger.warning("NGINX derivative %s is not officially supported by" " certbot", product_name) From bdaccb645b5af0bb1d14f2cc2ff3e40b7cde9b54 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 3 Apr 2018 12:14:23 -0700 Subject: [PATCH 424/631] Support quoted server names in Nginx (#5811) * Support quoted server names in Nginx * add unit test to check that we strip quotes * update configurator test --- certbot-nginx/certbot_nginx/parser.py | 2 +- certbot-nginx/certbot_nginx/tests/configurator_test.py | 2 +- .../tests/testdata/etc_nginx/sites-enabled/default | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 577e783fc..f06cd17a7 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -743,7 +743,7 @@ def _parse_server_raw(server): if addr.ssl: parsed_server['ssl'] = True elif directive[0] == 'server_name': - parsed_server['names'].update(directive[1:]) + parsed_server['names'].update(x.strip('"\'') for x in directive[1:]) elif _is_ssl_on_directive(directive): parsed_server['ssl'] = True apply_ssl_to_all_addrs = True diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 34abf2f0d..e88dcb8e0 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -639,7 +639,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual([[['server'], [['listen', 'myhost', 'default_server'], ['listen', 'otherhost', 'default_server'], - ['server_name', 'www.example.org'], + ['server_name', '"www.example.org"'], [['location', '/'], [['root', 'html'], ['index', 'index.html', 'index.htm']]]]], diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default index 4f67fa7d1..e167761d1 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/default @@ -1,7 +1,7 @@ server { listen myhost default_server; listen otherhost default_server; - server_name www.example.org; + server_name "www.example.org"; location / { root html; From 2c502e6f8b09898e958f96cff3ef47ba1b8afdc4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Apr 2018 14:04:51 -0700 Subject: [PATCH 425/631] document default is ACMEv2 (#5818) --- docs/using.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 7a25a5cc2..f478eb550 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -871,24 +871,16 @@ Example usage for DNS-01 (Cloudflare API v4) (for example purposes only, do not Changing the ACME Server ======================== -By default, Certbot uses Let's Encrypt's initial production server at -https://acme-v01.api.letsencrypt.org/. You can tell Certbot to use a +By default, Certbot uses Let's Encrypt's ACMEv2 production server at +https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a different CA by providing ``--server`` on the command line or in a :ref:`configuration file ` with the URL of the server's ACME directory. For example, if you would like to use Let's Encrypt's -new ACMEv2 server, you would add ``--server -https://acme-v02.api.letsencrypt.org/directory`` to the command line. +initial ACMEv1 server, you would add ``--server +https://acme-v01.api.letsencrypt.org/directory`` to the command line. Certbot will automatically select which version of the ACME protocol to use based on the contents served at the provided URL. -If you use ``--server`` to specify an ACME CA that implements a newer -version of the spec, you may be able to obtain a certificate for a -wildcard domain. Some CAs (such as Let's Encrypt) require that domain -validation for wildcard domains must be done through modifications to -DNS records which means that the dns-01_ challenge type must be used. To -see a list of Certbot plugins that support this challenge type and how -to use them, see plugins_. - Lock Files ========== From 9996730fb1a3eded162b54c6ca97731a145e9169 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 4 Apr 2018 00:05:37 +0300 Subject: [PATCH 426/631] If restart fails, try alternative restart command if available (#5500) * Use alternative restart command if available in distro overrides --- certbot-apache/certbot_apache/configurator.py | 19 ++++++++++++++++++- .../certbot_apache/override_centos.py | 1 + .../certbot_apache/override_gentoo.py | 1 + .../certbot_apache/tests/centos_test.py | 14 ++++++++++++++ .../certbot_apache/tests/gentoo_test.py | 8 ++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8b996c675..722e94e18 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -2000,10 +2000,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :raises .errors.MisconfigurationError: If reload fails """ + error = "" try: util.run_script(self.constant("restart_cmd")) except errors.SubprocessError as err: - raise errors.MisconfigurationError(str(err)) + logger.info("Unable to restart apache using %s", + self.constant("restart_cmd")) + alt_restart = self.constant("restart_cmd_alt") + if alt_restart: + logger.debug("Trying alternative restart command: %s", + alt_restart) + # There is an alternative restart command available + # This usually is "restart" verb while original is "graceful" + try: + util.run_script(self.constant( + "restart_cmd_alt")) + return + except errors.SubprocessError as secerr: + error = str(secerr) + else: + error = str(err) + raise errors.MisconfigurationError(error) def config_test(self): # pylint: disable=no-self-use """Check the configuration of Apache for errors. diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index db6cd6fba..6e75e361d 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -21,6 +21,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): version_cmd=['apachectl', '-v'], apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], + restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 92f1d4a20..165e44c96 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -21,6 +21,7 @@ class GentooConfigurator(configurator.ApacheConfigurator): version_cmd=['/usr/sbin/apache2', '-v'], apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], + restart_cmd_alt=['apache2ctl', 'restart'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index d7a2a2fd9..4ee8b5dcf 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -4,6 +4,8 @@ import unittest import mock +from certbot import errors + from certbot_apache import obj from certbot_apache import override_centos from certbot_apache.tests import util @@ -121,5 +123,17 @@ class MultipleVhostsTestCentOS(util.ApacheTest): self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) + @mock.patch("certbot_apache.configurator.util.run_script") + def test_alt_restart_works(self, mock_run_script): + mock_run_script.side_effect = [None, errors.SubprocessError, None] + self.config.restart() + self.assertEquals(mock_run_script.call_count, 3) + + @mock.patch("certbot_apache.configurator.util.run_script") + def test_alt_restart_errors(self, mock_run_script): + mock_run_script.side_effect = [None, + errors.SubprocessError, + errors.SubprocessError] + self.assertRaises(errors.MisconfigurationError, self.config.restart) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index cfbaffac7..d32551267 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -4,6 +4,8 @@ import unittest import mock +from certbot import errors + from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -123,5 +125,11 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.assertEquals(len(self.config.parser.modules), 4) self.assertTrue("mod_another.c" in self.config.parser.modules) + @mock.patch("certbot_apache.configurator.util.run_script") + def test_alt_restart_works(self, mock_run_script): + mock_run_script.side_effect = [None, errors.SubprocessError, None] + self.config.restart() + self.assertEquals(mock_run_script.call_count, 3) + if __name__ == "__main__": unittest.main() # pragma: no cover From b24d9dddc33c0e5d92c22e3a148ba061570ba70a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Apr 2018 17:55:12 -0700 Subject: [PATCH 427/631] Revert ACMEv2 default (#5819) * Revert "document default is ACMEv2 (#5818)" This reverts commit 2c502e6f8b09898e958f96cff3ef47ba1b8afdc4. * Revert "Update default to ACMEv2 server (#5722)" This reverts commit 4d706ac77e3eabb3134d0ea75ab76fd58b412bb6. --- certbot/constants.py | 2 +- docs/using.rst | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/certbot/constants.py b/certbot/constants.py index 9da5415d4..0d0ee8d3f 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -84,7 +84,7 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", - server="https://acme-v02.api.letsencrypt.org/directory", + server="https://acme-v01.api.letsencrypt.org/directory", # Plugins parsers configurator=None, diff --git a/docs/using.rst b/docs/using.rst index f478eb550..7a25a5cc2 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -871,16 +871,24 @@ Example usage for DNS-01 (Cloudflare API v4) (for example purposes only, do not Changing the ACME Server ======================== -By default, Certbot uses Let's Encrypt's ACMEv2 production server at -https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a +By default, Certbot uses Let's Encrypt's initial production server at +https://acme-v01.api.letsencrypt.org/. You can tell Certbot to use a different CA by providing ``--server`` on the command line or in a :ref:`configuration file ` with the URL of the server's ACME directory. For example, if you would like to use Let's Encrypt's -initial ACMEv1 server, you would add ``--server -https://acme-v01.api.letsencrypt.org/directory`` to the command line. +new ACMEv2 server, you would add ``--server +https://acme-v02.api.letsencrypt.org/directory`` to the command line. Certbot will automatically select which version of the ACME protocol to use based on the contents served at the provided URL. +If you use ``--server`` to specify an ACME CA that implements a newer +version of the spec, you may be able to obtain a certificate for a +wildcard domain. Some CAs (such as Let's Encrypt) require that domain +validation for wildcard domains must be done through modifications to +DNS records which means that the dns-01_ challenge type must be used. To +see a list of Certbot plugins that support this challenge type and how +to use them, see plugins_. + Lock Files ========== From b6afba0d64c7d0798afb8df609ca2ba4233b0816 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 4 Apr 2018 14:33:41 -0700 Subject: [PATCH 428/631] Include testdata (#5827) --- certbot-dns-google/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-dns-google/MANIFEST.in b/certbot-dns-google/MANIFEST.in index 18f018c08..c91330e38 100644 --- a/certbot-dns-google/MANIFEST.in +++ b/certbot-dns-google/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE.txt include README.rst recursive-include docs * +recursive-include certbot_dns_google/testdata * From 16b2539f72f1ac83cea731b7068b69fe8c822a8e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 4 Apr 2018 15:04:43 -0700 Subject: [PATCH 429/631] Release 0.23.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 4 +-- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 22 files changed, 76 insertions(+), 76 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 5660cf424..f54ed0e61 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f00b6d95d..f382ae7c7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 8c9745a6f..061b0c8f3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.2" +LE_AUTO_VERSION="0.23.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.2 \ - --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ - --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef -acme==0.22.2 \ - --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ - --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 -certbot-apache==0.22.2 \ - --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ - --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 -certbot-nginx==0.22.2 \ - --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ - --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b +certbot==0.23.0 \ + --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ + --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e +acme==0.23.0 \ + --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ + --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 +certbot-apache==0.23.0 \ + --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ + --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e +certbot-nginx==0.23.0 \ + --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ + --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 17abe65ec..a628c6530 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 956e37f79..e3bd566fa 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3493638a0..a81759271 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f136c7161..ed6247f5c 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index f2887d371..db894857c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a3ee12cf0..80ac7becc 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 6c25ed452..166b09a29 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index f872d7093..bca54f8d4 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 102ed48c2..b395a3327 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 2cbc29e6d..989ad29af 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 3f21c4dc5..ef04bab8e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 25023b307..1c019290d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0.dev0' +version = '0.23.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index ebc8d5343..53ba17e38 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.23.0.dev0' +__version__ = '0.23.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 399adc194..218e21a88 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,8 +107,8 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.22.2 (certbot; - darwin 10.13.3) Authenticator/XXX Installer/YYY + "". (default: CertbotACMEClient/0.23.0 (certbot; + darwin 10.13.4) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any diff --git a/letsencrypt-auto b/letsencrypt-auto index 8c9745a6f..061b0c8f3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.22.2" +LE_AUTO_VERSION="0.23.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.2 \ - --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ - --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef -acme==0.22.2 \ - --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ - --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 -certbot-apache==0.22.2 \ - --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ - --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 -certbot-nginx==0.22.2 \ - --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ - --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b +certbot==0.23.0 \ + --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ + --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e +acme==0.23.0 \ + --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ + --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 +certbot-apache==0.23.0 \ + --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ + --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e +certbot-nginx==0.23.0 \ + --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ + --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 3e1c4791c..620739670 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlqwWJwACgkQTRfJlc2X -dfIzmwgAghmc3W63/qpCtJdezYeGLJdu03LvKoWYc7dTNYj2+0P5qmAAgCvKNY34 -qYzXA1jfCOgILSzRNE5WY+rbgjcmxxsxH+luYm6Ik0909MaMQ0D3h+5cRFs/tTtd -5cX0gxL3RQQTBwpnwbAZibe7lhjs9pXBiob2ek67hVr+xEwem69BQMlOhtYJbOs1 -osccoKc4NqaKbrfgOjjtMaL8YoRPO9vJHS9rRr6hxRZlPsmvusAHAiCbIrbX4XKE -CgxJFnuHK+amtfRoZg/xCqIK3Z94yZXPezywsri/YvDteOIs+DZ2qG/StfUrNYFX -WYfFFFyld0xwQtb4Oi9u4mx4sPg7lw== -=jZDE +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlrFS/EACgkQTRfJlc2X +dfK+rQf8DcKY5bMi5eJnwwAlui6WIyWSrf1KAKt09tEGZSHQ1fcyCPrGVhk7VVDg +NJ1/XiYBquPW+7mYUcHrIRsiKYbTUcmVjyqP6tZd67IxRH9ToNqBzA6kq99T+IPd +iTGdczHMSPcxM6/Fa5PYMHXy2+ctTr/8+gnsxth9QfcM62Yd6ecfqIdoId3vk9Aw +UBMENZhUasIvgZDWuow+1XVZ/DAmdvj2Xl/E3sA9i2ArREJhkhVegtdrHkwSY+Hm +MKfZGqNVse6ZAF/8YdEVBum0OngMMs63DwucwFxmw5DqWtmnXm6awLNW/LQ/3R5L +xuKjcVaAT1h5TgIyRT6opH8JBKmLpg== +=Ouj4 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 07b313528..061b0c8f3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0.dev0" +LE_AUTO_VERSION="0.23.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.22.2 \ - --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \ - --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef -acme==0.22.2 \ - --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \ - --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4 -certbot-apache==0.22.2 \ - --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \ - --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759 -certbot-nginx==0.22.2 \ - --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \ - --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b +certbot==0.23.0 \ + --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ + --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e +acme==0.23.0 \ + --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ + --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 +certbot-apache==0.23.0 \ + --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ + --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e +certbot-nginx==0.23.0 \ + --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ + --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 088d5efa63c963d88dae21aa13b48cfaf0540917..738fa36e074721d2d7ea6d34b8dafc5937bbf2d1 100644 GIT binary patch literal 256 zcmV+b0ssCViCd%$o-hEy?<8CQwD8ADUBTS{idqO_gjrF5G{ zVRP?&R3bm%6xg|eMaq^mxiODsYes`VLVP%?>YI`_vpQkyqmump!{L*LC1Srlny=7{ zo3Y>fVHPDtRA0e84`j9mZXT*pgZ6cioA*IXpnwBfgM|DH+$uv9hdg9*BU!VA0HaH%xvk86LU? z;y5! literal 256 zcmV+b0ssDpWA~g+WJ85!lJf5bDl&;X`#p73O)Q@I8C^?;Js@g%A<#8V9``*mM3Q|k z+ORXI+b%w$fwmg4h&oGxh9VZz2M%;$=Jw|-*+*rr) Date: Wed, 4 Apr 2018 15:05:08 -0700 Subject: [PATCH 430/631] Bump version to 0.24.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index f54ed0e61..9cc616c36 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f382ae7c7..1ad3b7e43 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index a628c6530..93495d22e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index e3bd566fa..7f7eab517 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a81759271..6a411dedf 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index ed6247f5c..0195fb0da 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index db894857c..e004ef92c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 80ac7becc..75f588825 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 166b09a29..d8a8025ef 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index bca54f8d4..cebe69b42 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index b395a3327..a86d12819 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 989ad29af..82f813197 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index ef04bab8e..b584b872f 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 1c019290d..77ea4170a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.23.0' +version = '0.24.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 53ba17e38..9dbab3b70 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.23.0' +__version__ = '0.24.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 061b0c8f3..89072ec88 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0" +LE_AUTO_VERSION="0.24.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From db938dcc0eb9c6a4341ea89dad5c791771cb7e28 Mon Sep 17 00:00:00 2001 From: Peter Linss Date: Fri, 6 Apr 2018 05:39:35 +0900 Subject: [PATCH 431/631] Update messages.py (#5759) Add wildcard field to AuthorizationResource --- acme/acme/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 23cd66c63..a69b3bbc4 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -435,6 +435,7 @@ class Authorization(ResourceBody): # be absent'... then acme-spec gives example with 'expires' # present... That's confusing! expires = fields.RFC3339Field('expires', omitempty=True) + wildcard = jose.Field('wildcard', omitempty=True) @challenges.decoder def challenges(value): # pylint: disable=missing-docstring,no-self-argument From 58626c3197096969e98e687eb97e3e622c6d0ce1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Apr 2018 16:58:58 -0700 Subject: [PATCH 432/631] Double max_rounds (#5842) --- certbot/auth_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 9d7c75f57..caf112c61 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -189,7 +189,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges(self, aauthzrs, chall_update, - best_effort, min_sleep=3, max_rounds=15): + best_effort, min_sleep=3, max_rounds=30): """Wait for all challenge results to be determined.""" indices_to_check = set(chall_update.keys()) comp_indices = set() From 4a8e35289c967ccf47c7438954d83613b4a751c7 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 11 Apr 2018 18:54:55 +0300 Subject: [PATCH 433/631] PluginStorage to store variables between invocations. (#5468) The base class for Installer plugins `certbot.plugins.common.Installer` now provides functionality of `PluginStorage` to all installer plugins. This allows a plugin to save and retrieve variables in between of invocations. The on disk storage is basically a JSON file at `config_dir`/`.pluginstorage.json`, usually `/etc/letsencrypt/.pluginstorage.json`. The JSON structure is automatically namespaced using the internal plugin name as a namespace key. Because the actual storage is JSON, the supported data types are: dict, list, tuple, str, unicode, int, long, float, boolean and nonetype. To add a variable from inside the plugin class: `self.storage.put("my_variable_name", my_var)` To fetch a variable from inside the plugin class: `my_var = self.storage.fetch("my_variable_key")` The storage state isn't written on disk automatically, but needs to be called: `self.storage.save()` * Plugin storage implementation * Added config_dir to existing test mocks * PluginStorage test cases * Saner handling of bad config_dir paths * Storage moved to Installer and not initialized on plugin __init__ * Finetuning and renaming --- certbot/errors.py | 4 ++ certbot/plugins/common.py | 4 +- certbot/plugins/storage.py | 117 ++++++++++++++++++++++++++++++++ certbot/plugins/storage_test.py | 117 ++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 certbot/plugins/storage.py create mode 100644 certbot/plugins/storage_test.py diff --git a/certbot/errors.py b/certbot/errors.py index e9c4a0806..48aebc267 100644 --- a/certbot/errors.py +++ b/certbot/errors.py @@ -87,6 +87,10 @@ class NotSupportedError(PluginError): """Certbot Plugin function not supported error.""" +class PluginStorageError(PluginError): + """Certbot Plugin Storage error.""" + + class StandaloneBindError(Error): """Standalone plugin bind error.""" diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index c281534ca..147d9e21a 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -18,6 +18,8 @@ from certbot import interfaces from certbot import reverter from certbot import util +from certbot.plugins.storage import PluginStorage + logger = logging.getLogger(__name__) @@ -99,7 +101,6 @@ class Plugin(object): def conf(self, var): """Find a configuration value for variable ``var``.""" return getattr(self.config, self.dest(var)) -# other class Installer(Plugin): @@ -110,6 +111,7 @@ class Installer(Plugin): """ def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) + self.storage = PluginStorage(self.config, self.name) self.reverter = reverter.Reverter(self.config) def add_to_checkpoint(self, save_files, save_notes, temporary=False): diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py new file mode 100644 index 000000000..a0c3f8564 --- /dev/null +++ b/certbot/plugins/storage.py @@ -0,0 +1,117 @@ +"""Plugin storage class.""" +import json +import logging +import os + +from certbot import errors + +logger = logging.getLogger(__name__) + +class PluginStorage(object): + """Class implementing storage functionality for plugins""" + + def __init__(self, config, classkey): + """Initializes PluginStorage object storing required configuration + options. + + :param .configuration.NamespaceConfig config: Configuration object + :param str classkey: class name to use as root key in storage file + + """ + + self._config = config + self._classkey = classkey + self._initialized = False + self._data = None + self._storagepath = None + + def _initialize_storage(self): + """Initializes PluginStorage data and reads current state from the disk + if the storage json exists.""" + + self._storagepath = os.path.join(self._config.config_dir, ".pluginstorage.json") + self._load() + self._initialized = True + + def _load(self): + """Reads PluginStorage content from the disk to a dict structure + + :raises .errors.PluginStorageError: when unable to open or read the file + """ + data = dict() + filedata = "" + try: + with open(self._storagepath, 'r') as fh: + filedata = fh.read() + except IOError as e: + errmsg = "Could not read PluginStorage data file: {0} : {1}".format( + self._storagepath, str(e)) + if os.path.isfile(self._storagepath): + # Only error out if file exists, but cannot be read + logger.error(errmsg) + raise errors.PluginStorageError(errmsg) + try: + data = json.loads(filedata) + except ValueError: + if not filedata: + logger.debug("Plugin storage file %s was empty, no values loaded", + self._storagepath) + else: + errmsg = "PluginStorage file {0} is corrupted.".format( + self._storagepath) + logger.error(errmsg) + raise errors.PluginStorageError(errmsg) + self._data = data + + def save(self): + """Saves PluginStorage content to disk + + :raises .errors.PluginStorageError: when unable to serialize the data + or write it to the filesystem + """ + if not self._initialized: + errmsg = "Unable to save, no values have been added to PluginStorage." + logger.error(errmsg) + raise errors.PluginStorageError(errmsg) + + try: + serialized = json.dumps(self._data) + except TypeError as e: + errmsg = "Could not serialize PluginStorage data: {0}".format( + str(e)) + logger.error(errmsg) + raise errors.PluginStorageError(errmsg) + try: + with os.fdopen(os.open(self._storagepath, + os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fh: + fh.write(serialized) + except IOError as e: + errmsg = "Could not write PluginStorage data to file {0} : {1}".format( + self._storagepath, str(e)) + logger.error(errmsg) + raise errors.PluginStorageError(errmsg) + + def put(self, key, value): + """Put configuration value to PluginStorage + + :param str key: Key to store the value to + :param value: Data to store + """ + if not self._initialized: + self._initialize_storage() + + if not self._classkey in self._data.keys(): + self._data[self._classkey] = dict() + self._data[self._classkey][key] = value + + def fetch(self, key): + """Get configuration value from PluginStorage + + :param str key: Key to get value from the storage + + :raises KeyError: If the key doesn't exist in the storage + """ + if not self._initialized: + self._initialize_storage() + + return self._data[self._classkey][key] diff --git a/certbot/plugins/storage_test.py b/certbot/plugins/storage_test.py new file mode 100644 index 000000000..8d96f400c --- /dev/null +++ b/certbot/plugins/storage_test.py @@ -0,0 +1,117 @@ +"""Tests for certbot.plugins.storage.PluginStorage""" +import json +import mock +import os +import unittest + +from certbot import errors + +from certbot.plugins import common +from certbot.tests import util as test_util + +class PluginStorageTest(test_util.ConfigTestCase): + """Test for certbot.plugins.storage.PluginStorage""" + + def setUp(self): + super(PluginStorageTest, self).setUp() + self.plugin_cls = common.Installer + os.mkdir(self.config.config_dir) + with mock.patch("certbot.reverter.util"): + self.plugin = self.plugin_cls(config=self.config, name="mockplugin") + + def test_load_errors_cant_read(self): + with open(os.path.join(self.config.config_dir, + ".pluginstorage.json"), "w") as fh: + fh.write("dummy") + # When unable to read file that exists + mock_open = mock.mock_open() + mock_open.side_effect = IOError + self.plugin.storage.storagepath = os.path.join(self.config.config_dir, + ".pluginstorage.json") + with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch('os.path.isfile', return_value=True): + with mock.patch("certbot.reverter.util"): + self.assertRaises(errors.PluginStorageError, + self.plugin.storage._load) # pylint: disable=protected-access + + def test_load_errors_empty(self): + with open(os.path.join(self.config.config_dir, ".pluginstorage.json"), "w") as fh: + fh.write('') + with mock.patch("certbot.plugins.storage.logger.debug") as mock_log: + # Should not error out but write a debug log line instead + with mock.patch("certbot.reverter.util"): + nocontent = self.plugin_cls(self.config, "mockplugin") + self.assertRaises(KeyError, + nocontent.storage.fetch, "value") + self.assertTrue(mock_log.called) + self.assertTrue("no values loaded" in mock_log.call_args[0][0]) + + def test_load_errors_corrupted(self): + with open(os.path.join(self.config.config_dir, + ".pluginstorage.json"), "w") as fh: + fh.write('invalid json') + with mock.patch("certbot.plugins.storage.logger.error") as mock_log: + with mock.patch("certbot.reverter.util"): + corrupted = self.plugin_cls(self.config, "mockplugin") + self.assertRaises(errors.PluginError, + corrupted.storage.fetch, + "value") + self.assertTrue("is corrupted" in mock_log.call_args[0][0]) + + def test_save_errors_cant_serialize(self): + with mock.patch("certbot.plugins.storage.logger.error") as mock_log: + # Set data as something that can't be serialized + self.plugin.storage._initialized = True # pylint: disable=protected-access + self.plugin.storage.storagepath = "/tmp/whatever" + self.plugin.storage._data = self.plugin_cls # pylint: disable=protected-access + self.assertRaises(errors.PluginStorageError, + self.plugin.storage.save) + self.assertTrue("Could not serialize" in mock_log.call_args[0][0]) + + def test_save_errors_unable_to_write_file(self): + mock_open = mock.mock_open() + mock_open.side_effect = IOError + with mock.patch("os.open", mock_open): + with mock.patch("certbot.plugins.storage.logger.error") as mock_log: + self.plugin.storage._data = {"valid": "data"} # pylint: disable=protected-access + self.plugin.storage._initialized = True # pylint: disable=protected-access + self.assertRaises(errors.PluginStorageError, + self.plugin.storage.save) + self.assertTrue("Could not write" in mock_log.call_args[0][0]) + + def test_save_uninitialized(self): + with mock.patch("certbot.reverter.util"): + self.assertRaises(errors.PluginStorageError, + self.plugin_cls(self.config, "x").storage.save) + + def test_namespace_isolation(self): + with mock.patch("certbot.reverter.util"): + plugin1 = self.plugin_cls(self.config, "first") + plugin2 = self.plugin_cls(self.config, "second") + plugin1.storage.put("first_key", "first_value") + self.assertRaises(KeyError, + plugin2.storage.fetch, "first_key") + self.assertRaises(KeyError, + plugin2.storage.fetch, "first") + self.assertEqual(plugin1.storage.fetch("first_key"), "first_value") + + + def test_saved_state(self): + self.plugin.storage.put("testkey", "testvalue") + # Write to disk + self.plugin.storage.save() + with mock.patch("certbot.reverter.util"): + another = self.plugin_cls(self.config, "mockplugin") + self.assertEqual(another.storage.fetch("testkey"), "testvalue") + + with open(os.path.join(self.config.config_dir, + ".pluginstorage.json"), 'r') as fh: + psdata = fh.read() + psjson = json.loads(psdata) + self.assertTrue("mockplugin" in psjson.keys()) + self.assertEqual(len(psjson), 1) + self.assertEqual(psjson["mockplugin"]["testkey"], "testvalue") + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From e7db97df87f317ca959fe92b454d83997195563c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Apr 2018 11:16:12 -0700 Subject: [PATCH 434/631] Update CHANGELOG for 0.23.0 (#5822) * Update CHANGELOG for 0.23.0 * correct date --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1906858dc..89c7c41a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.23.0 - 2018-04-04 + +### Added + +* Support for OpenResty was added to the Nginx plugin. + +### Changed + +* The timestamps in Certbot's logfiles now use the system's local time zone + rather than UTC. +* Certbot's DNS plugins that use Lexicon now rely on Lexicon>=2.2.1 to be able + to create and delete multiple TXT records on a single domain. +* certbot-dns-google's test suite now works without an internet connection. + +### Fixed + +* Removed a small window that if during which an error occurred, Certbot + wouldn't clean up performed challenges. +* The parameters `default` and `ipv6only` are now removed from `listen` + directives when creating a new server block in the Nginx plugin. +* `server_name` directives enclosed in quotation marks in Nginx are now properly + supported. +* Resolved an issue preventing the Apache plugin from starting Apache when it's + not currently running on RHEL and Gentoo based systems. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* certbot +* certbot-apache +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-google +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-rfc2136 +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/50?closed=1 + ## 0.22.2 - 2018-03-19 ### Fixed From 88ceaa38d5f21bb81c4852fe1eb75ba049dfdf3f Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 11 Apr 2018 14:36:53 -0700 Subject: [PATCH 435/631] Bump certbot's acme depenency to 0.22.1 (#5826) --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3667a6976..ba521ed2a 100644 --- a/setup.py +++ b/setup.py @@ -34,9 +34,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - # Remember to update local-oldest-requirements.txt when changing the - # minimum acme version. - 'acme>0.21.1', + 'acme>=0.22.1', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From 6b29d159a2f221c3437770bdb43924ee6f953c4b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Apr 2018 16:14:55 -0700 Subject: [PATCH 436/631] use older boulder version (#5852) --- tests/boulder-fetch.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index fc9cbaae7..53485ffc0 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -7,10 +7,11 @@ set -xe export GOPATH=${GOPATH:-$HOME/gopath} BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder} if [ ! -d ${BOULDERPATH} ]; then - git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH} + git clone https://github.com/letsencrypt/boulder ${BOULDERPATH} fi cd ${BOULDERPATH} +git checkout fa5c9176655d9fa8dfca188de08bd5373aca422f FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) From 2d3159848448e47ccae438e8bc35a0257762549c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 12 Apr 2018 15:47:39 -0700 Subject: [PATCH 437/631] Get mypy tox env running in the current setup (#5861) * get mypy tox env running in the current setup * use any python3 with mypy * pin mypy dependencies --- .gitignore | 1 + setup.py | 5 +++++ tools/dev_constraints.txt | 3 +++ tox.ini | 4 ++-- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4dff20caf..e744a82a2 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ tests/letstest/venv/ # pytest cache .cache +.mypy_cache/ # docker files .docker diff --git a/setup.py b/setup.py index ba521ed2a..e674871a8 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,10 @@ dev_extras = [ 'wheel', ] +dev3_extras = [ + 'mypy', +] + docs_extras = [ 'repoze.sphinx.autointerface', # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 @@ -110,6 +114,7 @@ setup( install_requires=install_requires, extras_require={ 'dev': dev_extras, + 'dev3': dev3_extras, 'docs': docs_extras, }, diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d02204215..df13cdbef 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -30,6 +30,7 @@ josepy==1.0.1 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 +mypy==0.580 ndg-httpsclient==0.3.2 oauth2client==2.0.0 pathlib2==2.3.0 @@ -66,6 +67,8 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 +typed-ast==1.1.0 +typing==3.6.4 uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tox.ini b/tox.ini index 049220bbb..dce5911ed 100644 --- a/tox.ini +++ b/tox.ini @@ -136,9 +136,9 @@ commands = pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] -basepython = python3.4 +basepython = python3 commands = - {[base]pip_install} mypy + {[base]pip_install} .[dev3] {[base]install_packages} mypy --py2 --ignore-missing-imports {[base]source_paths} From c443db0618310c1f743b5d9bb0399d407212d295 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Thu, 12 Apr 2018 16:33:10 -0700 Subject: [PATCH 438/631] Update constants.py On FreeBSD or MacOS, "certbot --nginx" fails. The reason is, at these op. systems, nginx directory is different than linux. --- certbot-nginx/certbot_nginx/constants.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 3f263fea3..8422ab3cd 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -1,9 +1,14 @@ """nginx plugin constants.""" import pkg_resources +import platform - +if(platform.system() == ('FreeBSD' or 'Darwin')): + server_root_tmp = "/usr/local/etc/nginx" +else: + server_root_tmp = "/etc/nginx" + CLI_DEFAULTS = dict( - server_root="/etc/nginx", + server_root=server_root_tmp ctl="nginx", ) """CLI defaults.""" From a708504d5b6ad2a00311f163b3ce29bf5c27f51f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Apr 2018 18:28:00 -0700 Subject: [PATCH 439/631] remove test metaclass (#5863) --- .../certbot_apache/tests/http_01_test.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 9ed4ee509..f120674c7 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -16,30 +16,9 @@ from certbot_apache.tests import util NUM_ACHALLS = 3 -class ApacheHttp01TestMeta(type): - """Generates parmeterized tests for testing perform.""" - def __new__(mcs, name, bases, class_dict): - - def _gen_test(num_achalls, minor_version): - def _test(self): - achalls = self.achalls[:num_achalls] - vhosts = self.vhosts[:num_achalls] - self.config.version = (2, minor_version) - self.common_perform_test(achalls, vhosts) - return _test - - for i in range(1, NUM_ACHALLS + 1): - for j in (2, 4): - test_name = "test_perform_{0}_{1}".format(i, j) - class_dict[test_name] = _gen_test(i, j) - return type.__new__(mcs, name, bases, class_dict) - - class ApacheHttp01Test(util.ApacheTest): """Test for certbot_apache.http_01.ApacheHttp01.""" - __metaclass__ = ApacheHttp01TestMeta - def setUp(self, *args, **kwargs): super(ApacheHttp01Test, self).setUp(*args, **kwargs) @@ -137,6 +116,31 @@ class ApacheHttp01Test(util.ApacheTest): self.config.config.http01_port = 12345 self.assertRaises(errors.PluginError, self.http.perform) + def test_perform_1_achall_22(self): + self.combinations_perform_test(num_achalls=1, minor_version=2) + + def test_perform_1_achall_24(self): + self.combinations_perform_test(num_achalls=1, minor_version=4) + + def test_perform_2_achall_22(self): + self.combinations_perform_test(num_achalls=2, minor_version=2) + + def test_perform_2_achall_24(self): + self.combinations_perform_test(num_achalls=2, minor_version=4) + + def test_perform_3_achall_22(self): + self.combinations_perform_test(num_achalls=3, minor_version=2) + + def test_perform_3_achall_24(self): + self.combinations_perform_test(num_achalls=3, minor_version=4) + + def combinations_perform_test(self, num_achalls, minor_version): + """Test perform with the given achall count and Apache version.""" + achalls = self.achalls[:num_achalls] + vhosts = self.vhosts[:num_achalls] + self.config.version = (2, minor_version) + self.common_perform_test(achalls, vhosts) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 6253acf335da6f462376e09adbeb89cbd58f90e9 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 12 Apr 2018 18:53:07 -0700 Subject: [PATCH 440/631] Get mypy running with clean output (#5864) Fixes #5849. * extract mypy flags into mypy.ini file * Get mypy running with clean output --- .../certbot_dns_digitalocean/dns_digitalocean_test.py | 5 +++-- certbot/crypto_util.py | 2 +- certbot/tests/cert_manager_test.py | 5 +++-- certbot/util.py | 2 +- mypy.ini | 3 +++ tox.ini | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 mypy.ini diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py index 3b8edce64..0e2043f50 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -50,7 +50,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic class DigitalOceanClientTest(unittest.TestCase): - id = 1 + + id_num = 1 record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" @@ -70,7 +71,7 @@ class DigitalOceanClientTest(unittest.TestCase): domain_mock = mock.MagicMock() domain_mock.name = DOMAIN - domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}} + domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id_num}} self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index bd4e7fcfc..756bd7565 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -13,7 +13,7 @@ import pyrfc3339 import six import zope.component from cryptography.hazmat.backends import default_backend -from cryptography import x509 +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 98ff163cd..675b936b9 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -216,11 +216,12 @@ class CertificatesTest(BaseCertManagerTest): cert.is_test_cert = False parsed_certs = [cert] + mock_config = mock.MagicMock(certname=None, lineagename=None) + # pylint: disable=protected-access + # pylint: disable=protected-access get_report = lambda: cert_manager._report_human_readable(mock_config, parsed_certs) - mock_config = mock.MagicMock(certname=None, lineagename=None) - # pylint: disable=protected-access out = get_report() self.assertTrue("INVALID: EXPIRED" in out) diff --git a/certbot/util.py b/certbot/util.py index b3973d96b..55acd624f 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -54,7 +54,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() +_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] def run_script(params, log=logger.error): diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..aac60a2ad --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 2.7 +ignore_missing_imports = True diff --git a/tox.ini b/tox.ini index dce5911ed..f33802d98 100644 --- a/tox.ini +++ b/tox.ini @@ -140,7 +140,7 @@ basepython = python3 commands = {[base]pip_install} .[dev3] {[base]install_packages} - mypy --py2 --ignore-missing-imports {[base]source_paths} + mypy {[base]source_paths} [testenv:apacheconftest] #basepython = python2.7 From e0a5b1229f5d495bf147b048b02fae7bc2e231b2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Apr 2018 19:02:40 -0700 Subject: [PATCH 441/631] help clarify version number (#5865) (Hopefully) helps make it clearer that that 22 and 24 corresponds to Apache 2.2 and 2.4. --- .../certbot_apache/tests/http_01_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index f120674c7..dc1ca34d6 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -50,7 +50,7 @@ class ApacheHttp01Test(util.ApacheTest): self.assertFalse(self.http.perform()) @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_enable_modules_22(self, mock_enmod): + def test_enable_modules_apache_2_2(self, mock_enmod): self.config.version = (2, 2) self.config.parser.modules.remove("authz_host_module") self.config.parser.modules.remove("mod_authz_host.c") @@ -59,7 +59,7 @@ class ApacheHttp01Test(util.ApacheTest): self.assertEqual(enmod_calls[0][0][0], "authz_host") @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_enable_modules_24(self, mock_enmod): + def test_enable_modules_apache_2_4(self, mock_enmod): self.config.parser.modules.remove("authz_core_module") self.config.parser.modules.remove("mod_authz_core.c") @@ -116,22 +116,22 @@ class ApacheHttp01Test(util.ApacheTest): self.config.config.http01_port = 12345 self.assertRaises(errors.PluginError, self.http.perform) - def test_perform_1_achall_22(self): + def test_perform_1_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=1, minor_version=2) - def test_perform_1_achall_24(self): + def test_perform_1_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=1, minor_version=4) - def test_perform_2_achall_22(self): + def test_perform_2_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=2, minor_version=2) - def test_perform_2_achall_24(self): + def test_perform_2_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=2, minor_version=4) - def test_perform_3_achall_22(self): + def test_perform_3_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=3, minor_version=2) - def test_perform_3_achall_24(self): + def test_perform_3_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=3, minor_version=4) def combinations_perform_test(self, num_achalls, minor_version): From 523cdc578d90ee65fa4dc0e74a53dd89a87152ea Mon Sep 17 00:00:00 2001 From: Axel Date: Fri, 13 Apr 2018 18:17:08 +0200 Subject: [PATCH 442/631] Add port option for rfc2136 plugin (#5844) --- certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py | 7 +++++-- .../certbot_dns_rfc2136/dns_rfc2136.py | 12 ++++++++---- .../certbot_dns_rfc2136/dns_rfc2136_test.py | 11 ++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py index 0f97869e2..12b360959 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py @@ -21,8 +21,9 @@ Credentials ----------- Use of this plugin requires a configuration file containing the target DNS -server that supports RFC 2136 Dynamic Updates, the name of the TSIG key, the -TSIG key secret itself and the algorithm used if it's different to HMAC-MD5. +server and optional port that supports RFC 2136 Dynamic Updates, the name +of the TSIG key, the TSIG key secret itself and the algorithm used if it's +different to HMAC-MD5. .. code-block:: ini :name: credentials.ini @@ -30,6 +31,8 @@ TSIG key secret itself and the algorithm used if it's different to HMAC-MD5. # Target DNS server dns_rfc2136_server = 192.0.2.1 + # Target DNS port + dns_rfc2136_port = 53 # TSIG key name dns_rfc2136_name = keyname. # TSIG key secret diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py index 127773469..b8c01cdd3 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py @@ -36,6 +36,8 @@ class Authenticator(dns_common.DNSAuthenticator): 'HMAC-SHA512': dns.tsig.HMAC_SHA512 } + PORT = 53 + description = 'Obtain certificates using a DNS TXT record (if you are using BIND for DNS).' ttl = 120 @@ -78,6 +80,7 @@ class Authenticator(dns_common.DNSAuthenticator): def _get_rfc2136_client(self): return _RFC2136Client(self.credentials.conf('server'), + int(self.credentials.conf('port') or self.PORT), self.credentials.conf('name'), self.credentials.conf('secret'), self.ALGORITHMS.get(self.credentials.conf('algorithm'), @@ -88,8 +91,9 @@ class _RFC2136Client(object): """ Encapsulates all communication with the target DNS server. """ - def __init__(self, server, key_name, key_secret, key_algorithm): + def __init__(self, server, port, key_name, key_secret, key_algorithm): self.server = server + self.port = port self.keyring = dns.tsigkeyring.from_text({ key_name: key_secret }) @@ -118,7 +122,7 @@ class _RFC2136Client(object): update.add(rel, record_ttl, dns.rdatatype.TXT, record_content) try: - response = dns.query.tcp(update, self.server) + response = dns.query.tcp(update, self.server, port=self.port) except Exception as e: raise errors.PluginError('Encountered error adding TXT record: {0}' .format(e)) @@ -153,7 +157,7 @@ class _RFC2136Client(object): update.delete(rel, dns.rdatatype.TXT, record_content) try: - response = dns.query.tcp(update, self.server) + response = dns.query.tcp(update, self.server, port=self.port) except Exception as e: raise errors.PluginError('Encountered error deleting TXT record: {0}' .format(e)) @@ -202,7 +206,7 @@ class _RFC2136Client(object): request.flags ^= dns.flags.RD try: - response = dns.query.udp(request, self.server) + response = dns.query.udp(request, self.server, port=self.port) rcode = response.rcode() # Authoritative Answer bit should be set diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 8a5166330..89ce3d93e 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -14,6 +14,7 @@ from certbot.plugins.dns_test_common import DOMAIN from certbot.tests import util as test_util SERVER = '192.0.2.1' +PORT = 53 NAME = 'a-tsig-key.' SECRET = 'SSB3b25kZXIgd2hvIHdpbGwgYm90aGVyIHRvIGRlY29kZSB0aGlzIHRleHQK' VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret": SECRET} @@ -74,7 +75,7 @@ class RFC2136ClientTest(unittest.TestCase): def setUp(self): from certbot_dns_rfc2136.dns_rfc2136 import _RFC2136Client - self.rfc2136_client = _RFC2136Client(SERVER, NAME, SECRET, dns.tsig.HMAC_MD5) + self.rfc2136_client = _RFC2136Client(SERVER, PORT, NAME, SECRET, dns.tsig.HMAC_MD5) @mock.patch("dns.query.tcp") def test_add_txt_record(self, query_mock): @@ -84,7 +85,7 @@ class RFC2136ClientTest(unittest.TestCase): self.rfc2136_client.add_txt_record("bar", "baz", 42) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue("bar. 42 IN TXT \"baz\"" in str(query_mock.call_args[0][0])) @mock.patch("dns.query.tcp") @@ -117,7 +118,7 @@ class RFC2136ClientTest(unittest.TestCase): self.rfc2136_client.del_txt_record("bar", "baz") - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue("bar. 0 NONE TXT \"baz\"" in str(query_mock.call_args[0][0])) @mock.patch("dns.query.tcp") @@ -169,7 +170,7 @@ class RFC2136ClientTest(unittest.TestCase): # _query_soa | pylint: disable=protected-access result = self.rfc2136_client._query_soa(DOMAIN) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue(result == True) @mock.patch("dns.query.udp") @@ -179,7 +180,7 @@ class RFC2136ClientTest(unittest.TestCase): # _query_soa | pylint: disable=protected-access result = self.rfc2136_client._query_soa(DOMAIN) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue(result == False) @mock.patch("dns.query.udp") From 590ec375ec89cce2a10e984c86fbf23d4533108f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 13 Apr 2018 16:10:58 -0700 Subject: [PATCH 443/631] Get mypy running in travis for easier review (#5875) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9ec2f724b..370137f68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ matrix: addons: - python: "2.7" env: TOXENV=lint + - python: "3.5" + env: TOXENV=mypy - python: "2.7" env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required From b39507c5af53bd8511dec554a653cd7a4d3e8267 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Tue, 17 Apr 2018 09:09:27 -0700 Subject: [PATCH 444/631] Update constants.py Fixed comma. --- certbot-nginx/certbot_nginx/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 8422ab3cd..72fc5d4de 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -8,7 +8,7 @@ else: server_root_tmp = "/etc/nginx" CLI_DEFAULTS = dict( - server_root=server_root_tmp + server_root=server_root_tmp, ctl="nginx", ) """CLI defaults.""" From 6dc8b66760cc7ab120275bca4117c12f6c25c19a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Apr 2018 11:27:45 -0700 Subject: [PATCH 445/631] Add _choose_lineagename and update docs (#5650) --- certbot/client.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 2992c0cec..60e1ca37e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -338,9 +338,10 @@ class Client(object): authenticator and installer, and then create a new renewable lineage containing it. - :param list domains: Domains to request. - :param plugins: A PluginsFactory object. - :param str certname: Name of new cert + :param domains: domains to request a certificate for + :type domains: `list` of `str` + :param certname: requested name of lineage + :type certname: `str` or `None` :returns: A new :class:`certbot.storage.RenewableCert` instance referred to the enrolled cert lineage, False if the cert could not @@ -355,13 +356,7 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - if certname: - new_name = certname - elif util.is_wildcard_domain(domains[0]): - # Don't make files and directories starting with *. - new_name = domains[0][2:] - else: - new_name = domains[0] + new_name = self._choose_lineagename(domains, certname) if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", @@ -373,6 +368,26 @@ class Client(object): key.pem, chain, self.config) + def _choose_lineagename(self, domains, certname): + """Chooses a name for the new lineage. + + :param domains: domains in certificate request + :type domains: `list` of `str` + :param certname: requested name of lineage + :type certname: `str` or `None` + + :returns: lineage name that should be used + :rtype: str + + """ + if certname: + return certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + return domains[0][2:] + else: + return domains[0] + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. From 5c7fc07ccfca34e4ecfc4be4d124e921fdc8440e Mon Sep 17 00:00:00 2001 From: Kiel C Date: Tue, 17 Apr 2018 13:52:39 -0700 Subject: [PATCH 446/631] Adjust file paths message from Warning to Info. (#5743) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index 60e1ca37e..cf2afa2e5 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -352,7 +352,7 @@ class Client(object): if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): - logger.warning( + logger.info( "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") From a9e01ade4c2d42d48e6e9a77d975a28d7f0533b9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Apr 2018 17:17:15 -0700 Subject: [PATCH 447/631] Revert "use older boulder version (#5852)" (#5855) This reverts commit 6b29d159a2f221c3437770bdb43924ee6f953c4b. --- tests/boulder-fetch.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 53485ffc0..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -7,11 +7,10 @@ set -xe export GOPATH=${GOPATH:-$HOME/gopath} BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder} if [ ! -d ${BOULDERPATH} ]; then - git clone https://github.com/letsencrypt/boulder ${BOULDERPATH} + git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH} fi cd ${BOULDERPATH} -git checkout fa5c9176655d9fa8dfca188de08bd5373aca422f FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) From 261d063b10fdaa9a4e4d23763af0769965996a1b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 18 Apr 2018 10:02:31 -0700 Subject: [PATCH 448/631] Revert fix-macos-pytest (#5853) * Revert "Fix pytest on macOS in Travis (#5360)" This reverts commit 5388842e5b3868e29caf545fb771a23e7fce4143. * remove oldest passenv --- pytest.ini | 2 -- tools/install_and_test.sh | 2 +- tools/pytest.sh | 15 --------------- tox.cover.sh | 3 +-- tox.ini | 16 ---------------- 5 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 pytest.ini delete mode 100755 tools/pytest.sh diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b64550cb7..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --quiet diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index f0385470b..59832cbc3 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -19,5 +19,5 @@ for requirement in "$@" ; do if [ $pkg = "." ]; then pkg="certbot" fi - "$(dirname $0)/pytest.sh" --pyargs $pkg + pytest --numprocesses auto --quiet --pyargs $pkg done diff --git a/tools/pytest.sh b/tools/pytest.sh deleted file mode 100755 index 8e3619d5d..000000000 --- a/tools/pytest.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Runs pytest with the provided arguments, adding --numprocesses to the command -# line. This argument is set to "auto" if the environmnent variable TRAVIS is -# not set, otherwise, it is set to 2. This works around -# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis -# environnment provides two cores. See -# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. - -if ${TRAVIS:-false}; then - NUMPROCESSES="2" -else - NUMPROCESSES="auto" -fi - -pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index bc0e5a8bf..2b5a3cf19 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,8 +51,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest="$(dirname $0)/tools/pytest.sh" - "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index f33802d98..38d4e6ae1 100644 --- a/tox.ini +++ b/tox.ini @@ -60,8 +60,6 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 -passenv = - TRAVIS [testenv:py27-oldest] commands = @@ -69,40 +67,30 @@ commands = setenv = {[testenv]setenv} CERTBOT_OLDEST=1 -passenv = - {[testenv]passenv} [testenv:py27-acme-oldest] commands = {[base]install_and_test} acme[dev] setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-apache-oldest] commands = {[base]install_and_test} certbot-apache setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-certbot-oldest] commands = {[base]install_and_test} .[dev] setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-dns-oldest] commands = {[base]install_and_test} {[base]dns_packages} setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-nginx-oldest] commands = @@ -110,8 +98,6 @@ commands = python tests/lock_test.py setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27_install] basepython = python2.7 @@ -123,8 +109,6 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh -passenv = - {[testenv]passenv} [testenv:lint] basepython = python2.7 From a024aaf59d04e7f62b2a8e3d6a67f4db75c76345 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 18 Apr 2018 21:08:30 +0300 Subject: [PATCH 449/631] Enhance verb (#5596) * Add the cli parameters * Tests and error messages * Requested fixes * Only handle SSL vhosts * Interactive cert-name selection if not defined on CLI * Address PR comments * Address review comments * Added tests and addressed review comments * Move cert manager tests to correct file * Add display ops tests * Use display util constants instead of hardcoded values in tests --- certbot-apache/certbot_apache/configurator.py | 36 ++++-- .../certbot_apache/tests/configurator_test.py | 38 +++++- .../certbot_apache/tests/debian_test.py | 4 + certbot-apache/certbot_apache/tls_sni_01.py | 3 +- certbot/cert_manager.py | 32 ++--- certbot/cli.py | 28 +++-- certbot/client.py | 15 ++- certbot/display/ops.py | 32 ++++- certbot/main.py | 53 ++++++++- certbot/plugins/selection.py | 14 ++- certbot/tests/cert_manager_test.py | 98 ++++++++++++++++ certbot/tests/client_test.py | 23 ++++ certbot/tests/display/ops_test.py | 50 +++++++- certbot/tests/main_test.py | 109 ++++++++++++++++++ 14 files changed, 480 insertions(+), 55 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 722e94e18..03ba05bb0 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -323,7 +323,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Returned objects are guaranteed to be ssl vhosts return self._choose_vhosts_wildcard(domain, create_if_no_ssl) else: - return [self.choose_vhost(domain)] + return [self.choose_vhost(domain, create_if_no_ssl)] def _vhosts_for_wildcard(self, domain): """ @@ -475,20 +475,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if chain_path is not None: self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path - def choose_vhost(self, target_name, temp=False): + def choose_vhost(self, target_name, create_if_no_ssl=True): """Chooses a virtual host based on the given domain name. If there is no clear virtual host to be selected, the user is prompted with all available choices. - The returned vhost is guaranteed to have TLS enabled unless temp is - True. If temp is True, there is no such guarantee and the result is - not cached. + The returned vhost is guaranteed to have TLS enabled unless + create_if_no_ssl is set to False, in which case there is no such guarantee + and the result is not cached. :param str target_name: domain name - :param bool temp: whether the vhost is only used temporarily + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created - :returns: ssl vhost associated with name + :returns: vhost associated with name :rtype: :class:`~certbot_apache.obj.VirtualHost` :raises .errors.PluginError: If no vhost is available or chosen @@ -501,7 +502,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Try to find a reasonable vhost vhost = self._find_best_vhost(target_name) if vhost is not None: - if temp: + if not create_if_no_ssl: return vhost if not vhost.ssl: vhost = self.make_vhost_ssl(vhost) @@ -510,7 +511,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc[target_name] = vhost return vhost - return self._choose_vhost_from_list(target_name, temp) + # Negate create_if_no_ssl value to indicate if we want a SSL vhost + # to get created if a non-ssl vhost is selected. + return self._choose_vhost_from_list(target_name, temp=not create_if_no_ssl) def _choose_vhost_from_list(self, target_name, temp=False): # Select a vhost from a list @@ -1505,7 +1508,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) - vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) + matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) + # We should be handling only SSL vhosts for enhancements + vhosts = [vhost for vhost in matched_vhosts if vhost.ssl] + + if not vhosts: + msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a " + "domain {0} for enabling enhancement \"{1}\". The requested " + "enhancement was not configured.") + msg_enhancement = enhancement + if options: + msg_enhancement += ": " + options + msg = msg_tmpl.format(domain, msg_enhancement) + logger.warning(msg) + raise errors.PluginError(msg) try: for vhost in vhosts: func(vhost, options) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7b1e4fa86..e33e16843 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -246,7 +246,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.display_ops.select_vhost") def test_choose_vhost_select_vhost_with_temp(self, mock_select): mock_select.return_value = self.vh_truth[0] - chosen_vhost = self.config.choose_vhost("none.com", temp=True) + chosen_vhost = self.config.choose_vhost("none.com", create_if_no_ssl=False) self.assertEqual(self.vh_truth[0], chosen_vhost) @mock.patch("certbot_apache.display_ops.select_vhost") @@ -936,6 +936,22 @@ class MultipleVhostsTest(util.ApacheTest): errors.PluginError, self.config.enhance, "certbot.demo", "unknown_enhancement") + def test_enhance_no_ssl_vhost(self): + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.assertRaises(errors.PluginError, self.config.enhance, + "certbot.demo", "redirect") + # Check that correct logger.warning was printed + self.assertTrue("not able to find" in mock_log.call_args[0][0]) + self.assertTrue("\"redirect\"" in mock_log.call_args[0][0]) + + mock_log.reset_mock() + + self.assertRaises(errors.PluginError, self.config.enhance, + "certbot.demo", "ensure-http-header", "Test") + # Check that correct logger.warning was printed + self.assertTrue("not able to find" in mock_log.call_args[0][0]) + self.assertTrue("Test" in mock_log.call_args[0][0]) + @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() @@ -945,6 +961,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "staple-ocsp") # Get the ssl vhost for certbot.demo @@ -971,6 +988,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # Checking the case with already enabled ocsp stapling configuration + self.config.choose_vhost("ocspvhost.com") self.config.enhance("ocspvhost.com", "staple-ocsp") # Get the ssl vhost for letsencrypt.demo @@ -995,6 +1013,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("socache_shmcb_module") self.config.get_version = mock.Mock(return_value=(2, 2, 0)) + self.config.choose_vhost("certbot.demo") self.assertRaises(errors.PluginError, self.config.enhance, "certbot.demo", "staple-ocsp") @@ -1020,6 +1039,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Strict-Transport-Security") @@ -1039,7 +1059,8 @@ class MultipleVhostsTest(util.ApacheTest): # skip the enable mod self.config.parser.modules.add("headers_module") - # This will create an ssl vhost for certbot.demo + # This will create an ssl vhost for encryption-example.demo + self.config.choose_vhost("encryption-example.demo") self.config.enhance("encryption-example.demo", "ensure-http-header", "Strict-Transport-Security") @@ -1058,6 +1079,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1079,7 +1101,8 @@ class MultipleVhostsTest(util.ApacheTest): # skip the enable mod self.config.parser.modules.add("headers_module") - # This will create an ssl vhost for certbot.demo + # This will create an ssl vhost for encryption-example.demo + self.config.choose_vhost("encryption-example.demo") self.config.enhance("encryption-example.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1097,6 +1120,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 2)) # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") # These are not immediately available in find_dir even with save() and @@ -1147,6 +1171,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.save() # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") # These are not immediately available in find_dir even with save() and @@ -1213,6 +1238,9 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules.add("rewrite_module") self.config.get_version = mock.Mock(return_value=(2, 3, 9)) + # Creates ssl vhost for the domain + self.config.choose_vhost("red.blue.purple.com") + self.config.enhance("red.blue.purple.com", "redirect") verify_no_redirect = ("certbot_apache.configurator." "ApacheConfigurator._verify_no_certbot_redirect") @@ -1224,7 +1252,7 @@ class MultipleVhostsTest(util.ApacheTest): # Skip the enable mod self.config.parser.modules.add("rewrite_module") self.config.get_version = mock.Mock(return_value=(2, 3, 9)) - + self.config.choose_vhost("red.blue.purple.com") self.config.enhance("red.blue.purple.com", "redirect") # Clear state about enabling redirect on this run # pylint: disable=protected-access @@ -1446,6 +1474,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("headers_module") + self.vh_truth[3].ssl = True self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] self.config.enhance("*.certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1453,6 +1482,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") def test_enhance_wildcard_no_install(self, mock_choose): + self.vh_truth[3].ssl = True mock_choose.return_value = [self.vh_truth[3]] self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("headers_module") diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py index a648101e9..fde8d4c35 100644 --- a/certbot-apache/certbot_apache/tests/debian_test.py +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -161,6 +161,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config.parser.modules.add("mod_ssl.c") self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True + # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "staple-ocsp") self.assertTrue("socache_shmcb_module" in self.config.parser.modules) @@ -172,6 +174,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Strict-Transport-Security") self.assertTrue("headers_module" in self.config.parser.modules) @@ -183,6 +186,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") self.assertTrue("rewrite_module" in self.config.parser.modules) diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index 5ce96ac5f..549feb17d 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -123,7 +123,8 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port))) try: - vhost = self.configurator.choose_vhost(achall.domain, temp=True) + vhost = self.configurator.choose_vhost(achall.domain, + create_if_no_ssl=False) except (PluginError, MissingCommandlineFlag): # We couldn't find the virtualhost for this domain, possibly # because it's a new vhost that's not configured yet diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 4240a0523..d841c1912 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -46,7 +46,7 @@ def rename_lineage(config): """ disp = zope.component.getUtility(interfaces.IDisplay) - certname = _get_certnames(config, "rename")[0] + certname = get_certnames(config, "rename")[0] new_certname = config.new_certname if not new_certname: @@ -88,7 +88,7 @@ def certificates(config): def delete(config): """Delete Certbot files associated with a certificate lineage.""" - certnames = _get_certnames(config, "delete", allow_multiple=True) + certnames = get_certnames(config, "delete", allow_multiple=True) for certname in certnames: storage.delete_files(config, certname) disp = zope.component.getUtility(interfaces.IDisplay) @@ -288,11 +288,7 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): cert.privkey)) return "".join(certinfo) -################### -# Private Helpers -################### - -def _get_certnames(config, verb, allow_multiple=False): +def get_certnames(config, verb, allow_multiple=False, custom_prompt=None): """Get certname from flag, interactively, or error out. """ certname = config.certname @@ -305,22 +301,32 @@ def _get_certnames(config, verb, allow_multiple=False): if not choices: raise errors.Error("No existing certificates found.") if allow_multiple: + if not custom_prompt: + prompt = "Which certificate(s) would you like to {0}?".format(verb) + else: + prompt = custom_prompt code, certnames = disp.checklist( - "Which certificate(s) would you like to {0}?".format(verb), - choices, cli_flag="--cert-name", - force_interactive=True) + prompt, choices, cli_flag="--cert-name", force_interactive=True) if code != display_util.OK: raise errors.Error("User ended interaction.") else: - code, index = disp.menu("Which certificate would you like to {0}?".format(verb), - choices, cli_flag="--cert-name", - force_interactive=True) + if not custom_prompt: + prompt = "Which certificate would you like to {0}?".format(verb) + else: + prompt = custom_prompt + + code, index = disp.menu( + prompt, choices, cli_flag="--cert-name", force_interactive=True) if code != display_util.OK or index not in range(0, len(choices)): raise errors.Error("User ended interaction.") certnames = [choices[index]] return certnames +################### +# Private Helpers +################### + def _report_lines(msgs): """Format a results report for a category of single-line renewal outcomes""" return " " + "\n ".join(str(msg) for msg in msgs) diff --git a/certbot/cli.py b/certbot/cli.py index 1c2273c8a..9584c3904 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -76,6 +76,7 @@ obtain, install, and renew certificates: (default) run Obtain & install a certificate in your current webserver certonly Obtain or renew a certificate, but do not install it renew Renew all previously obtained certificates that are near expiry + enhance Add security enhancements to your existing configuration -d DOMAINS Comma-separated list of domains to obtain a certificate for %s @@ -415,6 +416,12 @@ VERB_HELP = [ os.path.join(flag_default("config_dir"), "live"))), "usage": "\n\n certbot update_symlinks [options]\n\n" }), + ("enhance", { + "short": "Add security enhancements to your existing configuration", + "opts": ("Helps to harden the TLS configration by adding security enhancements " + "to already existing configuration."), + "usage": "\n\n certbot enhance [options]\n\n" + }), ] # VERB_HELP is a list in order to preserve order, but a dict is sometimes useful @@ -449,6 +456,7 @@ class HelpfulArgumentParser(object): "update_symlinks": main.update_symlinks, "certificates": main.certificates, "delete": main.delete, + "enhance": main.enhance, } # Get notification function for printing @@ -883,21 +891,22 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "flag to 0 disables log rotation entirely, causing " "Certbot to always append to the same log file.") helpful.add( - [None, "automation", "run", "certonly"], "-n", "--non-interactive", "--noninteractive", + [None, "automation", "run", "certonly", "enhance"], + "-n", "--non-interactive", "--noninteractive", dest="noninteractive_mode", action="store_true", default=flag_default("noninteractive_mode"), help="Run without ever asking for user input. This may require " "additional command line flags; the client will try to explain " "which ones are required if it finds one missing") helpful.add( - [None, "register", "run", "certonly"], + [None, "register", "run", "certonly", "enhance"], constants.FORCE_INTERACTIVE_FLAG, action="store_true", default=flag_default("force_interactive"), help="Force Certbot to be interactive even if it detects it's not " "being run in a terminal. This flag cannot be used with the " "renew subcommand.") helpful.add( - [None, "run", "certonly", "certificates"], + [None, "run", "certonly", "certificates", "enhance"], "-d", "--domains", "--domain", dest="domains", metavar="DOMAIN", action=_DomainsAction, default=flag_default("domains"), @@ -913,8 +922,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "name. In the case of a name collision it will append a number " "like 0001 to the file path name. (default: Ask)") helpful.add( - [None, "run", "certonly", "manage", "delete", "certificates", "renew"], - "--cert-name", dest="certname", + [None, "run", "certonly", "manage", "delete", "certificates", + "renew", "enhance"], "--cert-name", dest="certname", metavar="CERTNAME", default=flag_default("certname"), help="Certificate name to apply. This name is used by Certbot for housekeeping " "and in file paths; it doesn't affect the content of the certificate itself. " @@ -1085,7 +1094,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis dest="must_staple", default=flag_default("must_staple"), help=config_help("must_staple")) helpful.add( - "security", "--redirect", action="store_true", dest="redirect", + ["security", "enhance"], + "--redirect", action="store_true", dest="redirect", default=flag_default("redirect"), help="Automatically redirect all HTTP traffic to HTTPS for the newly " "authenticated vhost. (default: Ask)") @@ -1095,7 +1105,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " "authenticated vhost. (default: Ask)") helpful.add( - "security", "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), + ["security", "enhance"], + "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), help="Add the Strict-Transport-Security header to every HTTP response." " Forcing browser to always use SSL for the domain." " Defends against SSL Stripping.") @@ -1103,7 +1114,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "security", "--no-hsts", action="store_false", dest="hsts", default=flag_default("hsts"), help=argparse.SUPPRESS) helpful.add( - "security", "--uir", action="store_true", dest="uir", default=flag_default("uir"), + ["security", "enhance"], + "--uir", action="store_true", dest="uir", default=flag_default("uir"), help='Add the "Content-Security-Policy: upgrade-insecure-requests"' ' header to every HTTP response. Forcing the browser to use' ' https:// for every http:// resource.') diff --git a/certbot/client.py b/certbot/client.py index cf2afa2e5..45dc9c63b 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -466,7 +466,7 @@ class Client(object): # sites may have been enabled / final cleanup self.installer.restart() - def enhance_config(self, domains, chain_path): + def enhance_config(self, domains, chain_path, ask_redirect=True): """Enhance the configuration. :param list domains: list of domains to configure @@ -493,8 +493,9 @@ class Client(object): for config_name, enhancement_name, option in enhancement_info: config_value = getattr(self.config, config_name) if enhancement_name in supported: - if config_name == "redirect" and config_value is None: - config_value = enhancements.ask(enhancement_name) + if ask_redirect: + if config_name == "redirect" and config_value is None: + config_value = enhancements.ask(enhancement_name) if config_value: self.apply_enhancement(domains, enhancement_name, option) enhanced = True @@ -530,8 +531,12 @@ class Client(object): try: self.installer.enhance(dom, enhancement, options) except errors.PluginEnhancementAlreadyPresent: - logger.warning("Enhancement %s was already set.", - enhancement) + if enhancement == "ensure-http-header": + logger.warning("Enhancement %s was already set.", + options) + else: + logger.warning("Enhancement %s was already set.", + enhancement) except errors.PluginError: logger.warning("Unable to set enhancement %s for %s", enhancement, dom) diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 57d2362fd..1e15a8474 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -86,13 +86,31 @@ def choose_account(accounts): else: return None +def choose_values(values, question=None): + """Display screen to let user pick one or multiple values from the provided + list. -def choose_names(installer): + :param list values: Values to select from + + :returns: List of selected values + :rtype: list + """ + code, items = z_util(interfaces.IDisplay).checklist( + question, tags=values, force_interactive=True) + if code == display_util.OK and items: + return items + else: + return [] + +def choose_names(installer, question=None): """Display screen to select domains to validate. :param installer: An installer object :type installer: :class:`certbot.interfaces.IInstaller` + :param `str` question: Overriding dialog question to ask the user if asked + to choose from domain names. + :returns: List of selected names :rtype: `list` of `str` @@ -108,7 +126,7 @@ def choose_names(installer): return _choose_names_manually( "No names were found in your configuration files. ") - code, names = _filter_names(names) + code, names = _filter_names(names, question) if code == display_util.OK and names: return names else: @@ -142,7 +160,7 @@ def _sort_names(FQDNs): return sorted(FQDNs, key=lambda fqdn: fqdn.split('.')[::-1][1:]) -def _filter_names(names): +def _filter_names(names, override_question=None): """Determine which names the user would like to select from a list. :param list names: domain names @@ -155,10 +173,12 @@ def _filter_names(names): """ #Sort by domain first, and then by subdomain sorted_names = _sort_names(names) - + if override_question: + question = override_question + else: + question = "Which names would you like to activate HTTPS for?" code, names = z_util(interfaces.IDisplay).checklist( - "Which names would you like to activate HTTPS for?", - tags=sorted_names, cli_flag="--domains", force_interactive=True) + question, tags=sorted_names, cli_flag="--domains", force_interactive=True) return code, [str(s) for s in names] diff --git a/certbot/main.py b/certbot/main.py index 7be852e83..dd0991c8d 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -382,7 +382,7 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): if not obj.yesno(msg, "Update cert", "Cancel", default=True): raise errors.ConfigurationError("Specified mismatched cert name and domains.") -def _find_domains_or_certname(config, installer): +def _find_domains_or_certname(config, installer, question=None): """Retrieve domains and certname from config or user input. :param config: Configuration object @@ -391,6 +391,8 @@ def _find_domains_or_certname(config, installer): :param installer: Installer object :type installer: interfaces.IInstaller + :param `str` question: Overriding dialog question to ask the user if asked + to choose from domain names. :returns: Two-part tuple of domains and certname :rtype: `tuple` of list of `str` and `str` @@ -411,7 +413,7 @@ def _find_domains_or_certname(config, installer): # that certname might not have existed, or there was a problem. # try to get domains from the user. if not domains: - domains = display_ops.choose_names(installer) + domains = display_ops.choose_names(installer, question) if not domains and not certname: raise errors.Error("Please specify --domains, or --installer that " @@ -859,6 +861,53 @@ def plugins_cmd(config, plugins): logger.debug("Prepared plugins: %s", available) notify(str(available)) +def enhance(config, plugins): + """Add security enhancements to existing configuration + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ + supported_enhancements = ["hsts", "redirect", "uir", "staple"] + # Check that at least one enhancement was requested on command line + if not any([getattr(config, enh) for enh in supported_enhancements]): + msg = ("Please specify one or more enhancement types to configure. To list " + "the available enhancement types, run:\n\n%s --help enhance\n") + logger.warning(msg, sys.argv[0]) + raise errors.MisconfigurationError("No enhancements requested, exiting.") + + try: + installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "enhance") + except errors.PluginSelectionError as e: + return str(e) + + certname_question = ("Which certificate would you like to use to enhance " + "your configuration?") + config.certname = cert_manager.get_certnames( + config, "enhance", allow_multiple=False, + custom_prompt=certname_question)[0] + cert_domains = cert_manager.domains_for_certname(config, config.certname) + if config.noninteractive_mode: + domains = cert_domains + else: + domain_question = ("Which domain names would you like to enable the " + "selected enhancements for?") + domains = display_ops.choose_values(cert_domains, domain_question) + if not domains: + raise errors.Error("User cancelled the domain selection. No domains " + "defined, exiting.") + if not config.chain_path: + lineage = cert_manager.lineage_for_certname(config, config.certname) + config.chain_path = lineage.chain_path + le_client = _init_le_client(config, authenticator=None, installer=installer) + le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + def rollback(config, plugins): """Rollback server configuration changes made during install. diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 5b1953187..030d5b6db 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -147,6 +147,7 @@ def record_chosen_plugins(config, plugins, auth, inst): def choose_configurator_plugins(config, plugins, verb): + # pylint: disable=too-many-branches """ Figure out which configurator we're going to use, modifies config.authenticator and config.installer strings to reflect that choice if @@ -159,6 +160,11 @@ def choose_configurator_plugins(config, plugins, verb): """ req_auth, req_inst = cli_plugin_requests(config) + installer_question = None + + if verb == "enhance": + installer_question = ("Which installer would you like to use to " + "configure the selected enhancements?") # Which plugins do we need? if verb == "run": @@ -176,11 +182,11 @@ def choose_configurator_plugins(config, plugins, verb): need_inst = need_auth = False if verb == "certonly": need_auth = True - if verb == "install": + if verb == "install" or verb == "enhance": need_inst = True if config.authenticator: - logger.warning("Specifying an authenticator doesn't make sense in install mode") - + logger.warning("Specifying an authenticator doesn't make sense when " + "running Certbot with verb \"%s\"", verb) # Try to meet the user's request and/or ask them to pick plugins authenticator = installer = None if verb == "run" and req_auth == req_inst: @@ -189,7 +195,7 @@ def choose_configurator_plugins(config, plugins, verb): authenticator = installer = pick_configurator(config, req_inst, plugins) else: if need_inst or req_inst: - installer = pick_installer(config, req_inst, plugins) + installer = pick_installer(config, req_inst, plugins, installer_question) if need_auth: authenticator = pick_authenticator(config, req_auth, plugins) logger.debug("Selected authenticator %s and installer %s", authenticator, installer) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 675b936b9..6ec1d4f5c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -569,5 +569,103 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self.assertRaises(errors.OverlappingMatchFound, self._call, self.config, None, None, None) +class GetCertnameTest(unittest.TestCase): + """Tests for certbot.cert_manager.""" + + def setUp(self): + self.get_utility_patch = test_util.patch_get_utility() + self.mock_get_utility = self.get_utility_patch.start() + self.config = mock.MagicMock() + self.config.certname = None + + def tearDown(self): + self.get_utility_patch.stop() + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "Which certificate would you" + self.mock_get_utility().menu.return_value = (display_util.OK, 0) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=False), ['example.com']) + self.assertTrue( + prompt in self.mock_get_utility().menu.call_args[0][0]) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_custom_prompt(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "custom prompt" + self.mock_get_utility().menu.return_value = (display_util.OK, 0) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=False, custom_prompt=prompt), + ['example.com']) + self.assertEquals(self.mock_get_utility().menu.call_args[0][0], + prompt) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_user_abort(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + self.mock_get_utility().menu.return_value = (display_util.CANCEL, 0) + self.assertRaises( + errors.Error, + cert_manager.get_certnames, + self.config, "erroring_anyway", allow_multiple=False) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "Which certificate(s) would you" + self.mock_get_utility().checklist.return_value = (display_util.OK, + ['example.com']) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=True), ['example.com']) + self.assertTrue( + prompt in self.mock_get_utility().checklist.call_args[0][0]) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple_custom_prompt(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "custom prompt" + self.mock_get_utility().checklist.return_value = (display_util.OK, + ['example.com']) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=True, custom_prompt=prompt), + ['example.com']) + self.assertEquals( + self.mock_get_utility().checklist.call_args[0][0], + prompt) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple_user_abort(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + self.mock_get_utility().checklist.return_value = (display_util.CANCEL, []) + self.assertRaises( + errors.Error, + cert_manager.get_certnames, + self.config, "erroring_anyway", allow_multiple=True) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 0f2c58161..6add141d4 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -433,6 +433,22 @@ class EnhanceConfigTest(ClientTestCommon): self.client.installer.enhance.assert_not_called() mock_enhancements.ask.assert_not_called() + @mock.patch("certbot.client.logger") + def test_already_exists_header(self, mock_log): + self.config.hsts = True + self._test_with_already_existing() + self.assertTrue(mock_log.warning.called) + self.assertEquals(mock_log.warning.call_args[0][1], + 'Strict-Transport-Security') + + @mock.patch("certbot.client.logger") + def test_already_exists_redirect(self, mock_log): + self.config.redirect = True + self._test_with_already_existing() + self.assertTrue(mock_log.warning.called) + self.assertEquals(mock_log.warning.call_args[0][1], + 'redirect') + def test_no_ask_hsts(self): self.config.hsts = True self._test_with_all_supported() @@ -508,6 +524,13 @@ class EnhanceConfigTest(ClientTestCommon): self.assertEqual(self.client.installer.save.call_count, 1) self.assertEqual(self.client.installer.restart.call_count, 1) + def _test_with_already_existing(self): + self.client.installer = mock.MagicMock() + self.client.installer.supported_enhancements.return_value = [ + "ensure-http-header", "redirect", "staple-ocsp"] + self.client.installer.enhance.side_effect = errors.PluginEnhancementAlreadyPresent() + self.client.enhance_config([self.domain], None) + class RollbackTest(unittest.TestCase): """Tests for certbot.client.rollback.""" diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index c4f58ba7c..9de8c5e9a 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -207,9 +207,9 @@ class ChooseNamesTest(unittest.TestCase): self.mock_install = mock.MagicMock() @classmethod - def _call(cls, installer): + def _call(cls, installer, question=None): from certbot.display.ops import choose_names - return choose_names(installer) + return choose_names(installer, question) @mock.patch("certbot.display.ops._choose_names_manually") def test_no_installer(self, mock_manual): @@ -281,6 +281,15 @@ class ChooseNamesTest(unittest.TestCase): self.assertEqual(names, ["example.com"]) self.assertEqual(mock_util().checklist.call_count, 1) + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_filter_namees_override_question(self, mock_util): + self.mock_install.get_all_names.return_value = set(["example.com"]) + mock_util().checklist.return_value = (display_util.OK, ["example.com"]) + names = self._call(self.mock_install, "Custom") + self.assertEqual(names, ["example.com"]) + self.assertEqual(mock_util().checklist.call_count, 1) + self.assertEqual(mock_util().checklist.call_args[0][0], "Custom") + @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_nothing_selected(self, mock_util): self.mock_install.get_all_names.return_value = set(["example.com"]) @@ -481,5 +490,42 @@ class ValidatorTests(unittest.TestCase): self.__validator, "msg", default="") +class ChooseValuesTest(unittest.TestCase): + """Test choose_values.""" + @classmethod + def _call(cls, values, question): + from certbot.display.ops import choose_values + return choose_values(values, question) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_success(self, mock_util): + items = ["first", "second", "third"] + mock_util().checklist.return_value = (display_util.OK, [items[2]]) + result = self._call(items, None) + self.assertEquals(result, [items[2]]) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], None) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_success_question(self, mock_util): + items = ["first", "second", "third"] + question = "Which one?" + mock_util().checklist.return_value = (display_util.OK, [items[1]]) + result = self._call(items, question) + self.assertEquals(result, [items[1]]) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], question) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_user_cancel(self, mock_util): + items = ["first", "second", "third"] + question = "Want to cancel?" + mock_util().checklist.return_value = (display_util.CANCEL, []) + result = self._call(items, question) + self.assertEquals(result, []) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], question) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b778f05ea..d168552cc 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1533,5 +1533,114 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): strict=self.config.strict_permissions) +class EnhanceTest(unittest.TestCase): + """Tests for certbot.main.enhance.""" + + def setUp(self): + self.get_utility_patch = test_util.patch_get_utility() + self.mock_get_utility = self.get_utility_patch.start() + + def tearDown(self): + self.get_utility_patch.stop() + + def _call(self, args): + plugins = disco.PluginsRegistry.find_all() + config = configuration.NamespaceConfig( + cli.prepare_and_parse_args(plugins, args)) + + with mock.patch('certbot.cert_manager.get_certnames') as mock_certs: + mock_certs.return_value = ['example.com'] + with mock.patch('certbot.cert_manager.domains_for_certname') as mock_dom: + mock_dom.return_value = ['example.com'] + with mock.patch('certbot.main._init_le_client') as mock_init: + mock_client = mock.MagicMock() + mock_client.config = config + mock_init.return_value = mock_client + main.enhance(config, plugins) + return mock_client # returns the client + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main._find_domains_or_certname') + def test_selection_question(self, mock_find, mock_choose, mock_lineage, _rec): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ['example.com'] + mock_find.return_value = (None, None) + with mock.patch('certbot.main.plug_sel.pick_installer') as mock_pick: + self._call(['enhance', '--redirect']) + self.assertTrue(mock_pick.called) + # Check that the message includes "enhancements" + self.assertTrue("enhancements" in mock_pick.call_args[0][3]) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main._find_domains_or_certname') + def test_selection_auth_warning(self, mock_find, mock_choose, mock_lineage, _rec): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + mock_find.return_value = (None, None) + with mock.patch('certbot.main.plug_sel.pick_installer'): + with mock.patch('certbot.main.plug_sel.logger.warning') as mock_log: + mock_client = self._call(['enhance', '-a', 'webroot', '--redirect']) + self.assertTrue(mock_log.called) + self.assertTrue("make sense" in mock_log.call_args[0][0]) + self.assertTrue(mock_client.enhance_config.called) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_enhance_config_call(self, _rec, mock_choose, mock_lineage): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + with mock.patch('certbot.main.plug_sel.pick_installer'): + mock_client = self._call(['enhance', '--redirect', '--hsts']) + req_enh = ["redirect", "hsts"] + not_req_enh = ["uir"] + self.assertTrue(mock_client.enhance_config.called) + self.assertTrue( + all([getattr(mock_client.config, e) for e in req_enh])) + self.assertFalse( + any([getattr(mock_client.config, e) for e in not_req_enh])) + self.assertTrue( + "example.com" in mock_client.enhance_config.call_args[0][0]) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_enhance_noninteractive(self, _rec, mock_choose, mock_lineage): + mock_lineage.return_value = mock.MagicMock( + chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + with mock.patch('certbot.main.plug_sel.pick_installer'): + mock_client = self._call(['enhance', '--redirect', + '--hsts', '--non-interactive']) + self.assertTrue(mock_client.enhance_config.called) + self.assertFalse(mock_choose.called) + + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_user_abort_domains(self, _rec, mock_choose): + mock_choose.return_value = [] + with mock.patch('certbot.main.plug_sel.pick_installer'): + self.assertRaises(errors.Error, + self._call, + ['enhance', '--redirect', '--hsts']) + + def test_no_enhancements_defined(self): + self.assertRaises(errors.MisconfigurationError, + self._call, ['enhance']) + + @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_plugin_selection_error(self, _rec, mock_choose, mock_pick): + mock_choose.return_value = ["example.com"] + mock_pick.return_value = (None, None) + mock_pick.side_effect = errors.PluginSelectionError() + mock_client = self._call(['enhance', '--hsts']) + self.assertFalse(mock_client.enhance_config.called) + if __name__ == '__main__': unittest.main() # pragma: no cover From 398bd4a2cd8e7a3b5f47b6a24fc3a0c604d0be5b Mon Sep 17 00:00:00 2001 From: Jeremy Gillula Date: Wed, 18 Apr 2018 16:46:39 -0700 Subject: [PATCH 450/631] =?UTF-8?q?Emphasizing=20the=20warnings=20in=20the?= =?UTF-8?q?=20READMEs=20in=20/etc/letsencrypt/live/exam=E2=80=A6=20(#5871)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Emphasizing the warnings in the READMEs in /etc/letsencrypt/live/example.com * Making the warning more of a statement --- certbot/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/certbot/storage.py b/certbot/storage.py index ed3922c58..c453e55b0 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -1053,6 +1053,9 @@ class RenewableCert(object): "`cert.pem` : will break many server configurations, and " "should not be used\n" " without reading further documentation (see link below).\n\n" + "WARNING: DO NOT MOVE THESE FILES!\n" + " Certbot expects these files to remain in this location in order\n" + " to function properly!\n\n" "We recommend not moving these files. For more information, see the Certbot\n" "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" "certificates.\n") From f40e04401fba78a54277053f97558a56dd78a882 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 19 Apr 2018 19:35:21 -0400 Subject: [PATCH 451/631] Don't install enum34 when using Python 3.4 or later. Fix #5456. (#5846) The re stdlib module requires attrs that don't exist in the backported 3.4 version. Technically, we are changing our install behavior beyond what is necessary. Previously, enum34 was used for 3.4 and 3.5 as well, and it happened not to conflict, but I think it's better to use the latest bug-fixed stdlib versions as long as they meet the needs of `cryptography`, which is what depends on enum34. That way, at least the various stdlib modules are guaranteed not to conflict with each other. --- letsencrypt-auto-source/letsencrypt-auto | 2 +- letsencrypt-auto-source/pieces/dependency-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 89072ec88..f3525aaee 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 0e2cec984..1e69af9c2 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -93,7 +93,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ From 726f3ce8b3540f6f0455a4fcbec1062d17e8cf25 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 19 Apr 2018 17:57:41 -0700 Subject: [PATCH 452/631] Remove EOL'd Ubuntu from targets.yaml (#5887) See https://wiki.ubuntu.com/Releases. Ubuntu 15.* repositories have been shut down for months now causing our tests to always fail on these systems. While the tests on Ubuntu 12.04 still work, it has been unsupported by Canonical for almost a year and I don't think we should hamstring ourselves trying to continue to support it ourselves. --- tests/letstest/targets.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 506225f86..9c1aca24e 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,16 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-26d5af4c - name: ubuntu15.10 - type: ubuntu - virt: hvm - user: ubuntu - - ami: ami-d92e6bb3 - name: ubuntu15.04LTS - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-7b89cc11 name: ubuntu14.04LTS type: ubuntu @@ -21,11 +11,6 @@ targets: type: ubuntu virt: pv user: ubuntu - - ami: ami-0611546c - name: ubuntu12.04LTS - type: ubuntu - virt: hvm - user: ubuntu #----------------------------------------------------------------------------- # Debian - ami: ami-116d857a From 9c15fd354f0e98e5b1e48b446304fb16cddced68 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Sat, 21 Apr 2018 00:17:05 +0200 Subject: [PATCH 453/631] Updated base image to python:2-alpine3.7 (#5889) Updated base image from python:2-alpine to python:2-alpine3.7. Python:2-alpine internally utilises alpine version 3.4 which end-of-life date is the first of May 2018. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cf4e9afad..28cd6b323 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2-alpine +FROM python:2-alpine3.7 ENTRYPOINT [ "certbot" ] EXPOSE 80 443 From f510f4bddf4da1411c1966a9cbed2b7e630cba30 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 24 Apr 2018 06:38:15 -0700 Subject: [PATCH 454/631] Update good volunteer task to good first issue. (#5891) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2098f7cdf..cbb7650b6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -63,7 +63,7 @@ Find issues to work on ---------------------- You can find the open issues in the `github issue tracker`_. Comparatively -easy ones are marked `Good Volunteer Task`_. If you're starting work on +easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan where appropriate. @@ -72,7 +72,7 @@ your pull request must have thorough unit test coverage, pass our tests, and be compliant with the :ref:`coding style `. .. _github issue tracker: https://github.com/certbot/certbot/issues -.. _Good Volunteer Task: https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+Volunteer+Task%22 +.. _good first issue: https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22 .. _testing: From bf30226c697402adac656bfd7956395fc104714b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 25 Apr 2018 15:09:50 -0700 Subject: [PATCH 455/631] Restore parallel waiting to Route53 plugin (#5712) * Bring back parallel updates to route53. * Re-add try * Fix TTL. * Remove unnecessary wait. * Add pylint exceptions. * Add dummy perform. * review.feedback * Fix underscore. * Fix lint. --- .../certbot_dns_route53/dns_route53.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 08b1d03f0..27d185656 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -42,14 +42,26 @@ class Authenticator(dns_common.DNSAuthenticator): def _setup_credentials(self): pass - def _perform(self, domain, validation_domain_name, validation): - try: - change_id = self._change_txt_record("UPSERT", validation_domain_name, validation) + def _perform(self, domain, validation_domain_name, validation): # pylint: disable=missing-docstring + pass - self._wait_for_change(change_id) + def perform(self, achalls): + self._attempt_cleanup = True + + try: + change_ids = [ + self._change_txt_record("UPSERT", + achall.validation_domain_name(achall.domain), + achall.validation(achall.account_key)) + for achall in achalls + ] + + for change_id in change_ids: + self._wait_for_change(change_id) except (NoCredentialsError, ClientError) as e: logger.debug('Encountered error during perform: %s', e, exc_info=True) raise errors.PluginError("\n".join([str(e), INSTRUCTIONS])) + return [achall.response(achall.account_key) for achall in achalls] def _cleanup(self, domain, validation_domain_name, validation): try: From 4b870ef940037f7978af31f014c65d5f0e897366 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 1 May 2018 16:59:32 -0700 Subject: [PATCH 456/631] Release 0.24.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 28 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 7 ++++- letsencrypt-auto | 28 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++++-------- 22 files changed, 82 insertions(+), 77 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 9cc616c36..98985b18e 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 1ad3b7e43..88f112551 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 061b0c8f3..0848080b3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 93495d22e..a5bbf880d 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 7f7eab517..28740ccef 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6a411dedf..9d35251ee 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0195fb0da..5be076ba4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index e004ef92c..29d981874 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 75f588825..2e6f430c5 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d8a8025ef..8ba9ac7db 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index cebe69b42..5d9ab5fbf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a86d12819..7a96bb5fd 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 82f813197..766bc7665 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index b584b872f..9eafd6d4e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 77ea4170a..6d88e220c 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 9dbab3b70..430535561 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.24.0.dev0' +__version__ = '0.24.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 218e21a88..259142e62 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -9,6 +9,7 @@ obtain, install, and renew certificates: (default) run Obtain & install a certificate in your current webserver certonly Obtain or renew a certificate, but do not install it renew Renew all previously obtained certificates that are near expiry + enhance Add security enhancements to your existing configuration -d DOMAINS Comma-separated list of domains to obtain a certificate for --apache Use the Apache plugin for authentication & installation @@ -107,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.23.0 (certbot; + "". (default: CertbotACMEClient/0.24.0 (certbot; darwin 10.13.4) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- @@ -397,6 +398,10 @@ update_symlinks: Recreates certificate and key symlinks in /etc/letsencrypt/live, if you changed them by hand or edited a renewal configuration file +enhance: + Helps to harden the TLS configration by adding security enhancements to + already existing configuration. + plugins: Plugin Selection: Certbot client supports an extensible plugins architecture. See 'certbot plugins' for a list of all installed plugins diff --git a/letsencrypt-auto b/letsencrypt-auto index 061b0c8f3..0848080b3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 620739670..641ebaef8 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlrFS/EACgkQTRfJlc2X -dfK+rQf8DcKY5bMi5eJnwwAlui6WIyWSrf1KAKt09tEGZSHQ1fcyCPrGVhk7VVDg -NJ1/XiYBquPW+7mYUcHrIRsiKYbTUcmVjyqP6tZd67IxRH9ToNqBzA6kq99T+IPd -iTGdczHMSPcxM6/Fa5PYMHXy2+ctTr/8+gnsxth9QfcM62Yd6ecfqIdoId3vk9Aw -UBMENZhUasIvgZDWuow+1XVZ/DAmdvj2Xl/E3sA9i2ArREJhkhVegtdrHkwSY+Hm -MKfZGqNVse6ZAF/8YdEVBum0OngMMs63DwucwFxmw5DqWtmnXm6awLNW/LQ/3R5L -xuKjcVaAT1h5TgIyRT6opH8JBKmLpg== -=Ouj4 +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlro/1AACgkQTRfJlc2X +dfLm5ggAxCrWU9dmYZKllcFzp7TFOdRap0pmarfL4gwSYj7B/bSceD7ysOyoQ8Ra +7UHuZKAQyurZn1seN49d88Kgor9KWZQ1jZiGkfiEpp8qAkdWzFR8UqYa2/CZtk2l +bExm8YQDwhuKvCObGLDGi3ydcIQpfg/rsBkSTphKYXN/Zebx9mAelZN4CgGRy03Y +3z2UqqnyqFPAg4wUGcNfCgUEbJ5bUPr733vQzjBS2IVUbDbu06/1Y8oYzurezXNS +6lEyvTfC5G8RGlSWupNu7yWviD14M4LnAo6WXWEVH+C+ssJaPrZVhZ6KfEt/Erg3 +k06WZSPDCtOm5EfhDm0Rumqm1owA2g== +=Bc4G -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f3525aaee..0848080b3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0.dev0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 738fa36e074721d2d7ea6d34b8dafc5937bbf2d1..4a937e7e0c442056410a4d75bc6fc64fa7fc5ed8 100644 GIT binary patch literal 256 zcmV+b0ssDE4g*`fFs}k@&Ip7vL<-AyWT*hCDr+-?vC;X|s(C>trU<`!&JJ7Yet?gj zP<-mLp*eOt8&aten_HT4Xaq=Kx#zQxgGH^wqLqmDbiNH^#Z>!uQ#lxbz9TmzQ``gA zepS_2=ZD+oI|Kl|@HhjznV9>uJqAGPzsGZ`_Tr$cOh5JZM+hOCqkF1gQK5rq?01Ha z$#L4=i}~HfWo>cf%K<$CERWB0i$y+n$+{zxu0TOGArrjm@c)#iwau&wz!kfWn^wMB GGm3kHFnBTS{idqO_gjrF5G{ zVRP?&R3bm%6xg|eMaq^mxiODsYes`VLVP%?>YI`_vpQkyqmump!{L*LC1Srlny=7{ zo3Y>fVHPDtRA0e84`j9mZXT*pgZ6cioA*IXpnwBfgM|DH+$uv9hdg9*BU!VA0HaH%xvk86LU? z;y5! diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 0c1161db7..fc4457771 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 From 0ec0d79c3560db13ed86a2ef8ae780655d14f37d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 1 May 2018 16:59:48 -0700 Subject: [PATCH 457/631] Bump version to 0.25.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 98985b18e..72ab5919b 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 88f112551..f0c20da7c 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index a5bbf880d..50df2a56e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 28740ccef..8e1f9d28b 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9d35251ee..05998ee6a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 5be076ba4..cd3b0613e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 29d981874..10ee710cd 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2e6f430c5..a7f44b989 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8ba9ac7db..c171e5014 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 5d9ab5fbf..2c0e35308 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 7a96bb5fd..821a40655 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 766bc7665..21d9dec29 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 9eafd6d4e..84b701638 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6d88e220c..6889d06e4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 430535561..27c63e266 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.24.0' +__version__ = '0.25.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0848080b3..853c66023 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 552bfa5eb70bb339f4274b412fac76037fe629dd Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 2 May 2018 10:52:54 +0300 Subject: [PATCH 458/631] New interfaces for installers to run tasks on renew verb (#5879) * ServerTLSUpdater and InstallerSpecificUpdater implementation * Fixed tests and added disables for linter :/ * Added error logging for misconfigurationerror from plugin check * Remove redundant parameter from interfaces * Renaming the interfaces * Finalize interface renaming and move tests to own file * Refactored the runners * Refactor the cli params * Fix the interface args * Fixed documentation * Documentation and naming fixes * Remove ServerTLSConfigurationUpdater * Remove unnecessary linter disable * Rename run_renewal_updaters to run_generic_updaters * Do not raise exception, but make log message more informative and visible for the user * Run renewal deployer before installer restart --- certbot/cli.py | 8 ++++ certbot/constants.py | 1 + certbot/interfaces.py | 75 +++++++++++++++++++++++++++++ certbot/main.py | 6 ++- certbot/renewal.py | 11 +++-- certbot/tests/main_test.py | 17 ++++++- certbot/tests/renewupdater_test.py | 76 ++++++++++++++++++++++++++++++ certbot/updater.py | 67 ++++++++++++++++++++++++++ 8 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 certbot/tests/renewupdater_test.py create mode 100644 certbot/updater.py diff --git a/certbot/cli.py b/certbot/cli.py index 9584c3904..b71d60055 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1192,6 +1192,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis default=flag_default("directory_hooks"), dest="directory_hooks", help="Disable running executables found in Certbot's hook directories" " during renewal. (default: False)") + helpful.add( + "renew", "--disable-renew-updates", action="store_true", + default=flag_default("disable_renew_updates"), dest="disable_renew_updates", + help="Disable automatic updates to your server configuration that" + " would otherwise be done by the selected installer plugin, and triggered" + " when the user executes \"certbot renew\", regardless of if the certificate" + " is renewed. This setting does not apply to important TLS configuration" + " updates.") helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) diff --git a/certbot/constants.py b/certbot/constants.py index 0d0ee8d3f..40557d287 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,6 +64,7 @@ CLI_DEFAULTS = dict( pref_challs=[], validate_hooks=True, directory_hooks=True, + disable_renew_updates=False, # Subparsers num=None, diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 501a5c57e..8061f0de3 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -256,6 +256,10 @@ class IConfig(zope.interface.Interface): "user; only needed if your config is somewhere unsafe like /tmp/." "This is a boolean") + disable_renew_updates = zope.interface.Attribute( + "If updates provided by installer enhancements when Certbot is being run" + " with \"renew\" verb should be disabled.") + class IInstaller(IPlugin): """Generic Certbot Installer Interface. @@ -591,3 +595,74 @@ class IReporter(zope.interface.Interface): def print_messages(self): """Prints messages to the user and clears the message queue.""" + + +# Updater interfaces +# +# When "certbot renew" is run, Certbot will iterate over each lineage and check +# if the selected installer for that lineage is a subclass of each updater +# class. If it is and the update of that type is configured to be run for that +# lineage, the relevant update function will be called for each domain in the +# lineage. These functions are never called for other subcommands, so if an +# installer wants to perform an update during the run or install subcommand, it +# should do so when :func:`IInstaller.deploy_cert` is called. + +class GenericUpdater(object): + """Interface for update types not currently specified by Certbot. + + This class allows plugins to perform types of updates that Certbot hasn't + defined (yet). + + To make use of this interface, the installer should implement the interface + methods, and interfaces.GenericUpdater.register(InstallerClass) should + be called from the installer code. + + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def generic_updates(self, domain, *args, **kwargs): + """Perform any update types defined by the installer. + + If an installer is a subclass of the class containing this method, this + function will always be called when "certbot renew" is run. If the + update defined by the installer should be run conditionally, the + installer needs to handle checking the conditions itself. + + This method is called once for each domain. + + :param str domain: domain to handle the updates for + + """ + + +class RenewDeployer(object): + """Interface for update types run when a lineage is renewed + + This class allows plugins to perform types of updates that need to run at + lineage renewal that Certbot hasn't defined (yet). + + To make use of this interface, the installer should implement the interface + methods, and interfaces.RenewDeployer.register(InstallerClass) should + be called from the installer code. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def renew_deploy(self, lineage, *args, **kwargs): + """Perform updates defined by installer when a certificate has been renewed + + If an installer is a subclass of the class containing this method, this + function will always be called when a certficate has been renewed by + running "certbot renew". For example if a plugin needs to copy a + certificate over, or change configuration based on the new certificate. + + This method is called once for each lineage renewed + + :param lineage: Certificate lineage object that is set if certificate + was renewed on this run. + :type lineage: storage.RenewableCert + + """ diff --git a/certbot/main.py b/certbot/main.py index dd0991c8d..a041b998f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -29,6 +29,7 @@ from certbot import log from certbot import renewal from certbot import reporter from certbot import storage +from certbot import updater from certbot import util from certbot.display import util as display_util, ops as display_ops @@ -1145,10 +1146,9 @@ def renew_cert(config, plugins, lineage): except errors.PluginSelectionError as e: logger.info("Could not choose appropriate plugin: %s", e) raise - le_client = _init_le_client(config, auth, installer) - _get_and_save_cert(le_client, config, lineage=lineage) + renewed_lineage = _get_and_save_cert(le_client, config, lineage=lineage) notify = zope.component.getUtility(interfaces.IDisplay).notification if installer is None: @@ -1158,9 +1158,11 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. + updater.run_renewal_deployer(renewed_lineage, installer, config) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) + # Run deployer def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. diff --git a/certbot/renewal.py b/certbot/renewal.py index ea5d87a5e..4651eeb36 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -12,13 +12,14 @@ import zope.component import OpenSSL from certbot import cli - from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util from certbot import hooks from certbot import storage +from certbot import updater + from certbot.plugins import disco as plugins_disco logger = logging.getLogger(__name__) @@ -411,9 +412,9 @@ def handle_renewal_request(config): # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(lineage_config) renewal_candidate.ensure_deployed() + from certbot import main + plugins = plugins_disco.PluginsRegistry.find_all() if should_renew(lineage_config, renewal_candidate): - plugins = plugins_disco.PluginsRegistry.find_all() - from certbot import main # domains have been restored into lineage_config by reconstitute # but they're unnecessary anyway because renew_cert here # will just grab them from the certificate @@ -426,6 +427,10 @@ def handle_renewal_request(config): "cert", renewal_candidate.latest_common_version())) renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, expiry.strftime("%Y-%m-%d"))) + # Run updater interface methods + updater.run_generic_updaters(lineage_config, plugins, + renewal_candidate) + except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index d168552cc..22653ca3a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -23,6 +23,7 @@ from certbot import configuration from certbot import crypto_util from certbot import errors from certbot import main +from certbot import updater from certbot import util from certbot.plugins import disco @@ -1232,7 +1233,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) - def test_renew_with_configurator(self): + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_renew_with_configurator(self, mock_sel): + mock_sel.return_value = (mock.MagicMock(), mock.MagicMock()) renewalparams = {'authenticator': 'webroot'} self._test_renew_common( renewalparams=renewalparams, assert_oc_called=True, @@ -1448,6 +1451,18 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met email in mock_utility().add_message.call_args[0][0]) self.assertTrue(mock_handle.called) + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_plugin_selection_error(self, mock_choose): + mock_choose.side_effect = errors.PluginSelectionError + self.assertRaises(errors.PluginSelectionError, main.renew_cert, + None, None, None) + + with mock.patch('certbot.updater.logger.warning') as mock_log: + updater.run_generic_updaters(None, None, None) + self.assertTrue(mock_log.called) + self.assertTrue("Could not choose appropriate plugin for updaters" + in mock_log.call_args[0][0]) + class UnregisterTest(unittest.TestCase): def setUp(self): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py new file mode 100644 index 000000000..9d0f8d515 --- /dev/null +++ b/certbot/tests/renewupdater_test.py @@ -0,0 +1,76 @@ +"""Tests for renewal updater interfaces""" +import unittest +import mock + +from certbot import interfaces +from certbot import main +from certbot import updater + +import certbot.tests.util as test_util + + +class RenewUpdaterTest(unittest.TestCase): + """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" + + def setUp(self): + class MockInstallerGenericUpdater(interfaces.GenericUpdater): + """Mock class that implements GenericUpdater""" + def __init__(self, *args, **kwargs): + # pylint: disable=unused-argument + self.restart = mock.MagicMock() + self.callcounter = mock.MagicMock() + def generic_updates(self, domain, *args, **kwargs): + self.callcounter(*args, **kwargs) + + class MockInstallerRenewDeployer(interfaces.RenewDeployer): + """Mock class that implements RenewDeployer""" + def __init__(self, *args, **kwargs): + # pylint: disable=unused-argument + self.callcounter = mock.MagicMock() + def renew_deploy(self, lineage, *args, **kwargs): + self.callcounter(*args, **kwargs) + + self.generic_updater = MockInstallerGenericUpdater() + self.renew_deployer = MockInstallerRenewDeployer() + + def get_config(self, args): + """Get mock config from dict of parameters""" + config = mock.MagicMock() + for key in args.keys(): + config.__dict__[key] = args[key] + return config + + @mock.patch('certbot.main._get_and_save_cert') + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + @test_util.patch_get_utility() + def test_server_updates(self, _, mock_select, mock_getsave): + config = self.get_config({"disable_renew_updates": False}) + + lineage = mock.MagicMock() + lineage.names.return_value = ['firstdomain', 'seconddomain'] + mock_getsave.return_value = lineage + mock_generic_updater = self.generic_updater + + # Generic Updater + mock_select.return_value = (mock_generic_updater, None) + with mock.patch('certbot.main._init_le_client'): + main.renew_cert(config, None, mock.MagicMock()) + self.assertTrue(mock_generic_updater.restart.called) + + mock_generic_updater.restart.reset_mock() + mock_generic_updater.callcounter.reset_mock() + updater.run_generic_updaters(config, None, lineage) + self.assertEqual(mock_generic_updater.callcounter.call_count, 2) + self.assertFalse(mock_generic_updater.restart.called) + + def test_renew_deployer(self): + config = self.get_config({"disable_renew_updates": False}) + lineage = mock.MagicMock() + lineage.names.return_value = ['firstdomain', 'seconddomain'] + mock_deployer = self.renew_deployer + updater.run_renewal_deployer(lineage, mock_deployer, config) + self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py new file mode 100644 index 000000000..f822c55ee --- /dev/null +++ b/certbot/updater.py @@ -0,0 +1,67 @@ +"""Updaters run at renewal""" +import logging + +from certbot import errors +from certbot import interfaces + +from certbot.plugins import selection as plug_sel + +logger = logging.getLogger(__name__) + +def run_generic_updaters(config, plugins, lineage): + """Run updaters that the plugin supports + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :returns: `None` + :rtype: None + """ + try: + # installers are used in auth mode to determine domain names + installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") + except errors.PluginSelectionError as e: + logger.warning("Could not choose appropriate plugin for updaters: %s", e) + return + _run_updaters(lineage, installer, config) + +def run_renewal_deployer(lineage, installer, config): + """Helper function to run deployer interface method if supported by the used + installer plugin. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: `None` + :rtype: None + """ + if not config.disable_renew_updates and isinstance(installer, + interfaces.RenewDeployer): + installer.renew_deploy(lineage) + +def _run_updaters(lineage, installer, config): + """Helper function to run the updater interface methods if supported by the + used installer plugin. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: `None` + :rtype: None + """ + for domain in lineage.names(): + if not config.disable_renew_updates: + if isinstance(installer, interfaces.GenericUpdater): + installer.generic_updates(domain) From 7fa3455dc6c22b51e7dee4bb33b51d63ca087412 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 2 May 2018 12:18:29 -0700 Subject: [PATCH 459/631] Update changelog for 0.24.0 (#5915) --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c7c41a9..044e55250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.24.0 - 2018-05-02 + +### Added + +* certbot now has an enhance subcommand which allows you to configure security + enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without + reinstalling a certificate. +* certbot-dns-rfc2136 now allows the user to specify the port to use to reach + the DNS server in its credentials file. +* acme now parses the wildcard field included in authorizations so it can be + used by users of the library. + +### Changed + +* certbot-dns-route53 used to wait for each DNS update to propagate before + sending the next one, but now it sends all updates before waiting which + speeds up issuance for multiple domains dramatically. +* Certbot's official Docker images are now based on Alpine Linux 3.7 rather + than 3.4 because 3.4 has reached its end-of-life. +* We've doubled the time Certbot will spend polling authorizations before + timing out. +* The level of the message logged when Certbot is being used with + non-standard paths warning that crontabs for renewal included in Certbot + packages from OS package managers may not work has been reduced. This stops + the message from being written to stderr every time `certbot renew` runs. + +### Fixed + +* certbot-auto now works with Python 3.6. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* acme +* certbot +* certbot-apache +* certbot-dns-digitalocean (only style improvements to tests) +* certbot-dns-rfc2136 + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/52?closed=1 + ## 0.23.0 - 2018-04-04 ### Added From 95c0c4a7083f45e4157485303624eab271024ca0 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 3 May 2018 01:56:37 +0300 Subject: [PATCH 460/631] Py2 and Py3 compatibility for metaclass interfaces (#5913) --- certbot/interfaces.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 8061f0de3..f5858a7fd 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -1,5 +1,6 @@ """Certbot client interfaces.""" import abc +import six import zope.interface # pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class @@ -607,6 +608,7 @@ class IReporter(zope.interface.Interface): # installer wants to perform an update during the run or install subcommand, it # should do so when :func:`IInstaller.deploy_cert` is called. +@six.add_metaclass(abc.ABCMeta) class GenericUpdater(object): """Interface for update types not currently specified by Certbot. @@ -619,8 +621,6 @@ class GenericUpdater(object): """ - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def generic_updates(self, domain, *args, **kwargs): """Perform any update types defined by the installer. @@ -637,6 +637,7 @@ class GenericUpdater(object): """ +@six.add_metaclass(abc.ABCMeta) class RenewDeployer(object): """Interface for update types run when a lineage is renewed @@ -648,8 +649,6 @@ class RenewDeployer(object): be called from the installer code. """ - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def renew_deploy(self, lineage, *args, **kwargs): """Perform updates defined by installer when a certificate has been renewed From 32e85e9a230cd202d6f7cf790164f36c6210396f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 3 May 2018 00:59:25 -0700 Subject: [PATCH 461/631] correct metaclass usage everywhere (#5919) --- acme/acme/challenges.py | 3 ++- certbot/interfaces.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 96997297b..9702a7a14 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -9,6 +9,7 @@ from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import OpenSSL import requests +import six from acme import errors from acme import crypto_util @@ -139,6 +140,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True +@six.add_metaclass(abc.ABCMeta) class KeyAuthorizationChallenge(_TokenChallenge): # pylint: disable=abstract-class-little-used,too-many-ancestors """Challenge based on Key Authorization. @@ -147,7 +149,6 @@ class KeyAuthorizationChallenge(_TokenChallenge): that will be used to generate `response`. """ - __metaclass__ = abc.ABCMeta response_cls = NotImplemented thumbprint_hash_function = ( diff --git a/certbot/interfaces.py b/certbot/interfaces.py index f5858a7fd..c96f6bd51 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -7,11 +7,10 @@ import zope.interface # pylint: disable=too-few-public-methods +@six.add_metaclass(abc.ABCMeta) class AccountStorage(object): """Accounts storage interface.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def find_all(self): # pragma: no cover """Find all accounts. From 3eaf35f1e2473187882dc746547886b859870b2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 3 May 2018 13:10:33 -0700 Subject: [PATCH 462/631] Check_untyped_defs in mypy with clean output for acme (#5874) * check_untyped_defs in mypy with clean output for acme * test entire acme module * Add typing as a dependency because it's only in the stdlib for 3.5+ * Add str_utils, modified for python2.7 compatibility * make mypy happy in acme * typing is needed in prod * we actually only need typing in acme so far * add tests and more docs for str_utils * pragma no cover * add magic_typing * s/from typing/from magic_typing/g * move typing to dev_extras * correctly set up imports * remove str_utils * only type: ignore for OpenSSL.SSL, not crypto * Since we only run mypy with python3 anyway and we're fine importing it when it's not actually there, there's no actual need for typing to be present as a dependency * comment magic_typing.py * disable wildcard-import im magic_typing * disable pylint errors * add magic_typing_test * make magic_typing tests work alongside other tests * make sure temp_typing is set * add typing as a dev dependency for python3.4 * run mypy with python3.4 on travis to get a little more testing with different environments * don't stick typing into sys.modules * reorder imports --- .travis.yml | 2 ++ acme/acme/challenges.py | 2 +- acme/acme/client.py | 9 ++--- acme/acme/client_test.py | 4 ++- acme/acme/crypto_util.py | 63 ++++++++++++++++++---------------- acme/acme/crypto_util_test.py | 3 +- acme/acme/magic_typing.py | 13 +++++++ acme/acme/magic_typing_test.py | 41 ++++++++++++++++++++++ acme/acme/messages_test.py | 3 +- acme/acme/standalone.py | 9 ++--- acme/acme/standalone_test.py | 5 +-- acme/acme/test_util.py | 14 ++++---- mypy.ini | 3 ++ setup.py | 1 + 14 files changed, 121 insertions(+), 51 deletions(-) create mode 100644 acme/acme/magic_typing.py create mode 100644 acme/acme/magic_typing_test.py diff --git a/.travis.yml b/.travis.yml index 370137f68..111ddb3d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ matrix: addons: - python: "2.7" env: TOXENV=lint + - python: "3.4" + env: TOXENV=mypy - python: "3.5" env: TOXENV=mypy - python: "2.7" diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 9702a7a14..b2a4882eb 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -478,7 +478,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): try: cert = self.probe_cert(domain=domain, **kwargs) except errors.Error as error: - logger.debug(error, exc_info=True) + logger.debug(str(error), exc_info=True) return False return self.verify_cert(cert) diff --git a/acme/acme/client.py b/acme/acme/client.py index 19615b087..7807f0ece 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -9,7 +9,6 @@ import time import six from six.moves import http_client # pylint: disable=import-error - import josepy as jose import OpenSSL import re @@ -20,6 +19,8 @@ from acme import crypto_util from acme import errors from acme import jws from acme import messages +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, List, Set, Text logger = logging.getLogger(__name__) @@ -415,7 +416,7 @@ class Client(ClientBase): """ # pylint: disable=too-many-locals assert max_attempts > 0 - attempts = collections.defaultdict(int) + attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -529,7 +530,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] + chain = [] # type: List[jose.ComparableX509] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -864,7 +865,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() + self._nonces = set() # type: Set[Text] self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index be08c2919..c17b83210 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -17,6 +17,7 @@ from acme import jws as acme_jws from acme import messages from acme import messages_test from acme import test_util +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module CERT_DER = test_util.load_vector('cert.der') @@ -61,7 +62,8 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - self.new_reg = messages.NewRegistration(**dict(reg)) + the_arg = dict(reg) # type: Dict + self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 2281196eb..ad914ca60 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -6,11 +6,13 @@ import os import re import socket -import OpenSSL +from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose - from acme import errors +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Text, Union logger = logging.getLogger(__name__) @@ -25,7 +27,7 @@ logger = logging.getLogger(__name__) # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni # should be changed to use "set_options" to disable SSLv2 and SSLv3, # in case it's used for things other than probing/serving! -_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD # type: ignore +_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore class SSLSocket(object): # pylint: disable=too-few-public-methods @@ -64,9 +66,9 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods logger.debug("Server name (%s) not recognized, dropping SSL", server_name) return - new_context = OpenSSL.SSL.Context(self.method) - new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2) - new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3) + new_context = SSL.Context(self.method) + new_context.set_options(SSL.OP_NO_SSLv2) + new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) connection.set_context(new_context) @@ -89,18 +91,18 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods def accept(self): # pylint: disable=missing-docstring sock, addr = self.sock.accept() - context = OpenSSL.SSL.Context(self.method) - context.set_options(OpenSSL.SSL.OP_NO_SSLv2) - context.set_options(OpenSSL.SSL.OP_NO_SSLv3) + context = SSL.Context(self.method) + context.set_options(SSL.OP_NO_SSLv2) + context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) - ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock)) + ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() logger.debug("Performing handshake with %s", addr) try: ssl_sock.do_handshake() - except OpenSSL.SSL.Error as error: + except SSL.Error as error: # _pick_certificate_cb might have returned without # creating SSL context (wrong server name) raise socket.error(error) @@ -128,7 +130,7 @@ def probe_sni(name, host, port=443, timeout=300, :rtype: OpenSSL.crypto.X509 """ - context = OpenSSL.SSL.Context(method) + context = SSL.Context(method) context.set_timeout(timeout) socket_kwargs = {'source_address': source_address} @@ -145,13 +147,13 @@ def probe_sni(name, host, port=443, timeout=300, raise errors.Error(error) with contextlib.closing(sock) as client: - client_ssl = OpenSSL.SSL.Connection(context, client) + client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 try: client_ssl.do_handshake() client_ssl.shutdown() - except OpenSSL.SSL.Error as error: + except SSL.Error as error: raise errors.Error(error) return client_ssl.get_peer_certificate() @@ -164,18 +166,18 @@ def make_csr(private_key_pem, domains, must_staple=False): OCSP Must Staple: https://tools.ietf.org/html/rfc7633). :returns: buffer PEM-encoded Certificate Signing Request. """ - private_key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, private_key_pem) - csr = OpenSSL.crypto.X509Req() + private_key = crypto.load_privatekey( + crypto.FILETYPE_PEM, private_key_pem) + csr = crypto.X509Req() extensions = [ - OpenSSL.crypto.X509Extension( + crypto.X509Extension( b'subjectAltName', critical=False, value=', '.join('DNS:' + d for d in domains).encode('ascii') ), ] if must_staple: - extensions.append(OpenSSL.crypto.X509Extension( + extensions.append(crypto.X509Extension( b"1.3.6.1.5.5.7.1.24", critical=False, value=b"DER:30:03:02:01:05")) @@ -183,8 +185,8 @@ def make_csr(private_key_pem, domains, must_staple=False): csr.set_pubkey(private_key) csr.set_version(2) csr.sign(private_key, 'sha256') - return OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) + return crypto.dump_certificate_request( + crypto.FILETYPE_PEM, csr) def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN @@ -221,11 +223,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): parts_separator = ", " prefix = "DNS" + part_separator - if isinstance(cert_or_req, OpenSSL.crypto.X509): - func = OpenSSL.crypto.dump_certificate + if isinstance(cert_or_req, crypto.X509): + # pylint: disable=line-too-long + func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] else: - func = OpenSSL.crypto.dump_certificate_request - text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") + func = crypto.dump_certificate_request + text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") # WARNING: this function does not support multiple SANs extensions. # Multiple X509v3 extensions of the same type is disallowed by RFC 5280. match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text) @@ -252,12 +255,12 @@ def gen_ss_cert(key, domains, not_before=None, """ assert domains, "Must provide one or more hostnames for the cert." - cert = OpenSSL.crypto.X509() + cert = crypto.X509() cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) extensions = [ - OpenSSL.crypto.X509Extension( + crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), ] @@ -266,7 +269,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_issuer(cert.get_subject()) if force_san or len(domains) > 1: - extensions.append(OpenSSL.crypto.X509Extension( + extensions.append(crypto.X509Extension( b"subjectAltName", critical=False, value=b", ".join(b"DNS:" + d.encode() for d in domains) @@ -281,7 +284,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.sign(key, "sha256") return cert -def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): +def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in @@ -298,7 +301,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): if isinstance(cert, jose.ComparableX509): # pylint: disable=protected-access cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) + return crypto.dump_certificate(filetype, cert) # assumes that OpenSSL.crypto.dump_certificate includes ending # newline character diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 3874ba9d9..36d62b324 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -13,6 +13,7 @@ import OpenSSL from acme import errors from acme import test_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module class SSLSocketAndProbeSNITest(unittest.TestCase): @@ -165,7 +166,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] + self.serial_num = [] # type: List[int] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py new file mode 100644 index 000000000..555088cf2 --- /dev/null +++ b/acme/acme/magic_typing.py @@ -0,0 +1,13 @@ +"""Shim class to not have to depend on typing module in prod.""" +import sys + +class TypingClass(object): + """Ignore import errors by getting anything""" + def __getattr__(self, name): + return None + +try: + # mypy doesn't respect modifying sys.modules + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +except ImportError: + sys.modules[__name__] = TypingClass() diff --git a/acme/acme/magic_typing_test.py b/acme/acme/magic_typing_test.py new file mode 100644 index 000000000..23dfe3367 --- /dev/null +++ b/acme/acme/magic_typing_test.py @@ -0,0 +1,41 @@ +"""Tests for acme.magic_typing.""" +import sys +import unittest + +import mock + + +class MagicTypingTest(unittest.TestCase): + """Tests for acme.magic_typing.""" + def test_import_success(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + typing_class_mock = mock.MagicMock() + text_mock = mock.MagicMock() + typing_class_mock.Text = text_mock + sys.modules['typing'] = typing_class_mock + if 'acme.magic_typing' in sys.modules: + del sys.modules['acme.magic_typing'] # pragma: no cover + from acme.magic_typing import Text # pylint: disable=no-name-in-module + self.assertEqual(Text, text_mock) + del sys.modules['acme.magic_typing'] + sys.modules['typing'] = temp_typing + + def test_import_failure(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + sys.modules['typing'] = None + if 'acme.magic_typing' in sys.modules: + del sys.modules['acme.magic_typing'] # pragma: no cover + from acme.magic_typing import Text # pylint: disable=no-name-in-module + self.assertTrue(Text is None) + del sys.modules['acme.magic_typing'] + sys.modules['typing'] = temp_typing + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 64bc81efd..0e2d8c62d 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -6,6 +6,7 @@ import mock from acme import challenges from acme import test_util +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module CERT = test_util.load_comparable_cert('cert.der') @@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: Dict self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index c221f5883..a370501ee 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -16,6 +16,7 @@ import OpenSSL from acme import challenges from acme import crypto_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object): def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] - self.servers = [] + self.threads = [] # type: List[threading.Thread] + self.servers = [] # type: List[ACMEServerMixin] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound @@ -189,7 +190,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) - socketserver.BaseRequestHandler.__init__(self, *args, **kwargs) + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # pylint: disable=redefined-builtin """Log arbitrary message.""" @@ -262,7 +263,7 @@ def simple_tls_sni_01_server(cli_args, forever=True): certs = {} - _, hosts, _ = next(os.walk('.')) + _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465 for host in hosts: with open(os.path.join(host, "cert.pem")) as cert_file: cert_contents = cert_file.read() diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 5b1b72c18..1591187e5 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -18,6 +18,7 @@ from acme import challenges from acme import crypto_util from acme import errors from acme import test_util +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module class TLSServerTest(unittest.TestCase): @@ -72,7 +73,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() + self.resources = set() # type: Set from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -201,7 +202,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() + self.resources = set() # type: Set from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index d5d57bd27..1a0b67056 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -10,7 +10,7 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import OpenSSL +from OpenSSL import crypto def vector_path(*names): @@ -39,8 +39,8 @@ def _guess_loader(filename, loader_pem, loader_der): def load_cert(*names): """Load certificate.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_certificate(loader, load_vector(*names)) def load_comparable_cert(*names): @@ -51,8 +51,8 @@ def load_comparable_cert(*names): def load_csr(*names): """Load certificate request.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_certificate_request(loader, load_vector(*names)) def load_comparable_csr(*names): @@ -71,8 +71,8 @@ def load_rsa_private_key(*names): def load_pyopenssl_private_key(*names): """Load pyOpenSSL private key.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_privatekey(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_privatekey(loader, load_vector(*names)) def skip_unless(condition, reason): # pragma: no cover diff --git a/mypy.ini b/mypy.ini index aac60a2ad..2bac9249f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,6 @@ [mypy] python_version = 2.7 ignore_missing_imports = True + +[mypy-acme.*] +check_untyped_defs = True diff --git a/setup.py b/setup.py index e674871a8..3760fd35b 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ dev_extras = [ dev3_extras = [ 'mypy', + 'typing', # for python3.4 ] docs_extras = [ From 83ea820525d4342ee58baa0d6c6547b6843d8dc6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 May 2018 12:11:36 -0700 Subject: [PATCH 463/631] disable apacheconftest (#5937) --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 111ddb3d4..fa68d376c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,6 @@ matrix: services: docker before_install: addons: - - python: "2.7" - env: TOXENV=apacheconftest - sudo: required - python: "2.7" env: TOXENV=nginxroundtrip From 832941279b23b2da73921c7b89878ebb5fac2fec Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 10 May 2018 16:46:57 -0700 Subject: [PATCH 464/631] Specify that every domain name needs to be under a server_name directive (#5949) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 3ba8bcb06..378f24b63 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -293,7 +293,7 @@ class NginxConfigurator(common.Installer): ("Cannot find a VirtualHost matching domain %s. " "In order for Certbot to correctly perform the challenge " "please add a corresponding server_name directive to your " - "nginx configuration: " + "nginx configuration for every domain on your certificate: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. for vhost in vhosts: From 8851141dcf93d8e6c080bcd1503e64762fcebf1b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 06:12:10 -0700 Subject: [PATCH 465/631] Revert "disable apacheconftest (#5937)" (#5954) This reverts commit 83ea820525d4342ee58baa0d6c6547b6843d8dc6. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index fa68d376c..111ddb3d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,9 @@ matrix: services: docker before_install: addons: + - python: "2.7" + env: TOXENV=apacheconftest + sudo: required - python: "2.7" env: TOXENV=nginxroundtrip From 2f89a10f50b1a81d3a7fa5fc264fffc521b575dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 07:07:02 -0700 Subject: [PATCH 466/631] Small dev docs improvements (#5953) * Clarify UNIX only * Have people develop natively. Some systems like Arch Linux and macOS require --debug in order to install dependencies. Our bootstrapping script for macOS works, so let's let people who want to develop natively. * briefly mention docker as dev option * remove bad _common.sh info * update OS dep section * Remove sudo from certbot-auto usage When sudo isn't available, Certbot is able to fall back to su. Removing it from the instructions here allows the command to work when its run in minimal systems like Docker where sudo may not be installed. * copy advice about missing interpreters * Improve integration tests docs Explain what a boulder is and tell people they probably should just let the tests run in Travis. * Don't tell people to run integration tests. I don't think any paid Certbot devs run integration tests locally and instead rely on Travis. Let's not make others do it. * fix spacing * you wouldn't download a CA * address review comments --- docs/contributing.rst | 104 ++++++++++++++++++------------------------ docs/install.rst | 21 +++++---- 2 files changed, 56 insertions(+), 69 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cbb7650b6..e86d1a5b3 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,6 +11,12 @@ Developer Guide Getting Started =============== +Certbot has the same :ref:`system requirements ` when set +up for development. While the section below will help you install Certbot and +its dependencies, Certbot needs to be run on a UNIX-like OS so if you're using +Windows, you'll need to set up a (virtual) machine running an OS such as Linux +and continue with these instructions on that UNIX-like OS. + Running a local copy of the client ---------------------------------- @@ -22,40 +28,35 @@ running: git clone https://github.com/certbot/certbot -If you're on macOS, we recommend you skip the rest of this section and instead -run Certbot in Docker. You can find instructions for how to do this :ref:`here -`. If you're running on Linux, you can run the following commands to -install dependencies and set up a virtual environment where you can run -Certbot. You will need to repeat this when Certbot's dependencies change or when -a new plugin is introduced. +Next you need to install dependencies and set up a virtual environment where +you can run Certbot. We recommend you do this using the commands below, +however, you can alternatively skip the rest of this section and :ref:`run +Certbot in Docker `. .. code-block:: shell cd certbot - sudo ./certbot-auto --os-packages-only - ./tools/venv.sh + ./certbot-auto --debug --os-packages-only + tools/venv.sh + +.. note:: You may need to repeat this when + Certbot's dependencies change or when a new plugin is introduced. You can now run the copy of Certbot from git either by executing -``venv/bin/certbot``, or by activating the virtual environment. If you're -actively modifying and testing the code, you may want to run commands like this in -each shell where you're working: +``venv/bin/certbot``, or by activating the virtual environment. You can do the +latter by running: .. code-block:: shell - source ./venv/bin/activate - export SERVER=https://acme-staging-v02.api.letsencrypt.org/directory - source tests/integration/_common.sh + source venv/bin/activate -After that, your shell will be using the virtual environment, your copy of -Certbot will default to requesting test (staging) certificates, and you run the -client by typing `certbot` or `certbot_test`. The latter is an alias that -includes several flags useful for testing. For instance, it sets various output -directories to point to /tmp/, and uses non-privileged ports for challenges, so -root privileges are not required. - -Activating a shell with `venv/bin/activate` sets environment variables so that -Python pulls in the correct versions of various packages needed by Certbot. -More information can be found in the `virtualenv docs`_. +After running this command, ``certbot`` and development tools like ``ipdb``, +``ipython``, ``pytest``, and ``tox`` are available in the shell where you ran +the command. These tools are installed in the virtual environment and are kept +separate from your global Python installation. This works by setting +environment variables so the right executables are found and Python can pull in +the versions of various packages needed by Certbot. More information can be +found in the `virtualenv docs`_. .. _`virtualenv docs`: https://virtualenv.pypa.io @@ -95,25 +96,24 @@ Once all the unittests pass, check for sufficient test coverage using ``tox -e cover``, and then check for code style with ``tox -e lint`` (all files) or ``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a time). -Once all of the above is successful, you may run the full test suite, -including integration tests, using ``tox``. We recommend running the -commands above first, because running all tests with ``tox`` is very -slow, and the large amount of ``tox`` output can make it hard to find -specific failures when they happen. Also note that the full test suite -will attempt to modify your system's Apache config if your user has sudo -permissions, so it should not be run on a production Apache server. +Once all of the above is successful, you may run the full test suite using +``tox --skip-missing-interpreters``. We recommend running the commands above +first, because running all tests like this is very slow, and the large amount +of output can make it hard to find specific failures when they happen. -If you have trouble getting the full ``tox`` suite to run locally, it is -generally sufficient to open a pull request and let Github and Travis run -integration tests for you. +.. warning:: The full test suite may attempt to modify your system's Apache + config if your user has sudo permissions, so it should not be run on a + production Apache server. .. _integration: Integration testing with the Boulder CA ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To run integration tests locally, you need Docker and docker-compose installed -and working. Fetch and start Boulder using: +Generally it is sufficient to open a pull request and let Github and Travis run +integration tests for you, however, if you want to run them locally you need +Docker and docker-compose installed and working. Fetch and start Boulder, Let's +Encrypt's ACME CA software, by using: .. code-block:: shell @@ -316,14 +316,13 @@ Steps: 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. -5. If your code touches communication with an ACME server/Boulder, you - should run the integration tests, see `integration`_. -6. Submit the PR. -7. Did your tests pass on Travis? If they didn't, fix any errors. +5. Submit the PR. +6. Did your tests pass on Travis? If they didn't, fix any errors. Updating certbot-auto and letsencrypt-auto ========================================== + Updating the scripts -------------------- Developers should *not* modify the ``certbot-auto`` and ``letsencrypt-auto`` files @@ -386,8 +385,8 @@ Running the client with Docker ============================== You can use Docker Compose to quickly set up an environment for running and -testing Certbot. This is especially useful for macOS users. To install Docker -Compose, follow the instructions at https://docs.docker.com/compose/install/. +testing Certbot. To install Docker Compose, follow the instructions at +https://docs.docker.com/compose/install/. .. note:: Linux users can simply run ``pip install docker-compose`` to get Docker Compose after installing Docker Engine and activating your shell as @@ -420,38 +419,23 @@ OS-level dependencies can be installed like so: .. code-block:: shell - letsencrypt-auto-source/letsencrypt-auto --os-packages-only + ./certbot-auto --debug --os-packages-only In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.7 is required +* `Python`_ 2.7 or 3.4+ is required * `Augeas`_ is required for the Python bindings -* ``virtualenv`` and ``pip`` are used for managing other python library - dependencies +* ``virtualenv`` is used for managing other Python library dependencies .. _Python: https://wiki.python.org/moin/BeginnersGuide/Download .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Debian ------- - -For squeeze you will need to: - -- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. - - FreeBSD ------- -Packages can be installed on FreeBSD using ``pkg``, -or any other port-management tool (``portupgrade``, ``portmanager``, etc.) -from the pre-built package or can be built and installed from ports. -Either way will ensure proper installation of all the dependencies required -for the package. - FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see above), you will need a compatible shell, e.g. ``pkg install bash && bash``. diff --git a/docs/install.rst b/docs/install.rst index d47264545..ead59350d 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,18 +19,21 @@ your system. .. _certbot.eff.org: https://certbot.eff.org +.. _system_requirements: + System Requirements =================== -Certbot currently requires Python 2.7, or 3.4+. By default, it requires -root access in order to write to ``/etc/letsencrypt``, -``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 -(if you use the ``standalone`` plugin) and to read and modify webserver -configurations (if you use the ``apache`` or ``nginx`` plugins). If none of -these apply to you, it is theoretically possible to run without root privileges, -but for most users who want to avoid running an ACME client as root, either -`letsencrypt-nosudo `_ or -`simp_le `_ are more appropriate choices. +Certbot currently requires Python 2.7 or 3.4+ running on a UNIX-like operating +system. By default, it requires root access in order to write to +``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to +bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and +modify webserver configurations (if you use the ``apache`` or ``nginx`` +plugins). If none of these apply to you, it is theoretically possible to run +without root privileges, but for most users who want to avoid running an ACME +client as root, either `letsencrypt-nosudo +`_ or `simp_le +`_ are more appropriate choices. The Apache plugin currently requires an OS with augeas version 1.0; currently `it supports From 9568f9d5b046c2114efc4ebde43f20ae31c6bdfb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 10:52:45 -0700 Subject: [PATCH 467/631] Add instructions on how to ask for help (#5957) * Add instructions on how to ask for help * s/setup/set up --- docs/contributing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index e86d1a5b3..4628c23ca 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -319,6 +319,13 @@ Steps: 5. Submit the PR. 6. Did your tests pass on Travis? If they didn't, fix any errors. +Asking for help +=============== + +If you have any questions while working on a Certbot issue, don't hesitate to +ask for help! You can do this in the #letsencrypt-dev IRC channel on Freenode. +If you don't already have an IRC client set up, we recommend you join using +`Riot `_. Updating certbot-auto and letsencrypt-auto ========================================== From 5940ee92ab5c9a9f05f7067974f6e15c9fa3205a Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 11 May 2018 14:25:02 -0700 Subject: [PATCH 468/631] add ready status type (#5941) --- acme/acme/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index a69b3bbc4..03dbc3255 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing') STATUS_VALID = Status('valid') STATUS_INVALID = Status('invalid') STATUS_REVOKED = Status('revoked') +STATUS_READY = Status('ready') class IdentifierType(_Constant): From 68359086fffca8805893bf6133c53b5f75357a7f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 13 May 2018 08:06:19 -0700 Subject: [PATCH 469/631] Add link to pycon issues (#5959) * add link to pycon issues * add especially --- docs/contributing.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4628c23ca..ba89e70fa 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -63,6 +63,10 @@ found in the `virtualenv docs`_. Find issues to work on ---------------------- +.. note:: If you're sprinting on Certbot at PyCon, you can find especially good + issues to work on during the event `here + `_. + You can find the open issues in the `github issue tracker`_. Comparatively easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan From cce23c86c7fb79d883438497dd76e601f2adf687 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 14 May 2018 08:08:24 -0700 Subject: [PATCH 470/631] partially revert #5953 (#5964) --- docs/contributing.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ba89e70fa..f152a7600 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -28,10 +28,11 @@ running: git clone https://github.com/certbot/certbot -Next you need to install dependencies and set up a virtual environment where -you can run Certbot. We recommend you do this using the commands below, -however, you can alternatively skip the rest of this section and :ref:`run -Certbot in Docker `. +If you're on macOS, we recommend you skip the rest of this section and instead +run Certbot in Docker. You can find instructions for how to do this :ref:`here +`. If you're running on Linux, you can run the following commands to +install dependencies and set up a virtual environment where you can run +Certbot. .. code-block:: shell From a0775f42baac28e5dc15206a5647c15e2c9eba8f Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 12:34:27 -0400 Subject: [PATCH 471/631] fixed issue #5969 for certbot-dns-dnsmadeeasy --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 2bac9249f..29b3ee7a6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,3 +4,6 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True + +[mypy-certbot_dns_dnsmadeeasy.*] +check_untyped_defs = True \ No newline at end of file From 2d45b0b07ac189adf0a997b3294f1e5c0de1796e Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 13:26:33 -0400 Subject: [PATCH 472/631] Check_untyped_defs in mypy with clean output for certbot_dns_rfc2136 (#5975) * check_untyped_defs in mypy with clean output for certbot_dns_rfc2136 Resolves #5973 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 2bac9249f..01c1f7af6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,3 +4,6 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True + +[mypy-certbot_dns_rfc2136.*] +check_untyped_defs = True From 4bd9f4dac487928131a2df67ed8227a40e4f9254 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:30:38 -0400 Subject: [PATCH 473/631] fix for issue #5970 regarding certbot-dns-google --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 29b3ee7a6..f5ab78490 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,5 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True -[mypy-certbot_dns_dnsmadeeasy.*] +[mypy-certbot_dns_google.*] check_untyped_defs = True \ No newline at end of file From 5636b5550749b03a23c44a652fa5dc361fd8e1a0 Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 13:40:27 -0400 Subject: [PATCH 474/631] Check_untyped_defs in mypy with clean output for certbot-dns-luadns * check_untyped_defs in mypy with clean output for certbot-dns-luadns Resolves #5971 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 01c1f7af6..135964702 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,3 +7,6 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True + +[mypy-certbot_dns_luadns.*] +check_untyped_defs = True From 430f9414a987fc0b0f3a04edce518db170ea76a7 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:42:12 -0400 Subject: [PATCH 475/631] fix for issue #5968 for certbot-dns-dnsimple --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index f5ab78490..79daeadd4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,5 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True -[mypy-certbot_dns_google.*] +[mypy-certbot_dns_dnsimple.*] check_untyped_defs = True \ No newline at end of file From 33583792fae361e6b12232c895ffdebe53a9d1b5 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:43:15 -0400 Subject: [PATCH 476/631] blank line at eof --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 79daeadd4..a8fa8edee 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,4 +6,4 @@ ignore_missing_imports = True check_untyped_defs = True [mypy-certbot_dns_dnsimple.*] -check_untyped_defs = True \ No newline at end of file +check_untyped_defs = True From 74448e934491f7d8e731a5473febe1dc54dd1d15 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Mon, 14 May 2018 13:45:12 -0400 Subject: [PATCH 477/631] Set pause=False to fix view_config_changes (#5977) --- certbot/reverter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/reverter.py b/certbot/reverter.py index 34feafc7e..15ad1a987 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -181,7 +181,7 @@ class Reverter(object): if for_logging: return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( - os.linesep.join(output), force_interactive=True) + os.linesep.join(output), force_interactive=True, pause=False) def add_to_temp_checkpoint(self, save_files, save_notes): """Add files to temporary checkpoint. From be03a976d5a649a24810a85bb4f86d7caae95ddd Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:54:26 -0400 Subject: [PATCH 478/631] fixed issue #5969 for certbot-dns-dnsmadeeasy (#5976) --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 01c1f7af6..6580de6de 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,8 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-certbot_dns_dnsmadeeasy.*] +check_untyped_defs = True + [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True From c1471fe873e4b72a01638e75b2f30bb111c3bf92 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Mon, 14 May 2018 16:15:02 -0400 Subject: [PATCH 479/631] Document IPv6/IPv4 binding on standalone. (#5983) --- docs/using.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 7a25a5cc2..272f5ac6e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -170,6 +170,14 @@ one of the options shown below on the command line. It must still be possible for your machine to accept inbound connections from the Internet on the specified port using each requested domain name. +By default, Certbot first attempts to bind to the port for all interfaces using +IPv6 and then bind to that port using IPv4; Certbot continues so long as at +least one bind succeeds. On most Linux systems, IPv4 traffic will be routed to +the bound IPv6 port and the failure during the second bind is expected. + +Use ``---address`` to explicitly tell Certbot which interface +(and protocol) to bind. + .. note:: The ``--standalone-supported-challenges`` option has been deprecated since ``certbot`` version 0.9.0. From 907ee797151f270bec3a2697743568362db497cd Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 16:18:49 -0400 Subject: [PATCH 480/631] Check_untyped_defs in mypy with clean output for certbot-dns-nsone (#5987) * check_untyped_defs in mypy with clean output for certbot-dns-nsone Resolves #5972 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 8f1bc91ef..449262795 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,5 +17,8 @@ check_untyped_defs = True [mypy-certbot_dns_luadns.*] check_untyped_defs = True +[mypy-certbot_dns_nsone.*] +check_untyped_defs = True + [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True From a724dc659ba7590dfae21fd6bfdf6c0a13766283 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 17:21:09 -0400 Subject: [PATCH 481/631] correct error message text now prompts user to run $certbot certificates (#5988) --- certbot/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index a041b998f..0ae5b9d7a 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -324,7 +324,7 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): return "newcert", None else: raise errors.ConfigurationError("No certificate with name {0} found. " - "Use -d to specify domains, or run certbot --certificates to see " + "Use -d to specify domains, or run certbot certificates to see " "possible certificate names.".format(certname)) def _get_added_removed(after, before): From 372d4a046cb5b3f937469d4220c0fb6ba4a4f85c Mon Sep 17 00:00:00 2001 From: speter Date: Mon, 14 May 2018 23:40:06 +0200 Subject: [PATCH 482/631] docs/using.rst: fix typo (#5962) --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 272f5ac6e..40d8f8452 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -609,7 +609,7 @@ commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. If you write a custom script and expect to run a command only after a certificate was actually renewed - you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal + you will need to use the ``--deploy-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. .. _renewal-config-file: From 02b128a12831f4c43fb734dd1e1ca08deb66142c Mon Sep 17 00:00:00 2001 From: signop <39252060+signop@users.noreply.github.com> Date: Mon, 14 May 2018 14:43:43 -0700 Subject: [PATCH 483/631] Add support for specifying source_address to ClientNetwork. (#5990) For certbot/certbot#3489 --- acme/acme/client.py | 14 +++++++++++++- acme/acme/client_test.py | 25 +++++++++++++++++++++++++ acme/setup.py | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 7807f0ece..d4f5a4787 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -12,7 +12,9 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import OpenSSL import re +from requests_toolbelt.adapters.source import SourceAddressAdapter import requests +from requests.adapters import HTTPAdapter import sys from acme import crypto_util @@ -857,9 +859,12 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. :param float timeout: Timeout for requests. + :param source_address: Optional source address to bind to when making requests. + :type source_address: str or tuple(str, int) """ def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT, + source_address=None): # pylint: disable=too-many-arguments self.key = key self.account = account @@ -869,6 +874,13 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout + adapter = HTTPAdapter() + + if source_address is not None: + adapter = SourceAddressAdapter(source_address) + + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) def __del__(self): # Try to close the session, but don't show exceptions to the diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index c17b83210..f3018ed81 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1129,6 +1129,31 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertRaises(requests.exceptions.RequestException, self.net.post, 'uri', obj=self.obj) +class ClientNetworkSourceAddressBindingTest(unittest.TestCase): + """Tests that if ClientNetwork has a source IP set manually, the underlying library has + used the provided source address.""" + + def setUp(self): + self.source_address = "8.8.8.8" + + def test_source_address_set(self): + from acme.client import ClientNetwork + net = ClientNetwork(key=None, alg=None, source_address=self.source_address) + for adapter in net.session.adapters.values(): + self.assertTrue(self.source_address in adapter.source_address) + + def test_behavior_assumption(self): + """This is a test that guardrails the HTTPAdapter behavior so that if the default for + a Session() changes, the assumptions here aren't violated silently.""" + from acme.client import ClientNetwork + # Source address not specified, so the default adapter type should be bound -- this + # test should fail if the default adapter type is changed by requests + net = ClientNetwork(key=None, alg=None) + session = requests.Session() + for scheme in session.adapters.keys(): + client_network_adapter = net.session.adapters.get(scheme) + default_adapter = session.adapters.get(scheme) + self.assertEqual(client_network_adapter.__class__, default_adapter.__class__) if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/setup.py b/acme/setup.py index 72ab5919b..e91c36b3d 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,6 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] From 99d94cc7e8311aaedccbdc9445d8bc7323ededde Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 17:45:25 -0400 Subject: [PATCH 484/631] Make request logs pretty in Python 3 (#5992) Decode response data as UTF-8 to eliminate ugly bytes repr in Python 3. Resolves #5932 --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d4f5a4787..bdc07fb1c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1030,7 +1030,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes if response.headers.get("Content-Type") == DER_CONTENT_TYPE: debug_content = base64.b64encode(response.content) else: - debug_content = response.content + debug_content = response.content.decode("utf-8") logger.debug('Received response:\nHTTP %d\n%s\n\n%s', response.status_code, "\n".join(["{0}: {1}".format(k, v) From 42ef2520432fa8b40307e612ad3de8185af5ffc2 Mon Sep 17 00:00:00 2001 From: James Hiebert Date: Tue, 15 May 2018 10:58:33 -0400 Subject: [PATCH 485/631] Adds a note about Python3 in the Developer Guide (#5998) --- docs/contributing.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index f152a7600..52f08efe0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -40,6 +40,12 @@ Certbot. ./certbot-auto --debug --os-packages-only tools/venv.sh +If you have Python3 available and want to use it, run the ``venv3.sh`` script. + +.. code-block:: shell + + tools/venv3.sh + .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -50,6 +56,8 @@ latter by running: .. code-block:: shell source venv/bin/activate + # or + source venv3/bin/activate After running this command, ``certbot`` and development tools like ``ipdb``, ``ipython``, ``pytest``, and ``tox`` are available in the shell where you ran From 2d68c9b81e2ed437edc7a9175562be5c246d4929 Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Tue, 15 May 2018 11:46:36 -0400 Subject: [PATCH 486/631] Display (None) instead of a bullet for empty lists (#5999) Include a line break before "(None)" to maintain consistency with output for lists that are not empty. Previous result as expected for non-empty lists: >>> _format_list('+', ['one', 'two', 'three']) '\n+ one\n+ two\n+ three' Previous unexpected result for empty lists: >>> _format_list('+', []) '\n+ ' New result as expected (unchanged) for non-empty lists: >>> _format_list('+', ['one', 'two', 'three']) '\n+ one\n+ two\n+ three' New behavior more explicit for empty lists: >>> _format_list('+', []) '\n(None)' Resolves #5886 --- certbot/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index 0ae5b9d7a..8a9a37084 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -340,7 +340,10 @@ def _get_added_removed(after, before): def _format_list(character, strings): """Format list with given character """ - formatted = "{br}{ch} " + "{br}{ch} ".join(strings) + if len(strings) == 0: + formatted = "{br}(None)" + else: + formatted = "{br}{ch} " + "{br}{ch} ".join(strings) return formatted.format( ch=character, br=os.linesep From 802fcc99ee2150989e5a1e860b6aad490691d72f Mon Sep 17 00:00:00 2001 From: signop <39252060+signop@users.noreply.github.com> Date: Tue, 15 May 2018 08:50:09 -0700 Subject: [PATCH 487/631] Add requests-toolbelt hashes to requirements. (#6001) Fixes certbot/certbot#5993 --- letsencrypt-auto-source/letsencrypt-auto | 3 +++ letsencrypt-auto-source/pieces/dependency-requirements.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 853c66023..0b83b08a7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1187,6 +1187,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1e69af9c2..a30a32b48 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -191,3 +191,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 From 307f45f88f09872e5e9f5c1e7f2bae4361a3096d Mon Sep 17 00:00:00 2001 From: James Hiebert Date: Tue, 15 May 2018 12:36:47 -0400 Subject: [PATCH 488/631] Enable checking of type annotation in Nginx plugin (#5997) * Adds type checking for certbot-nginx * First pass at type annotation in certbot-nginx * Ensure linting is disabled for timing imports * Makes container types specific per PR comments * Removes unnecessary lint option --- certbot-nginx/certbot_nginx/configurator.py | 13 +++++--- certbot-nginx/certbot_nginx/http_01.py | 3 +- certbot-nginx/certbot_nginx/nginxparser.py | 2 +- certbot-nginx/certbot_nginx/parser.py | 31 +++++++++++-------- .../certbot_nginx/tests/parser_test.py | 3 +- mypy.ini | 3 ++ 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 378f24b63..118699aa2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -28,6 +28,9 @@ from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 from certbot_nginx import http_01 +from certbot_nginx import obj # pylint: disable=unused-import +from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module + logger = logging.getLogger(__name__) @@ -98,8 +101,8 @@ class NginxConfigurator(common.Installer): # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} - self._wildcard_redirect_vhosts = {} + self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] # Add number of outstanding challenges self._chall_out = 0 @@ -528,7 +531,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() + all_names = set() # type: Set[str] for vhost in self.parser.get_vhosts(): all_names.update(vhost.names) @@ -824,7 +827,7 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives(vhost, stapling_directives) except errors.MisconfigurationError as error: - logger.debug(error) + logger.debug(str(error)) raise errors.PluginError("An error occurred while enabling OCSP " "stapling for {0}.".format(vhost.names)) @@ -892,7 +895,7 @@ class NginxConfigurator(common.Installer): universal_newlines=True) text = proc.communicate()[1] # nginx prints output to stderr except (OSError, ValueError) as error: - logger.debug(error, exc_info=True) + logger.debug(str(error), exc_info=True) raise errors.PluginError( "Unable to run %s -V" % self.conf('ctl')) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index d08a3b1cb..677ce0737 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -10,6 +10,7 @@ from certbot.plugins import common from certbot_nginx import obj from certbot_nginx import nginxparser +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -113,7 +114,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx.obj.Addr` to apply :rtype: list """ - addresses = [] + addresses = [] # type: List[obj.Addr] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 14481e298..8818bc040 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -248,7 +248,7 @@ class UnspacedList(list): """Recurse through the parse tree to figure out if any sublists are dirty""" if self.dirty: return True - return any((isinstance(x, list) and x.is_dirty() for x in self)) + return any((isinstance(x, UnspacedList) and x.is_dirty() for x in self)) def _spaced_position(self, idx): "Convert from indexes in the unspaced list to positions in the spaced one" diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index f06cd17a7..5bc7946dc 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -13,7 +13,7 @@ from certbot import errors from certbot_nginx import obj from certbot_nginx import nginxparser - +from acme.magic_typing import Union, Dict, Set, Any, List, Tuple # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class NginxParser(object): """ def __init__(self, root): - self.parsed = {} + self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -90,7 +90,7 @@ class NginxParser(object): """ servers = self._get_raw_servers() - addr_to_ssl = {} + addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -104,9 +104,10 @@ class NginxParser(object): def _get_raw_servers(self): # pylint: disable=cell-var-from-loop + # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} + servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -727,9 +728,9 @@ def _parse_server_raw(server): :rtype: dict """ - parsed_server = {'addrs': set(), - 'ssl': False, - 'names': set()} + addrs = set() # type: Set[obj.Addr] + ssl = False # type: bool + names = set() # type: Set[str] apply_ssl_to_all_addrs = False @@ -739,17 +740,21 @@ def _parse_server_raw(server): if directive[0] == 'listen': addr = obj.Addr.fromstring(" ".join(directive[1:])) if addr: - parsed_server['addrs'].add(addr) + addrs.add(addr) if addr.ssl: - parsed_server['ssl'] = True + ssl = True elif directive[0] == 'server_name': - parsed_server['names'].update(x.strip('"\'') for x in directive[1:]) + names.update(x.strip('"\'') for x in directive[1:]) elif _is_ssl_on_directive(directive): - parsed_server['ssl'] = True + ssl = True apply_ssl_to_all_addrs = True if apply_ssl_to_all_addrs: - for addr in parsed_server['addrs']: + for addr in addrs: addr.ssl = True - return parsed_server + return { + 'addrs': addrs, + 'ssl': ssl, + 'names': names + } diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 1e9703185..5a37c9565 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -11,6 +11,7 @@ from certbot_nginx import nginxparser from certbot_nginx import obj from certbot_nginx import parser from certbot_nginx.tests import util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods @@ -99,7 +100,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] + paths = [] # type: List[List[int]] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and diff --git a/mypy.ini b/mypy.ini index 449262795..9709fa5ed 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,3 +22,6 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True + +[mypy-certbot_nginx.*] +check_untyped_defs = True From 9bd5b3dda2ca606614892cb178feeab8efe9a1f9 Mon Sep 17 00:00:00 2001 From: dschlessman <36820111+dschlessman@users.noreply.github.com> Date: Tue, 15 May 2018 13:40:32 -0400 Subject: [PATCH 489/631] Issue 5951/check untyped defs apache (#5989) * resolved mypy untyped defs in parser.py * resolved mypy untyped defs in obj.py * removed unused imports * resolved mypy untyped defs in http_01.py * resolved mypy untyped defs in tls_sni_01.py * resolved mypy untyped defs in configurator.py * address mypy too-many-arguments error in override_centos.py * resolved mypy untyped defs in http_01_test.py * removed unused 'conf' argument that was causing mypy method assignment error * address mypy error where same variable reassigned to different type * address pylint and coverage issues * one character space change for formatting * fix required acme version for certbot-apache --- certbot-apache/certbot_apache/configurator.py | 24 ++++++++++--------- certbot-apache/certbot_apache/http_01.py | 5 ++-- certbot-apache/certbot_apache/obj.py | 5 ++-- .../certbot_apache/override_centos.py | 4 ++-- certbot-apache/certbot_apache/parser.py | 11 +++++---- .../certbot_apache/tests/configurator_test.py | 9 +++---- .../certbot_apache/tests/http_01_test.py | 4 ++-- certbot-apache/certbot_apache/tests/util.py | 5 ---- certbot-apache/certbot_apache/tls_sni_01.py | 3 ++- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 2 +- mypy.ini | 3 +++ 12 files changed, 39 insertions(+), 38 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 03ba05bb0..bb82a9d3f 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -13,11 +13,13 @@ import zope.component import zope.interface from acme import challenges +from acme.magic_typing import DefaultDict, Dict, List, Set # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot import util +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge # pylint: disable=unused-import from certbot.plugins import common from certbot.plugins.util import path_surgery @@ -150,14 +152,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = dict() + self.assoc = dict() # type: Dict[str, obj.VirtualHost] # Outstanding challenges - self._chall_out = set() + self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = dict() + self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) + self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] # These will be set in the prepare function self.parser = None @@ -659,7 +661,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: set """ - all_names = set() + all_names = set() # type: Set[str] vhost_macro = [] @@ -800,8 +802,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} - internal_paths = defaultdict(set) + file_paths = {} # type: Dict[str, str] + internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -1239,7 +1241,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self.parser.parsed_in_current(ssl_fp): self.parser.parse_file(ssl_fp) except IOError: - logger.fatal("Error writing/reading to file in make_vhost_ssl") + logger.critical("Error writing/reading to file in make_vhost_ssl", exc_info=True) raise errors.PluginError("Unable to write/read in make_vhost_ssl") if sift: @@ -1327,7 +1329,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): try: span_val = self.aug.span(vhost.path) except ValueError: - logger.fatal("Error while reading the VirtualHost %s from " + logger.critical("Error while reading the VirtualHost %s from " "file %s", vhost.name, vhost.filep, exc_info=True) raise errors.PluginError("Unable to read VirtualHost from file") span_filep = span_val[0] @@ -1770,7 +1772,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) + rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -1864,7 +1866,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] + rewrite_rule_args = [] # type: List[str] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index cce93a646..37545e9cc 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -2,9 +2,10 @@ import logging import os +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot import errors - from certbot.plugins import common +from certbot_apache.obj import VirtualHost # pylint: disable=unused-import logger = logging.getLogger(__name__) @@ -51,7 +52,7 @@ class ApacheHttp01(common.TLSSNI01): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() + self.moded_vhosts = set() # type: Set[VirtualHost] def perform(self): """Perform all HTTP-01 challenges.""" diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index fcf3bfe08..290979f27 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -1,6 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot.plugins import common @@ -140,7 +141,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def get_names(self): """Return a set of all names.""" - all_names = set() + all_names = set() # type: Set[str] all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -251,7 +252,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() + already_found = set() # type: Set[str] for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index 6e75e361d..0b6b12b96 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -47,10 +47,10 @@ class CentOSParser(parser.ApacheParser): self.sysconfig_filep = "/etc/sysconfig/httpd" super(CentOSParser, self).__init__(*args, **kwargs) - def update_runtime_variables(self, *args, **kwargs): + def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(CentOSParser, self).update_runtime_variables(*args, **kwargs) + super(CentOSParser, self).update_runtime_variables() self.parse_sysconfig_var() def parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index d7da1e55e..43878eda2 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -9,6 +9,7 @@ import sys import six +from acme.magic_typing import Dict, List, Set # pylint: disable=unused-import, no-name-in-module from certbot import errors logger = logging.getLogger(__name__) @@ -38,9 +39,9 @@ class ApacheParser(object): # issues with aug.load() after adding new files / defines to parse tree self.configurator = configurator - self.modules = set() - self.parser_paths = {} - self.variables = {} + self.modules = set() # type: Set[str] + self.parser_paths = {} # type: Dict[str, List[str]] + self.variables = {} # type: Dict[str, str] self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -119,7 +120,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = set() + mods = set() # type: Set[str] matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -408,7 +409,7 @@ class ApacheParser(object): else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] + ordered_matches = [] # type: List[str] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index e33e16843..23c1ee82b 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -353,14 +353,11 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.find_dir = mock_find_dir mock_add.reset_mock() - self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access - tried_to_add = [] for a in mock_add.call_args_list: - tried_to_add.append(a[0][1] == "Include" and - a[0][2] == self.config.mod_ssl_conf) - # Include shouldn't be added, as patched find_dir "finds" existing one - self.assertFalse(any(tried_to_add)) + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \ + # pragma: no cover def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index dc1ca34d6..98bf412ae 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -4,12 +4,12 @@ import os import unittest from acme import challenges +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.tests import acme_util - from certbot_apache.tests import util @@ -23,7 +23,7 @@ class ApacheHttp01Test(util.ApacheTest): super(ApacheHttp01Test, self).setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] + self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo, and diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 1daaa00c5..6d3cfa109 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -87,7 +87,6 @@ class ParserTest(ApacheTest): def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-locals config_path, vhost_path, config_dir, work_dir, version=(2, 4, 7), - conf=None, os_info="generic", conf_vhost_path=None): """Create an Apache Configurator with the specified options. @@ -133,10 +132,6 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc config_class = configurator.ApacheConfigurator config = config_class(config=mock_le_config, name="apache", version=version) - # This allows testing scripts to set it a bit more - # quickly - if conf is not None: - config.conf = conf # pragma: no cover config.prepare() return config diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index 549feb17d..65230cdcb 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -3,6 +3,7 @@ import os import logging +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot.plugins import common from certbot.errors import PluginError, MissingCommandlineFlag @@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01): :rtype: set """ - addrs = set() + addrs = set() # type: Set[obj.Addr] config_text = "\n" for achall in self.achalls: diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 8368d266e..724b61d3f 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 +-e acme[dev] certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0c20da7c..0e4304300 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -9,7 +9,7 @@ version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', + 'acme>0.24.0', 'certbot>=0.21.1', 'mock', 'python-augeas', diff --git a/mypy.ini b/mypy.ini index 9709fa5ed..7e08a1f99 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,9 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-certbot_apache.*] +check_untyped_defs = True + [mypy-certbot_dns_dnsimple.*] check_untyped_defs = True From 751f9843b482f276bbe6975eaa52be86059833e6 Mon Sep 17 00:00:00 2001 From: GmH Date: Tue, 15 May 2018 14:22:09 -0400 Subject: [PATCH 490/631] fixed issue #5974 for certbot-dns-route53 (#5984) * fixed issue #5974 for certbot-dns-route53 * fixed issue #5967 for certbot-dns-digitalocean * update to use acme.magic_typing and DefaultDict class * added no-name-in-module identifier, for issue #5974 * added unused-import identifier to disable option, for issue #5974 --- certbot-dns-route53/certbot_dns_route53/dns_route53.py | 4 +++- certbot-dns-route53/local-oldest-requirements.txt | 2 +- certbot-dns-route53/setup.py | 2 +- mypy.ini | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 27d185656..f71935de2 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -11,6 +11,8 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from acme.magic_typing import DefaultDict, List, Dict # pylint: disable=unused-import, no-name-in-module + logger = logging.getLogger(__name__) INSTRUCTIONS = ( @@ -34,7 +36,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) + self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt index 8368d266e..724b61d3f 100644 --- a/certbot-dns-route53/local-oldest-requirements.txt +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 +-e acme[dev] certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 84b701638..083cd15ae 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -8,7 +8,7 @@ version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', + 'acme>0.24.0', 'certbot>=0.21.1', 'boto3', 'mock', diff --git a/mypy.ini b/mypy.ini index 7e08a1f99..506c253a8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -26,5 +26,11 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True +[mypy-certbot_dns_route53.*] +check_untyped_defs = True + +[mypy-certbot_dns_digitalocean.*] +check_untyped_defs = True + [mypy-certbot_nginx.*] check_untyped_defs = True From 722dac86d558af58c1aed27ad9d2b71bfee946ae Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Tue, 15 May 2018 15:50:47 -0400 Subject: [PATCH 491/631] Fix crash when email submission endpoint unavailable (#6002) * Fix crash when email submission endpoint unavailable Handle KeyError and ValueError so that if the email submission endpoint goes down, Certbot can still run. Add tests to eff_test.py: - simulate non-JSON response as described in issue #5858 - simulate JSON response without 'status' element Non-JSON response throws an uncaught ValueError when attempting to decode as JSON. A JSON response missing the 'status' element throws an uncaught KeyError when checking whether status is True or False. Teach _check_response to handle ValueError and KeyError and report an issue to the user. Rewrite if statement as assertion with try-except block to make error handling consistent within the function. Update test_not_ok to make mocked raise_for_status function raise a requests.exceptions.HTTPError. Resolves #5858 * Update PR with requested changes - Use `if` instead of `assert` to check `status` element of response JSON - Handle KeyError and ValueError in the same way - Import requests at the beginning of eff_test.py - Clear JSON in test case in a more idiomatic way --- certbot/eff.py | 9 ++++++--- certbot/tests/eff_test.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/certbot/eff.py b/certbot/eff.py index 746261faa..b047c0b97 100644 --- a/certbot/eff.py +++ b/certbot/eff.py @@ -71,11 +71,14 @@ def _check_response(response): """ logger.debug('Received response:\n%s', response.content) - if response.ok: - if not response.json()['status']: + try: + response.raise_for_status() + if response.json()['status'] == False: _report_failure('your e-mail address appears to be invalid') - else: + except requests.exceptions.HTTPError: _report_failure() + except (ValueError, KeyError): + _report_failure('there was a problem with the server response') def _report_failure(reason=None): diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index 160af1993..8d0d5778c 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,4 +1,5 @@ """Tests for certbot.eff.""" +import requests import unittest import mock @@ -118,11 +119,28 @@ class SubscribeTest(unittest.TestCase): @test_util.patch_get_utility() def test_not_ok(self, mock_get_utility): self.response.ok = False + self.response.raise_for_status.side_effect = requests.exceptions.HTTPError self._call() # pylint: disable=no-value-for-parameter actual = self._get_reported_message(mock_get_utility) unexpected_part = 'because' self.assertFalse(unexpected_part in actual) + @test_util.patch_get_utility() + def test_response_not_json(self, mock_get_utility): + self.response.json.side_effect = ValueError() + self._call() # pylint: disable=no-value-for-parameter + actual = self._get_reported_message(mock_get_utility) + expected_part = 'problem' + self.assertTrue(expected_part in actual) + + @test_util.patch_get_utility() + def test_response_json_missing_status_element(self, mock_get_utility): + self.json.clear() + self._call() # pylint: disable=no-value-for-parameter + actual = self._get_reported_message(mock_get_utility) + expected_part = 'problem' + self.assertTrue(expected_part in actual) + def _get_reported_message(self, mock_get_utility): self.assertTrue(mock_get_utility().add_message.called) return mock_get_utility().add_message.call_args[0][0] From 41e1976c178dac8875584e8f11d68e28515e0a01 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 May 2018 06:24:14 -0700 Subject: [PATCH 492/631] Fix noisy tests (#6004) * Fixes #5570. The issue is calls to atexit aren't mocked out. During the tests there are many repeated calls registering functions to be called when the process exits so when the tests finishes, it prints a ton of output from running those registered functions. This suppresses that by mocking out atexit. * Mock at a lower level. This ensures we don't mess with any other mocks in this test class by mocking at the lowest level we can. Other tests shouldn't be mocking out specific internals of functions in other modules, so this should work just fine. --- certbot/tests/main_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 22653ca3a..0986ff060 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -626,7 +626,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met toy_stdout = stdout if stdout else six.StringIO() with mock.patch('certbot.main.sys.stdout', new=toy_stdout): with mock.patch('certbot.main.sys.stderr') as stderr: - ret = main.main(args[:]) # NOTE: parser can alter its args! + with mock.patch("certbot.util.atexit"): + ret = main.main(args[:]) # NOTE: parser can alter its args! return ret, toy_stdout, stderr def test_no_flags(self): From 20418cdd6899b0ac5c0e7669bdc69ed135594276 Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Thu, 17 May 2018 09:52:11 -0400 Subject: [PATCH 493/631] Fixed #5859 (#6011) --- tests/boulder-fetch.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index fc9cbaae7..d513ec064 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,12 +17,6 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -# If we're testing against ACMEv2, we need to use a newer boulder config for -# now. See https://github.com/letsencrypt/boulder#quickstart. -if [ "$BOULDER_INTEGRATION" = "v2" ]; then - sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml -fi - docker-compose up -d set +x # reduce verbosity while waiting for boulder From 1be1bd92115101e019d71a23ee14e7bfadf72226 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 May 2018 09:23:05 -0700 Subject: [PATCH 494/631] remove PYTHONPATH (#6016) --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 38d4e6ae1..8c4d6c38d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,6 @@ commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py setenv = - PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 [testenv:py27-oldest] From 9b2862ebb0f1618161fcf5726ac4de812b959ac9 Mon Sep 17 00:00:00 2001 From: TyrannosourceExe <38411281+TyrannosourceExe@users.noreply.github.com> Date: Thu, 17 May 2018 19:03:01 -0400 Subject: [PATCH 495/631] 3692 --dry-run expiration emails (#6015) * If --dry-run is used and there exists no staging account, create account with no email * added unit testing of dry-run to ensure certbot does not ask the user to create an email, and that certbot creates an account with no email --- certbot/client.py | 4 ++++ certbot/tests/client_test.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 45dc9c63b..cdfbdc252 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -155,6 +155,10 @@ def register(config, account_storage, tos_cb=None): if not config.dry_run: logger.info("Registering without email!") + # If --dry-run is used, and there is no staging account, create one with no email. + if config.dry_run: + config.email = None + # Each new registration shall use a fresh new key key = jose.JWKRSA(key=jose.ComparableRSAKey( rsa.generate_private_key( diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6add141d4..a4425bca9 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -12,7 +12,6 @@ from certbot import util import certbot.tests.util as test_util - KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") @@ -92,6 +91,20 @@ class RegisterTest(test_util.ConfigTestCase): mock_logger.info.assert_called_once_with(mock.ANY) self.assertTrue(mock_handle.called) + @mock.patch("certbot.account.report_new_account") + @mock.patch("certbot.client.display_ops.get_email") + def test_dry_run_no_staging_account(self, _rep, mock_get_email): + """Tests dry-run for no staging account, expect account created with no email""" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + with mock.patch("certbot.eff.handle_subscription"): + with mock.patch("certbot.account.report_new_account"): + self.config.dry_run = True + self._call() + # check Certbot did not ask the user to provide an email + self.assertFalse(mock_get_email.called) + # check Certbot created an account with no email. Contact should return empty + self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) + def test_unsupported_error(self): from acme import messages msg = "Test" @@ -105,6 +118,7 @@ class RegisterTest(test_util.ConfigTestCase): class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot.client.Client tests.""" + def setUp(self): super(ClientTestCommon, self).setUp() self.config.no_verify_ssl = False @@ -124,6 +138,7 @@ class ClientTestCommon(test_util.ConfigTestCase): class ClientTest(ClientTestCommon): """Tests for certbot.client.Client.""" + def setUp(self): super(ClientTest, self).setUp() @@ -286,10 +301,10 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, - mock_storage, mock_obtain_certificate): + mock_storage, mock_obtain_certificate): domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), - mock.MagicMock(), mock.MagicMock(), None) + mock.MagicMock(), mock.MagicMock(), None) self.client.config.dry_run = False self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) @@ -318,8 +333,8 @@ class ClientTest(ClientTestCommon): candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") mock_parser.verb = "certonly" mock_parser.args = ["--cert-path", candidate_cert_path, - "--chain-path", candidate_chain_path, - "--fullchain-path", candidate_fullchain_path] + "--chain-path", candidate_chain_path, + "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, @@ -407,6 +422,7 @@ class ClientTest(ClientTestCommon): class EnhanceConfigTest(ClientTestCommon): """Tests for certbot.client.Client.enhance_config.""" + def setUp(self): super(EnhanceConfigTest, self).setUp() From 1239d7a881f7ad88c736aac526a821658cea0f96 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 17 May 2018 20:02:27 -0700 Subject: [PATCH 496/631] check platform with correct python --- certbot-nginx/certbot_nginx/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 72fc5d4de..dfc451202 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -2,11 +2,11 @@ import pkg_resources import platform -if(platform.system() == ('FreeBSD' or 'Darwin')): +if platform.system() in ('FreeBSD', 'Darwin'): server_root_tmp = "/usr/local/etc/nginx" else: server_root_tmp = "/etc/nginx" - + CLI_DEFAULTS = dict( server_root=server_root_tmp, ctl="nginx", From 94bf97b812fac7b56541992ef6ab81a6abedec49 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 02:26:10 -0700 Subject: [PATCH 497/631] Add remaining DNS plugins to mypy.ini and sort (#6018) --- mypy.ini | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy.ini b/mypy.ini index 506c253a8..d00c21ae7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,15 @@ check_untyped_defs = True [mypy-certbot_apache.*] check_untyped_defs = True +[mypy-certbot_dns_cloudflare.*] +check_untyped_defs = True + +[mypy-certbot_dns_cloudxns.*] +check_untyped_defs = True + +[mypy-certbot_dns_digitalocean.*] +check_untyped_defs = True + [mypy-certbot_dns_dnsimple.*] check_untyped_defs = True @@ -29,8 +38,5 @@ check_untyped_defs = True [mypy-certbot_dns_route53.*] check_untyped_defs = True -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - [mypy-certbot_nginx.*] check_untyped_defs = True From 250c0d6691f8fce9568fa13df0dc256181403cfe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 06:05:26 -0700 Subject: [PATCH 498/631] cd before running tests (#6017) When importing a module, Python first searches the current directory. See https://docs.python.org/3/tutorial/modules.html#the-module-search-path. This means that running something like `import certbot` from the root of the Certbot repo will use the local Certbot files regardless of the version installed on the system or virtual environment. Normally this behavior is fine because the local files are what we want to test, however, during our "oldest" tests, we test against older versions of our packages to make sure we're keeping compatibility. To make sure our tests use the correct versions, this commit has our tests cd to an empty temporary directory before running tests. We also had to change the package names given to pytest to be the names used in Python to import the package rather than the name of the files locally to accommodate this. --- tools/install_and_test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 59832cbc3..819f683aa 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -12,12 +12,18 @@ else pip_install="$(dirname $0)/pip_install_editable.sh" fi +temp_cwd=$(mktemp -d) +trap "rm -rf $temp_cwd" EXIT + set -x for requirement in "$@" ; do $pip_install $requirement pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] + pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names if [ $pkg = "." ]; then pkg="certbot" fi + cd "$temp_cwd" pytest --numprocesses auto --quiet --pyargs $pkg + cd - done From 36dfd065037678bf38b7adb95ab16ed951ebbd18 Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Fri, 18 May 2018 09:28:17 -0400 Subject: [PATCH 499/631] Prepare certbot module for mypy check untyped defs (#6005) * Prepare certbot module for mypy check untyped defs * Fix #5952 * Bump mypy to version 0.600 and fix associated bugs * Fix pylint bugs after introducing mypy * Implement Brad's suggestions * Reenabling pylint and adding nginx mypy back --- acme/acme/challenges.py | 4 +- acme/acme/crypto_util.py | 22 ++-- acme/acme/magic_typing.py | 5 +- certbot/auth_handler.py | 24 +++-- certbot/cert_manager.py | 5 +- certbot/cli.py | 54 ++++++---- certbot/client.py | 16 +-- certbot/crypto_util.py | 128 ++++++++++++------------ certbot/error_handler.py | 16 +-- certbot/hooks.py | 17 ++-- certbot/log.py | 3 +- certbot/main.py | 8 +- certbot/plugins/common.py | 6 +- certbot/plugins/disco.py | 3 +- certbot/plugins/disco_test.py | 3 +- certbot/plugins/manual.py | 5 +- certbot/plugins/selection_test.py | 3 +- certbot/plugins/standalone.py | 29 ++++-- certbot/plugins/standalone_test.py | 19 ++-- certbot/plugins/storage.py | 3 +- certbot/plugins/webroot.py | 17 ++-- certbot/renewal.py | 17 ++-- certbot/reverter.py | 12 ++- certbot/tests/auth_handler_test.py | 4 +- certbot/tests/cli_test.py | 3 +- certbot/tests/display/completer_test.py | 3 +- certbot/tests/error_handler_test.py | 6 +- certbot/tests/hook_test.py | 21 ++-- certbot/tests/log_test.py | 13 +-- certbot/tests/main_test.py | 17 ++-- certbot/tests/reporter_test.py | 18 ++-- certbot/util.py | 14 ++- mypy.ini | 6 ++ setup.py | 2 +- tools/dev_constraints.txt | 2 +- tox.ini | 2 +- 36 files changed, 316 insertions(+), 214 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index b2a4882eb..674f2c38f 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -147,9 +147,9 @@ class KeyAuthorizationChallenge(_TokenChallenge): :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` that will be used to generate `response`. - + :param str typ: type of the challenge """ - + typ = NotImplemented response_cls = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index ad914ca60..d0e203417 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -12,7 +12,8 @@ import josepy as jose from acme import errors # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Callable, Text, Union +from acme.magic_typing import Callable, Union, Tuple, Optional +# pylint: enable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -135,14 +136,23 @@ def probe_sni(name, host, port=443, timeout=300, socket_kwargs = {'source_address': source_address} - host_protocol_agnostic = None if host == '::' or host == '0' else host + host_protocol_agnostic = host + if host == '::' or host == '0': + # https://github.com/python/typeshed/pull/2136 + # while PR is not merged, we need to ignore + host_protocol_agnostic = None try: # pylint: disable=star-args - logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, - " from {0}:{1}".format(source_address[0], source_address[1]) if \ - socket_kwargs else "") - sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs) + logger.debug( + "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, + " from {0}:{1}".format( + source_address[0], + source_address[1] + ) if socket_kwargs else "" + ) + socket_tuple = (host_protocol_agnostic, port) # type: Tuple[Optional[str], int] + sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 555088cf2..471b8dfa9 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -8,6 +8,9 @@ class TypingClass(object): try: # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index caf112c61..e7d658b25 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -8,7 +8,9 @@ import zope.component from acme import challenges from acme import messages - +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +# pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler @@ -117,7 +119,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] + resp = [] # type: Collection[acme.challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -133,10 +135,9 @@ class AuthHandler(object): def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" - all_achalls = [] + all_achalls = [] # type: Collection[challenges.ChallengeResponse] for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) - return all_achalls def _respond(self, aauthzrs, resp, best_effort): @@ -146,7 +147,8 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() + chall_update = dict() \ + # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... @@ -198,7 +200,7 @@ class AuthHandler(object): while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) - all_failed_achalls = set() + all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( aauthzrs, index, chall_update[index]) @@ -424,7 +426,7 @@ def _find_smart_path(challbs, preferences, combinations): # max_cost is now equal to sum(indices) + 1 - best_combo = [] + best_combo = None # Set above completing all of the available challenges best_combo_cost = max_cost @@ -479,7 +481,7 @@ def _report_no_chall_path(challbs): msg += ( " You may need to use an authenticator " "plugin that can do challenges over DNS.") - logger.fatal(msg) + logger.critical(msg) raise errors.AuthorizationError(msg) @@ -522,11 +524,11 @@ def _report_failed_challs(failed_achalls): :class:`certbot.achallenges.AnnotatedChallenge`. """ - problems = dict() + problems = collections.defaultdict(list)\ + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: - problems.setdefault(achall.error.typ, []).append(achall) - + problems[achall.error.typ].append(achall) reporter = zope.component.getUtility(interfaces.IReporter) for achalls in six.itervalues(problems): reporter.add_message( diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d841c1912..d1205835a 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -7,6 +7,7 @@ import re import traceback import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -226,7 +227,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] + acceptable_matches_rv = [] # type: List[str] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -340,7 +341,7 @@ def _report_human_readable(config, parsed_certs): def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] + out = [] # type: List[str] notify = out.append diff --git a/certbot/cli.py b/certbot/cli.py index b71d60055..8a1ad381a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -12,10 +12,14 @@ import sys import configargparse import six import zope.component +import zope.interface from zope.interface import interfaces as zope_interfaces from acme import challenges +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict, Optional +# pylint: enable=unused-import, no-name-in-module import certbot @@ -33,7 +37,7 @@ import certbot.plugins.selection as plugin_selection logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None +helpful_parser = None # type: Optional[HelpfulArgumentParser] # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: @@ -196,17 +200,17 @@ def set_by_cli(var): (CLI or config file) including if the user explicitly set it to the default. Returns False if the variable was assigned a default value. """ - detector = set_by_cli.detector - if detector is None: + detector = set_by_cli.detector # type: ignore + if detector is None and helpful_parser is not None: # Setup on first run: `detector` is a weird version of config in which # the default value of every attribute is wrangled to be boolean-false plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = helpful_parser.args + [helpful_parser.verb] - detector = set_by_cli.detector = prepare_and_parse_args( + detector = set_by_cli.detector = prepare_and_parse_args( # type: ignore plugins, reconstructed_args, detect_defaults=True) # propagate plugin requests: eg --standalone modifies config.authenticator - detector.authenticator, detector.installer = ( + detector.authenticator, detector.installer = ( # type: ignore plugin_selection.cli_plugin_requests(detector)) if not isinstance(getattr(detector, var), _Default): @@ -220,7 +224,10 @@ def set_by_cli(var): return True return False + # static housekeeping var +# functions attributed are not supported by mypy +# https://github.com/python/mypy/issues/2087 set_by_cli.detector = None # type: ignore @@ -236,8 +243,10 @@ def has_default_value(option, value): :rtype: bool """ - return (option in helpful_parser.defaults and - helpful_parser.defaults[option] == value) + if helpful_parser is not None: + return (option in helpful_parser.defaults and + helpful_parser.defaults[option] == value) + return False def option_was_set(option, value): @@ -254,11 +263,12 @@ def option_was_set(option, value): def argparse_type(variable): - "Return our argparse type function for a config variable (default: str)" + """Return our argparse type function for a config variable (default: str)""" # pylint: disable=protected-access - for action in helpful_parser.parser._actions: - if action.type is not None and action.dest == variable: - return action.type + if helpful_parser is not None: + for action in helpful_parser.parser._actions: + if action.type is not None and action.dest == variable: + return action.type return str def read_file(filename, mode="rb"): @@ -291,10 +301,12 @@ def flag_default(name): def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" + # pylint: disable=no-member if hidden: return argparse.SUPPRESS else: - return interfaces.IConfig[name].__doc__ + field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + return field.__doc__ class HelpfulArgumentGroup(object): @@ -473,7 +485,7 @@ class HelpfulArgumentParser(object): HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"] plugin_names = list(plugins) - self.help_topics = HELP_TOPICS + plugin_names + [None] + self.help_topics = HELP_TOPICS + plugin_names + [None] # type: ignore self.detect_defaults = detect_defaults self.args = args @@ -492,8 +504,11 @@ class HelpfulArgumentParser(object): short_usage = self._usage_string(plugins, self.help_arg) self.visible_topics = self.determine_help_topics(self.help_arg) - self.groups = {} # elements are added by .add_group() - self.defaults = {} # elements are added by .parse_args() + + # elements are added by .add_group() + self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + # elements are added by .parse_args() + self.defaults = {} # type: Dict[str, Any] self.parser = configargparse.ArgParser( prog="certbot", @@ -805,7 +820,6 @@ class HelpfulArgumentParser(object): if self.help_arg: for v in verbs: self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"]) - return HelpfulArgumentGroup(self, topic) def add_plugin_args(self, plugins): @@ -1296,14 +1310,14 @@ def _paths_parser(helpful): verb = helpful.help_arg cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - section = ["paths", "install", "revoke", "certonly", "manage"] + sections = ["paths", "install", "revoke", "certonly", "manage"] if verb == "certonly": - add(section, "--cert-path", type=os.path.abspath, + add(sections, "--cert-path", type=os.path.abspath, default=flag_default("auth_cert_path"), help=cph) elif verb == "revoke": - add(section, "--cert-path", type=read_file, required=True, help=cph) + add(sections, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, help=cph) + add(sections, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): diff --git a/certbot/client.py b/certbot/client.py index cdfbdc252..1932ab83e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,7 +5,9 @@ import os import platform from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +# https://github.com/python/typeshed/blob/master/third_party/ +# 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi +from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key # type: ignore import josepy as jose import OpenSSL import zope.component @@ -160,11 +162,11 @@ def register(config, account_storage, tos_cb=None): config.email = None # Each new registration shall use a fresh new key - key = jose.JWKRSA(key=jose.ComparableRSAKey( - rsa.generate_private_key( + rsa_key = generate_private_key( public_exponent=65537, key_size=config.rsa_key_size, - backend=default_backend()))) + backend=default_backend()) + key = jose.JWKRSA(key=jose.ComparableRSAKey(rsa_key)) acme = acme_from_config_key(config, key) # TODO: add phone? regr = perform_registration(acme, config, tos_cb) @@ -609,8 +611,10 @@ def validate_key_csr(privkey, csr=None): if csr.form == "der": csr_obj = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, csr.data) - csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem") + cert_buffer = OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_obj + ) + csr = util.CSR(csr.file, cert_buffer, "pem") # If CSR is provided, it must be readable and valid. if csr.data and not crypto_util.valid_csr(csr.data): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 756bd7565..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -8,15 +8,18 @@ import hashlib import logging import os -import OpenSSL + import pyrfc3339 import six import zope.component +from OpenSSL import crypto +from OpenSSL import SSL # type: ignore from cryptography.hazmat.backends import default_backend -from cryptography import x509 # type: ignore +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util - +from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot import util @@ -47,7 +50,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): try: key_pem = make_key(key_size) except ValueError as err: - logger.exception(err) + logger.error("", exc_info=True) raise err config = zope.component.getUtility(interfaces.IConfig) @@ -111,11 +114,11 @@ def valid_csr(csr): """ try: - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) return req.verify(req.get_pubkey()) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -129,13 +132,13 @@ def csr_matches_pubkey(csr, privkey): :rtype: bool """ - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) - pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) + pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey) try: return req.verify(pkey) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -145,26 +148,26 @@ def import_csr_file(csrfile, data): :param str csrfile: CSR filename :param str data: contents of the CSR file - :returns: (`OpenSSL.crypto.FILETYPE_PEM`, + :returns: (`crypto.FILETYPE_PEM`, util.CSR object representing the CSR, list of domains requested in the CSR) :rtype: tuple """ - PEM = OpenSSL.crypto.FILETYPE_PEM - load = OpenSSL.crypto.load_certificate_request + PEM = crypto.FILETYPE_PEM + load = crypto.load_certificate_request try: # Try to parse as DER first, then fall back to PEM. - csr = load(OpenSSL.crypto.FILETYPE_ASN1, data) - except OpenSSL.crypto.Error: + csr = load(crypto.FILETYPE_ASN1, data) + except crypto.Error: try: csr = load(PEM, data) - except OpenSSL.crypto.Error: + except crypto.Error: raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) domains = _get_names_from_loaded_cert_or_req(csr) # Internally we always use PEM, so re-encode as PEM before returning. - data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr) + data_pem = crypto.dump_certificate_request(PEM, csr) return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains @@ -178,9 +181,9 @@ def make_key(bits): """ assert bits >= 1024 # XXX - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) - return OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + return crypto.dump_privatekey(crypto.FILETYPE_PEM, key) def valid_privkey(privkey): @@ -193,9 +196,9 @@ def valid_privkey(privkey): """ try: - return OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, privkey).check() - except (TypeError, OpenSSL.crypto.Error): + return crypto.load_privatekey( + crypto.FILETYPE_PEM, privkey).check() + except (TypeError, crypto.Error): return False @@ -224,13 +227,14 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain, 'rb') as chain: - chain, _ = pyopenssl_load_certificate(chain.read()) - with open(renewable_cert.cert, 'rb') as cert: - cert = x509.load_pem_x509_certificate(cert.read(), default_backend()) + with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] + chain, _ = pyopenssl_load_certificate(chain_file.read()) + with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) hash_name = cert.signature_hash_algorithm.name - OpenSSL.crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, OpenSSL.crypto.Error) as e: + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) @@ -246,11 +250,11 @@ def verify_cert_matches_priv_key(cert_path, key_path): :raises errors.Error: If they don't match. """ try: - context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + context = SSL.Context(SSL.SSLv23_METHOD) context.use_certificate_file(cert_path) context.use_privatekey_file(key_path) context.check_privatekey() - except (IOError, OpenSSL.SSL.Error) as e: + except (IOError, SSL.Error) as e: error_str = "verifying the cert located at {0} matches the \ private key located at {1} has failed. \ Details: {2}".format(cert_path, @@ -267,12 +271,12 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain) as chain: - chain = chain.read() - with open(renewable_cert.cert) as cert: - cert = cert.read() - with open(renewable_cert.fullchain) as fullchain: - fullchain = fullchain.read() + with open(renewable_cert.chain) as chain_file: # type: IO[str] + chain = chain_file.read() + with open(renewable_cert.cert) as cert_file: # type: IO[str] + cert = cert_file.read() + with open(renewable_cert.fullchain) as fullchain_file: # type: IO[str] + fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" error_str = error_str.format(renewable_cert.lineagename) @@ -294,43 +298,43 @@ def pyopenssl_load_certificate(data): openssl_errors = [] - for file_type in (OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1): + for file_type in (crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1): try: - return OpenSSL.crypto.load_certificate(file_type, data), file_type - except OpenSSL.crypto.Error as error: # TODO: other errors? + return crypto.load_certificate(file_type, data), file_type + except crypto.Error as error: # TODO: other errors? openssl_errors.append(error) raise errors.Error("Unable to load: {0}".format(",".join( str(error) for error in openssl_errors))) def _load_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): try: return load_func(typ, cert_or_req_str) - except OpenSSL.crypto.Error as error: - logger.exception(error) + except crypto.Error: + logger.error("", exc_info=True) raise def _get_sans_from_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): # pylint: disable=protected-access return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req( cert_or_req_str, load_func, typ)) -def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_sans_from_cert(cert, typ=crypto.FILETYPE_PEM): """Get a list of Subject Alternative Names from a certificate. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of Subject Alternative Names. :rtype: list """ return _get_sans_from_cert_or_req( - cert, OpenSSL.crypto.load_certificate, typ) + cert, crypto.load_certificate, typ) def _get_names_from_cert_or_req(cert_or_req, load_func, typ): @@ -343,24 +347,24 @@ def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) -def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_names_from_cert(csr, typ=crypto.FILETYPE_PEM): """Get a list of domains from a cert, including the CN if it is set. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of domain names. :rtype: list """ return _get_names_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate, typ) + csr, crypto.load_certificate, typ) -def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): +def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. - :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :param list chain: List of `crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). """ @@ -378,7 +382,7 @@ def notBefore(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notBefore) + return _notAfterBefore(cert_path, crypto.X509.get_notBefore) def notAfter(cert_path): @@ -390,15 +394,15 @@ def notAfter(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notAfter) + return _notAfterBefore(cert_path, crypto.X509.get_notAfter) def _notAfterBefore(cert_path, method): """Internal helper function for finding notbefore/notafter. :param str cert_path: path to a cert in PEM format - :param function method: one of ``OpenSSL.crypto.X509.get_notBefore`` - or ``OpenSSL.crypto.X509.get_notAfter`` + :param function method: one of ``crypto.X509.get_notBefore`` + or ``crypto.X509.get_notAfter`` :returns: the notBefore or notAfter value from the cert at cert_path :rtype: :class:`datetime.datetime` @@ -406,7 +410,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name with open(cert_path) as f: - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -443,7 +447,7 @@ def cert_and_chain_from_fullchain(fullchain_pem): :rtype: tuple """ - cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() + cert = crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) diff --git a/certbot/error_handler.py b/certbot/error_handler.py index e2737711e..5e72f8153 100644 --- a/certbot/error_handler.py +++ b/certbot/error_handler.py @@ -5,6 +5,10 @@ import os import signal import traceback +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Callable, Dict, List, Union +# pylint: enable=unused-import, no-name-in-module + from certbot import errors logger = logging.getLogger(__name__) @@ -56,9 +60,9 @@ class ErrorHandler(object): def __init__(self, func=None, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] - self.prev_handlers = {} - self.received_signals = [] + self.funcs = [] # type: List[Callable[[], Any]] + self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] + self.received_signals = [] # type: List[int] if func is not None: self.register(func, *args, **kwargs) @@ -88,6 +92,7 @@ class ErrorHandler(object): return retval def register(self, func, *args, **kwargs): + # type: (Callable, *Any, **Any) -> None """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error @@ -101,9 +106,8 @@ class ErrorHandler(object): while self.funcs: try: self.funcs[-1]() - except Exception as error: # pylint: disable=broad-except - logger.error("Encountered exception during recovery") - logger.exception(error) + except Exception: # pylint: disable=broad-except + logger.error("Encountered exception during recovery: ", exc_info=True) self.funcs.pop() def _set_signal_handlers(self): diff --git a/certbot/hooks.py b/certbot/hooks.py index b5c9046e9..d5239a437 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -6,6 +6,7 @@ import os from subprocess import Popen, PIPE +from acme.magic_typing import Set, List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util @@ -76,7 +77,8 @@ def pre_hook(config): if cmd: _run_pre_hook_if_necessary(cmd) -pre_hook.already = set() # type: ignore + +executed_pre_hooks = set() # type: Set[str] def _run_pre_hook_if_necessary(command): @@ -88,12 +90,12 @@ def _run_pre_hook_if_necessary(command): :param str command: pre-hook to be run """ - if command in pre_hook.already: + if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: logger.info("Running pre-hook command: %s", command) _run_hook(command) - pre_hook.already.add(command) + executed_pre_hooks.add(command) def post_hook(config): @@ -127,7 +129,8 @@ def post_hook(config): logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) -post_hook.eventually = [] # type: ignore + +post_hooks = [] # type: List[str] def _run_eventually(command): @@ -139,13 +142,13 @@ def _run_eventually(command): :param str command: post-hook to register to be run """ - if command not in post_hook.eventually: - post_hook.eventually.append(command) + if command not in post_hooks: + post_hooks.append(command) def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" - for cmd in post_hook.eventually: + for cmd in post_hooks: logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) diff --git a/certbot/log.py b/certbot/log.py index face93cb3..89626af99 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -191,9 +191,8 @@ class MemoryHandler(logging.handlers.MemoryHandler): only happens when flush(force=True) is called. """ - def __init__(self, target=None): + def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - capacity = float('inf') super(MemoryHandler, self).__init__(capacity, target=target) def close(self): diff --git a/certbot/main.py b/certbot/main.py index 8a9a37084..6c1d82793 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -11,6 +11,7 @@ import josepy as jose import zope.component from acme import errors as acme_errors +from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module import certbot @@ -520,8 +521,8 @@ def _determine_account(config): config, account_storage, tos_cb=_tos_cb) except errors.MissingCommandlineFlag: raise - except errors.Error as error: - logger.debug(error, exc_info=True) + except errors.Error: + logger.debug("", exc_info=True) raise errors.Error( "Unable to register an account with ACME server") @@ -1271,7 +1272,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) + displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ + # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 147d9e21a..ee1af4978 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -11,6 +11,8 @@ import zope.interface from josepy import util as jose_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import constants from certbot import crypto_util from certbot import errors @@ -331,8 +333,8 @@ class ChallengePerformer(object): def __init__(self, configurator): self.configurator = configurator - self.achalls = [] - self.indices = [] + self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.indices = [] # type: List[int] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 062c11650..c33a56785 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -10,6 +10,7 @@ from collections import OrderedDict import zope.interface import zope.interface.verify +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import interfaces @@ -189,7 +190,7 @@ class PluginsRegistry(collections.Mapping): @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} + plugins = {} # type: Dict[str, PluginEntryPoint] # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( diff --git a/certbot/plugins/disco_test.py b/certbot/plugins/disco_test.py index 220b902b3..720b90b16 100644 --- a/certbot/plugins/disco_test.py +++ b/certbot/plugins/disco_test.py @@ -8,6 +8,7 @@ import pkg_resources import six import zope.interface +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces @@ -250,7 +251,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] + order = [] # type: List[str] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 614449d34..53533d35a 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -5,7 +5,9 @@ import zope.component import zope.interface from acme import challenges +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import interfaces from certbot import errors from certbot import hooks @@ -98,7 +100,8 @@ when it receives a TLS ClientHello with the SNI extension set to super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() + self.env = dict() \ + # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None @classmethod diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index 4112810a2..ab480544a 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -6,6 +6,7 @@ import unittest import mock import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.display import util as display_util from certbot.tests import util as test_util from certbot import interfaces @@ -47,7 +48,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] + self.ifaces = [] # type: List[interfaces.IPlugin] def _call(self): from certbot.plugins.selection import pick_plugin diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 817403bd3..cb2e69511 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -3,6 +3,8 @@ import argparse import collections import logging import socket +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import OpenSSL import six @@ -10,7 +12,10 @@ import zope.interface from acme import challenges from acme import standalone as acme_standalone +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, Set, Tuple, List, Type, TYPE_CHECKING +from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces @@ -18,6 +23,11 @@ from certbot.plugins import common logger = logging.getLogger(__name__) +if TYPE_CHECKING: + ServedType = DefaultDict[ + acme_standalone.BaseDualNetworkedServers, + Set[achallenges.KeyAuthorizationAnnotatedChallenge] + ] class ServerManager(object): """Standalone servers manager. @@ -33,7 +43,7 @@ class ServerManager(object): """ def __init__(self, certs, http_01_resources): - self._instances = {} + self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] self.certs = certs self.http_01_resources = http_01_resources @@ -59,7 +69,8 @@ class ServerManager(object): address = (listenaddr, port) try: if challenge_type is challenges.TLSSNI01: - servers = acme_standalone.TLSSNI01DualNetworkedServers(address, self.certs) + servers = acme_standalone.TLSSNI01DualNetworkedServers( + address, self.certs) # type: acme_standalone.BaseDualNetworkedServers else: # challenges.HTTP01 servers = acme_standalone.HTTP01DualNetworkedServers( address, self.http_01_resources) @@ -103,7 +114,8 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] +SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \ +# type: List[Type[challenges.KeyAuthorizationChallenge]] class SupportedChallengesAction(argparse.Action): @@ -179,14 +191,15 @@ class Authenticator(common.Plugin): self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) - self.served = collections.defaultdict(set) + self.served = collections.defaultdict(set) # type: ServedType # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} - self.http_01_resources = set() + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = set() \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.servers = ServerManager(self.certs, self.http_01_resources) @@ -265,13 +278,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket.errno.EACCES: + if error.socket_error.errno == socket_errors.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - elif error.socket_error.errno == socket.errno.EADDRINUSE: + elif error.socket_error.errno == socket_errors.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 5227bc59e..47f44ff77 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -2,12 +2,18 @@ import argparse import socket import unittest +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import josepy as jose import mock import six +import OpenSSL.crypto # pylint: disable=unused-import + from acme import challenges +from acme import standalone as acme_standalone # pylint: disable=unused-import +from acme.magic_typing import Dict, Tuple, Set # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -21,8 +27,9 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot.plugins.standalone import ServerManager - self.certs = {} - self.http_01_resources = {} + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = {} \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): @@ -159,7 +166,7 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno @@ -174,7 +181,7 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE self.assertRaises(errors.PluginError, self._fail_perform, errno) self._assert_correct_yesno_call(mock_yesno) @@ -184,11 +191,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket.errno.EACCES + errno = socket_errors.EACCES self.assertRaises(errors.PluginError, self._fail_perform, errno) def test_perform_unexpected_socket_error(self): - errno = socket.errno.ENOTCONN + errno = socket_errors.ENOTCONN self.assertRaises( errors.StandaloneBindError, self._fail_perform, errno) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index a0c3f8564..9472a1ebb 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -3,6 +3,7 @@ import json import logging import os +from acme.magic_typing import Any, Dict # pylint: disable=unused-import, no-name-in-module from certbot import errors logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() + data = dict() # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 6328b16ef..5d0d7d586 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -10,8 +10,12 @@ import six import zope.component import zope.interface -from acme import challenges +from acme import challenges # pylint: disable=unused-import +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Set, DefaultDict, List +# pylint: enable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import cli from certbot import errors from certbot import interfaces @@ -64,10 +68,11 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} - self.performed = collections.defaultdict(set) + self.full_roots = {} # type: Dict[str, str] + self.performed = collections.defaultdict(set) \ + # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] # stack of dirs successfully created by this authenticator - self._created_dirs = [] + self._created_dirs = [] # type: List[str] def prepare(self): # pylint: disable=missing-docstring pass @@ -156,7 +161,6 @@ to serve all files under specified web root ({0}).""" " --help webroot for examples.") for name, path in path_map.items(): self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH) - logger.debug("Creating root challenges validation dir at %s", self.full_roots[name]) @@ -207,7 +211,6 @@ to serve all files under specified web root ({0}).""" os.umask(old_umask) self.performed[root_path].add(achall) - return response def cleanup(self, achalls): # pylint: disable=missing-docstring @@ -219,7 +222,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] + not_removed = [] # type: List[str] while len(self._created_dirs) > 0: path = self._created_dirs.pop() try: diff --git a/certbot/renewal.py b/certbot/renewal.py index 4651eeb36..0a6568426 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -11,6 +11,8 @@ import zope.component import OpenSSL +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + from certbot import cli from certbot import crypto_util from certbot import errors @@ -59,8 +61,8 @@ def _reconstitute(config, full_path): """ try: renewal_candidate = storage.RenewableCert(full_path, config) - except (errors.CertStorageError, IOError) as exc: - logger.warning(exc) + except (errors.CertStorageError, IOError): + logger.warning("", exc_info=True) logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None @@ -133,14 +135,15 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. + plugin_prefixes = [] # type: List[str] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) - plugin_prefixes = [] else: - plugin_prefixes = [renewalparams["authenticator"]] + plugin_prefixes.append(renewalparams["authenticator"]) - if renewalparams.get("installer", None) is not None: + if renewalparams.get("installer") is not None: plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') for config_item, config_value in six.iteritems(renewalparams): @@ -316,13 +319,13 @@ def report(msgs, category): def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): - out = [] + out = [] # type: List[str] notify = out.append disp = zope.component.getUtility(interfaces.IDisplay) def notify_error(err): """Notify and log errors.""" - notify(err) + notify(str(err)) logger.error(err) if config.dry_run: diff --git a/certbot/reverter.py b/certbot/reverter.py index 15ad1a987..683c0cc32 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -82,8 +82,10 @@ class Reverter(object): self._recover_checkpoint(self.config.temp_checkpoint_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for %s", - self.config.temp_checkpoint_dir) + logger.critical( + "Incomplete or failed recovery for %s", + self.config.temp_checkpoint_dir, + ) raise errors.ReverterError("Unable to revert temporary config") def rollback_checkpoints(self, rollback=1): @@ -123,7 +125,7 @@ class Reverter(object): try: self._recover_checkpoint(cp_dir) except errors.ReverterError: - logger.fatal("Failed to load checkpoint during rollback") + logger.critical("Failed to load checkpoint during rollback") raise errors.ReverterError( "Unable to load checkpoint during rollback") rollback -= 1 @@ -457,7 +459,7 @@ class Reverter(object): self._recover_checkpoint(self.config.in_progress_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for IN_PROGRESS " + logger.critical("Incomplete or failed recovery for IN_PROGRESS " "checkpoint - %s", self.config.in_progress_dir) raise errors.ReverterError( @@ -494,7 +496,7 @@ class Reverter(object): "Certbot probably shut down unexpectedly", os.linesep, path) except (IOError, OSError): - logger.fatal( + logger.critical( "Unable to remove filepaths contained within %s", file_list) raise errors.ReverterError( "Unable to remove filepaths contained within " diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 9a8a13498..76d1df90f 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -10,6 +10,7 @@ import zope.component from acme import challenges from acme import client as acme_client from acme import messages +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -354,12 +355,13 @@ class PollChallengesTest(unittest.TestCase): acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) ] - self.chall_update = {} + self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] + @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 1bba6991a..979cd97c1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -495,7 +495,8 @@ class SetByCliTest(unittest.TestCase): for v in ('manual', 'manual_auth_hook', 'manual_public_ip_logging_ok'): self.assertTrue(_call_set_by_cli(v, args, verb)) - cli.set_by_cli.detector = None + # https://github.com/python/mypy/issues/2087 + cli.set_by_cli.detector = None # type: ignore args = ['--manual-auth-hook', 'command'] for v in ('manual_auth_hook', 'manual_public_ip_logging_ok'): diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 333acf2b3..ac01103b8 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -8,6 +8,7 @@ import unittest import mock from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.tests.util import TempDirTestCase class CompleterTest(TempDirTestCase): @@ -21,7 +22,7 @@ class CompleterTest(TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] + self.paths = [] # type: List[str] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index d4c48c242..a4a65e2d4 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -6,6 +6,9 @@ import sys import unittest import mock +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Dict, Union +# pylint: enable=unused-import, no-name-in-module def get_signals(signums): @@ -23,8 +26,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = {} - prev_handlers = get_signals(signums) + prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] set_signals(dict((s, lambda s, _: signals.append(s)) for s in signums)) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 8619a1a2e..c9cfc69f9 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -5,6 +5,7 @@ import unittest import mock +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.tests import util @@ -106,8 +107,8 @@ class PreHookTest(HookTest): super(PreHookTest, self).tearDown() def _reset_pre_hook_already(self): - from certbot.hooks import pre_hook - pre_hook.already.clear() + from certbot.hooks import executed_pre_hooks + executed_pre_hooks.clear() def test_certonly(self): self.config.verb = "certonly" @@ -184,8 +185,8 @@ class PostHookTest(HookTest): super(PostHookTest, self).tearDown() def _reset_post_hook_eventually(self): - from certbot.hooks import post_hook - post_hook.eventually = [] + from certbot.hooks import post_hooks + del post_hooks[:] def test_certonly_and_run_with_hook(self): for verb in ("certonly", "run",): @@ -238,8 +239,8 @@ class PostHookTest(HookTest): self.assertEqual(self._get_eventually(), expected) def _get_eventually(self): - from certbot.hooks import post_hook - return post_hook.eventually + from certbot.hooks import post_hooks + return post_hooks class RunSavedPostHooksTest(HookTest): @@ -248,23 +249,23 @@ class RunSavedPostHooksTest(HookTest): @classmethod def _call(cls, *args, **kwargs): from certbot.hooks import run_saved_post_hooks - return run_saved_post_hooks(*args, **kwargs) + return run_saved_post_hooks() def _call_with_mock_execute_and_eventually(self, *args, **kwargs): """Call run_saved_post_hooks but mock out execute and eventually - certbot.hooks.post_hook.eventually is replaced with + certbot.hooks.post_hooks is replaced with self.eventually. The mock execute object is returned rather than the return value of run_saved_post_hooks. """ - eventually_path = "certbot.hooks.post_hook.eventually" + eventually_path = "certbot.hooks.post_hooks" with mock.patch(eventually_path, new=self.eventually): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] + self.eventually = [] # type: List[str] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 549d2c5e1..c5991347e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -10,6 +10,7 @@ import mock import six from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors @@ -21,9 +22,9 @@ class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot.log.pre_arg_parse_setup.""" @classmethod - def _call(cls, *args, **kwargs): + def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot.log import pre_arg_parse_setup - return pre_arg_parse_setup(*args, **kwargs) + return pre_arg_parse_setup() @mock.patch('certbot.log.sys') @mock.patch('certbot.log.pre_arg_parse_except_hook') @@ -38,16 +39,16 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - MemoryHandler = logging.handlers.MemoryHandler - memory_handler = None + memory_handler = None # type: Optional[logging.handlers.MemoryHandler] for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] - if memory_handler is None and isinstance(handler, MemoryHandler): + if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): memory_handler = handler + target = memory_handler.target # type: ignore else: self.assertTrue(isinstance(handler, logging.StreamHandler)) self.assertTrue( - isinstance(memory_handler.target, logging.StreamHandler)) + isinstance(target, logging.StreamHandler)) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 0986ff060..14cde27ee 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -16,12 +16,14 @@ import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli from certbot import constants from certbot import configuration from certbot import crypto_util from certbot import errors +from certbot import interfaces # pylint: disable=unused-import from certbot import main from certbot import updater from certbot import util @@ -600,14 +602,14 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met if mockisfile: orig_open = os.path.isfile - def mock_isfile(fn, *args, **kwargs): + def mock_isfile(fn, *args, **kwargs): # pylint: disable=unused-argument """Mock os.path.isfile()""" if (fn.endswith("cert") or fn.endswith("chain") or fn.endswith("privkey")): return True else: - return orig_open(fn, *args, **kwargs) + return orig_open(fn) with mock.patch("os.path.isfile") as mock_if: mock_if.side_effect = mock_isfile @@ -836,7 +838,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -851,7 +853,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, uid, strict): @@ -873,7 +875,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -891,7 +893,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -1040,9 +1042,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') - def write_msg(message, *args, **kwargs): + def write_msg(message, *args, **kwargs): # pylint: disable=unused-argument """Write message to stdout.""" - _, _ = args, kwargs stdout.write(message) try: diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 9ec8dca28..22e11e672 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -12,7 +12,7 @@ class ReporterTest(unittest.TestCase): from certbot import reporter self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) - self.old_stdout = sys.stdout + self.old_stdout = sys.stdout # type: ignore sys.stdout = six.StringIO() def tearDown(self): @@ -21,32 +21,32 @@ class ReporterTest(unittest.TestCase): def test_multiline_message(self): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("Line 1\n" in output) self.assertTrue("Line 2" in output) def test_tty_print_empty(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self.test_no_tty_print_empty() def test_no_tty_print_empty(self): self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore try: raise ValueError except ValueError: self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore def test_tty_successful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._successful_exit_common() def test_no_tty_successful_exit(self): self._successful_exit_common() def test_tty_unsuccessful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._unsuccessful_exit_common() def test_no_tty_unsuccessful_exit(self): @@ -55,7 +55,7 @@ class ReporterTest(unittest.TestCase): def _successful_exit_common(self): self._add_messages() self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" in output) @@ -67,7 +67,7 @@ class ReporterTest(unittest.TestCase): raise ValueError except ValueError: self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" not in output) diff --git a/certbot/util.py b/certbot/util.py index 55acd624f..8e84c29ba 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -20,6 +20,7 @@ from collections import OrderedDict import configargparse +from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import lock @@ -218,8 +219,12 @@ def safe_open(path, mode="w", chmod=None, buffering=None): """ # pylint: disable=star-args - open_args = () if chmod is None else (chmod,) - fdopen_args = () if buffering is None else (buffering,) + open_args = () # type: Union[Tuple[()], Tuple[int]] + if chmod is not None: + open_args = (chmod,) + fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + if buffering is not None: + fdopen_args = (buffering,) return os.fdopen( os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), mode, *fdopen_args) @@ -303,9 +308,8 @@ def get_filtered_names(all_names): for name in all_names: try: filtered_names.add(enforce_le_validity(name)) - except errors.ConfigurationError as error: - logger.debug('Not suggesting name "%s"', name) - logger.debug(error) + except errors.ConfigurationError: + logger.debug('Not suggesting name "%s"', name, exc_info=True) return filtered_names diff --git a/mypy.ini b/mypy.ini index d00c21ae7..f0c99e65f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,12 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-acme.magic_typing_test] +ignore_errors = True + +[mypy-certbot.*] +check_untyped_defs = True + [mypy-certbot_apache.*] check_untyped_defs = True diff --git a/setup.py b/setup.py index 3760fd35b..ee0470d3a 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.22.1', + 'acme>0.24.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index df13cdbef..d965d4470 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -30,7 +30,7 @@ josepy==1.0.1 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -mypy==0.580 +mypy==0.600 ndg-httpsclient==0.3.2 oauth2client==2.0.0 pathlib2==2.3.0 diff --git a/tox.ini b/tox.ini index 8c4d6c38d..2834ef9f9 100644 --- a/tox.ini +++ b/tox.ini @@ -121,8 +121,8 @@ commands = [testenv:mypy] basepython = python3 commands = - {[base]pip_install} .[dev3] {[base]install_packages} + {[base]pip_install} .[dev3] mypy {[base]source_paths} [testenv:apacheconftest] From 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Fri, 18 May 2018 09:10:41 -0700 Subject: [PATCH 500/631] switch signature verification to use pure cryptography (#6000) * switch signature verification to use pure cryptography On systems that prevent write/execute pages this prevents a segfault that is caused by pyopenssl creating a dynamic callback in the verification helper. * switch to using a verifier for older cryptography releases also add ec support, test vectors, and a test --- certbot/crypto_util.py | 36 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 ++++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 ++++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 +++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..71f6c990c 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,11 +12,16 @@ import os import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,26 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY----- From dec97fc1269636a94a9763d6a2e641d2ada3ac6f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 17:48:30 -0700 Subject: [PATCH 501/631] Revert "Add link to pycon issues (#5959)" This reverts commit 68359086fffca8805893bf6133c53b5f75357a7f. --- docs/contributing.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 52f08efe0..ed986c562 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -72,10 +72,6 @@ found in the `virtualenv docs`_. Find issues to work on ---------------------- -.. note:: If you're sprinting on Certbot at PyCon, you can find especially good - issues to work on during the event `here - `_. - You can find the open issues in the `github issue tracker`_. Comparatively easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan From c9a206ca890c3f39c5f0dffce9862cf439b9d5a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 21 May 2018 20:23:21 -0700 Subject: [PATCH 502/631] Get mypy passing with check_untyped_defs everywhere (#6021) * unchecked_typed_defs everywhere * fix mypy for lock_test * add magic_typing * fix mypy in letshelp * fix validator errors in compat test * fix mypy for test_driver.py * fix mypy in util.py * delint --- .../certbot_compatibility_test/test_driver.py | 48 ++++++++----------- .../certbot_compatibility_test/util.py | 12 ++--- .../certbot_compatibility_test/validator.py | 5 +- letshelp-certbot/letshelp_certbot/apache.py | 5 +- .../letshelp_certbot/apache_test.py | 16 +++++-- .../letshelp_certbot/magic_typing.py | 16 +++++++ .../letshelp_certbot/magic_typing_test.py | 41 ++++++++++++++++ mypy.ini | 46 ++---------------- tests/lock_test.py | 2 +- 9 files changed, 106 insertions(+), 85 deletions(-) create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing.py create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing_test.py diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 2c6c917b3..9eea95e67 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -15,6 +15,7 @@ from six.moves import xrange # pylint: disable=import-error,redefined-builtin from acme import challenges from acme import crypto_util from acme import messages +from acme.magic_typing import List, Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util @@ -52,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir): try: responses = plugin.perform(achalls) - except le_errors.Error as error: - logger.error("Performing challenges on %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Performing challenges on %s caused an error:", config, exc_info=True) return False success = True @@ -82,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir): if success: try: plugin.cleanup(achalls) - except le_errors.Error as error: - logger.error("Challenge cleanup for %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True) success = False if _dirs_are_unequal(config, backup): @@ -147,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains): try: plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path) plugin.save() # Needed by the Apache plugin - except le_errors.Error as error: - logger.error("**** Plugin failed to deploy certificate for %s:", domain) - logger.exception(error) + except le_errors.Error: + logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True) return False if not _save_and_restart(plugin, "deployed"): @@ -179,7 +177,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] + domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] for domain, info in domains_and_info: try: @@ -192,10 +190,9 @@ def test_enhancements(plugin, domains): # Don't immediately fail because a redirect may already be enabled logger.warning("*** Plugin failed to enable redirect for %s:", domain) logger.warning("%s", error) - except le_errors.Error as error: + except le_errors.Error: logger.error("*** An error occurred while enabling redirect for %s:", - domain) - logger.exception(error) + domain, exc_info=True) if not _save_and_restart(plugin, "enhanced"): return False @@ -222,9 +219,8 @@ def _save_and_restart(plugin, title=None): plugin.save(title) plugin.restart() return True - except le_errors.Error as error: - logger.error("*** Plugin failed to save and restart server:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin failed to save and restart server:", exc_info=True) return False @@ -232,9 +228,8 @@ def test_rollback(plugin, config, backup): """Tests the rollback checkpoints function""" try: plugin.rollback_checkpoints(1337) - except le_errors.Error as error: - logger.error("*** Plugin raised an exception during rollback:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin raised an exception during rollback:", exc_info=True) return False if _dirs_are_unequal(config, backup): @@ -263,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2): logger.error("The following files and directories are only " "present in one directory") if dircmp.left_only: - logger.error(dircmp.left_only) + logger.error(str(dircmp.left_only)) else: - logger.error(dircmp.right_only) + logger.error(str(dircmp.right_only)) return True elif dircmp.common_funny or dircmp.funny_files: logger.error("The following files and directories could not be " "compared:") if dircmp.common_funny: - logger.error(dircmp.common_funny) + logger.error(str(dircmp.common_funny)) else: - logger.error(dircmp.funny_files) + logger.error(str(dircmp.funny_files)) return True elif dircmp.diff_files: logger.error("The following files differ:") - logger.error(dircmp.diff_files) + logger.error(str(dircmp.diff_files)) return True for subdir in dircmp.subdirs.itervalues(): @@ -354,9 +349,8 @@ def main(): success = test_authenticator(plugin, config, temp_dir) if success and args.install: success = test_installer(args, plugin, config, temp_dir) - except errors.Error as error: - logger.error("Tests on %s raised:", config) - logger.exception(error) + except errors.Error: + logger.error("Tests on %s raised:", config, exc_info=True) success = False if success: diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index 4155944bd..6051bbc2e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -26,12 +26,12 @@ def create_le_config(parent_dir): config = copy.deepcopy(constants.CLI_DEFAULTS) le_dir = os.path.join(parent_dir, "certbot") - config["config_dir"] = os.path.join(le_dir, "config") - config["work_dir"] = os.path.join(le_dir, "work") - config["logs_dir"] = os.path.join(le_dir, "logs_dir") - os.makedirs(config["config_dir"]) - os.mkdir(config["work_dir"]) - os.mkdir(config["logs_dir"]) + os.mkdir(le_dir) + for dir_name in ("config", "logs", "work"): + full_path = os.path.join(le_dir, dir_name) + os.mkdir(full_path) + full_name = dir_name + "_dir" + config[full_name] = full_path config["domains"] = None diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 791fe0da2..fd2f95702 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -33,7 +33,7 @@ class Validator(object): try: presented_cert = crypto_util.probe_sni(name, host, port) except acme_errors.Error as error: - logger.exception(error) + logger.exception(str(error)) return False return presented_cert.digest("sha256") == cert.digest("sha256") @@ -86,8 +86,7 @@ class Validator(object): return False try: - _, max_age_value = max_age[0] - max_age_value = int(max_age_value) + max_age_value = int(max_age[0][1]) except ValueError: logger.error("Server responded with invalid HSTS header field") return False diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index f77a6a1b0..50f3c5ef6 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -16,6 +16,8 @@ import textwrap import six +from letshelp_certbot.magic_typing import List # pylint: disable=unused-import, no-name-in-module + _DESCRIPTION = """ Let's Help is a simple script you can run to help out the Certbot project. Since Certbot will support automatically configuring HTTPS on @@ -87,7 +89,8 @@ def copy_config(server_root, temp_dir): :rtype: `tuple` of `list` of `str` """ - copied_files, copied_dirs = [], [] + copied_files = [] # type: List[str] + copied_dirs = [] # type: List[str] dir_len = len(os.path.dirname(server_root)) for config_path, config_dirs, config_files in os.walk(server_root): diff --git a/letshelp-certbot/letshelp_certbot/apache_test.py b/letshelp-certbot/letshelp_certbot/apache_test.py index e0656ae05..a1115bc06 100644 --- a/letshelp-certbot/letshelp_certbot/apache_test.py +++ b/letshelp-certbot/letshelp_certbot/apache_test.py @@ -203,13 +203,19 @@ class LetsHelpApacheTest(unittest.TestCase): tempdir_path, "config.tar.gz")) tempdir = tar.next() - self.assertTrue(tempdir.isdir()) - self.assertEqual(tempdir.name, ".") + if tempdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(tempdir.isdir()) + self.assertEqual(tempdir.name, ".") testdir = tar.next() - self.assertTrue(testdir.isdir()) - self.assertEqual(os.path.basename(testdir.name), - testdir_basename) + if testdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(testdir.isdir()) + self.assertEqual(os.path.basename(testdir.name), + testdir_basename) self.assertEqual(tar.next(), None) diff --git a/letshelp-certbot/letshelp_certbot/magic_typing.py b/letshelp-certbot/letshelp_certbot/magic_typing.py new file mode 100644 index 000000000..471b8dfa9 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing.py @@ -0,0 +1,16 @@ +"""Shim class to not have to depend on typing module in prod.""" +import sys + +class TypingClass(object): + """Ignore import errors by getting anything""" + def __getattr__(self, name): + return None + +try: + # mypy doesn't respect modifying sys.modules + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import +except ImportError: + sys.modules[__name__] = TypingClass() diff --git a/letshelp-certbot/letshelp_certbot/magic_typing_test.py b/letshelp-certbot/letshelp_certbot/magic_typing_test.py new file mode 100644 index 000000000..200ca03b8 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing_test.py @@ -0,0 +1,41 @@ +"""Tests for letshelp_certbot.magic_typing.""" +import sys +import unittest + +import mock + + +class MagicTypingTest(unittest.TestCase): + """Tests for letshelp_certbot.magic_typing.""" + def test_import_success(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + typing_class_mock = mock.MagicMock() + text_mock = mock.MagicMock() + typing_class_mock.Text = text_mock + sys.modules['typing'] = typing_class_mock + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertEqual(Text, text_mock) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + def test_import_failure(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + sys.modules['typing'] = None + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertTrue(Text is None) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/mypy.ini b/mypy.ini index f0c99e65f..188ed031f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,48 +1,10 @@ [mypy] -python_version = 2.7 -ignore_missing_imports = True - -[mypy-acme.*] check_untyped_defs = True +ignore_missing_imports = True +python_version = 2.7 [mypy-acme.magic_typing_test] ignore_errors = True -[mypy-certbot.*] -check_untyped_defs = True - -[mypy-certbot_apache.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudflare.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudxns.*] -check_untyped_defs = True - -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsimple.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsmadeeasy.*] -check_untyped_defs = True - -[mypy-certbot_dns_google.*] -check_untyped_defs = True - -[mypy-certbot_dns_luadns.*] -check_untyped_defs = True - -[mypy-certbot_dns_nsone.*] -check_untyped_defs = True - -[mypy-certbot_dns_rfc2136.*] -check_untyped_defs = True - -[mypy-certbot_dns_route53.*] -check_untyped_defs = True - -[mypy-certbot_nginx.*] -check_untyped_defs = True +[mypy-letshelp_certbot.magic_typing_test] +ignore_errors = True diff --git a/tests/lock_test.py b/tests/lock_test.py index 4bb2865b4..b01cc5d58 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -198,7 +198,7 @@ def report_failure(err_msg, out, err): :param str err: stderr output """ - logger.fatal(err_msg) + logger.critical(err_msg) log_output(logging.INFO, out, err) sys.exit(err_msg) From cfd4b8f3634df21370f0e21876400e46d8b5b8eb Mon Sep 17 00:00:00 2001 From: Quang Vu Date: Tue, 22 May 2018 15:32:44 -0700 Subject: [PATCH 503/631] #4242 Support multi emails register (#5994) This change will allow registering/updating account with multi emails. Detail is enclosed in #4242 * support multi emails register * add more test cases * update test to unregister before register * update create path to support multi emaill * refactor payload updating * fix typo * move command line doc to another place * revert the change for updating account registration info, added unit test * rearrange text for consistency --- acme/acme/messages.py | 2 +- certbot/cli.py | 2 +- certbot/client.py | 3 ++- certbot/interfaces.py | 4 +++- certbot/main.py | 33 +++++++++++++++++---------------- certbot/tests/main_test.py | 10 +++++++--- tests/boulder-integration.sh | 9 ++++++++- 7 files changed, 39 insertions(+), 24 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 03dbc3255..827a4dd11 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -285,7 +285,7 @@ class Registration(ResourceBody): if phone is not None: details.append(cls.phone_prefix + phone) if email is not None: - details.append(cls.email_prefix + email) + details.extend([cls.email_prefix + mail for mail in email.split(',')]) kwargs['contact'] = tuple(details) return cls(**kwargs) diff --git a/certbot/cli.py b/certbot/cli.py index 8a1ad381a..25319bbd8 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -430,7 +430,7 @@ VERB_HELP = [ }), ("enhance", { "short": "Add security enhancements to your existing configuration", - "opts": ("Helps to harden the TLS configration by adding security enhancements " + "opts": ("Helps to harden the TLS configuration by adding security enhancements " "to already existing configuration."), "usage": "\n\n certbot enhance [options]\n\n" }), diff --git a/certbot/client.py b/certbot/client.py index 1932ab83e..dadc3a0f8 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -185,8 +185,9 @@ def perform_registration(acme, config, tos_cb): Actually register new account, trying repeatedly if there are email problems - :param .IConfig config: Client configuration. :param acme.client.Client client: ACME client object. + :param .IConfig config: Client configuration. + :param Callable tos_cb: a callback to handle Term of Service agreement. :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` diff --git a/certbot/interfaces.py b/certbot/interfaces.py index c96f6bd51..6233e3592 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -201,7 +201,9 @@ class IConfig(zope.interface.Interface): """ server = zope.interface.Attribute("ACME Directory Resource URI.") email = zope.interface.Attribute( - "Email used for registration and recovery contact. (default: Ask)") + "Email used for registration and recovery contact. Use comma to " + "register multiple emails, ex: u1@example.com,u2@example.com. " + "(default: Ask).") rsa_key_size = zope.interface.Attribute("Size of the RSA key.") must_staple = zope.interface.Attribute( "Adds the OCSP Must Staple extension to the certificate. " diff --git a/certbot/main.py b/certbot/main.py index 6c1d82793..dad1b793e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -487,6 +487,21 @@ def _determine_account(config): :raises errors.Error: If unable to register an account with ACME server """ + def _tos_cb(terms_of_service): + if config.tos: + return True + msg = ("Please read the Terms of Service at {0}. You " + "must agree in order to register with the ACME " + "server at {1}".format( + terms_of_service, config.server)) + obj = zope.component.getUtility(interfaces.IDisplay) + result = obj.yesno(msg, "Agree", "Cancel", + cli_flag="--agree-tos", force_interactive=True) + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") + account_storage = account.AccountFileStorage(config) acme = None @@ -501,21 +516,6 @@ def _determine_account(config): else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - - def _tos_cb(terms_of_service): - if config.tos: - return True - msg = ("Please read the Terms of Service at {0}. You " - "must agree in order to register with the ACME " - "server at {1}".format( - terms_of_service, config.server)) - obj = zope.component.getUtility(interfaces.IDisplay) - result = obj.yesno(msg, "Agree", "Cancel", - cli_flag="--agree-tos", force_interactive=True) - if not result: - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) @@ -735,8 +735,9 @@ def register(config, unused_plugins): acc, acme = _determine_account(config) cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. + acc_contacts = ['mailto:' + email for email in config.email.split(',')] acc.regr = cb_client.acme.update_registration(acc.regr.update( - body=acc.regr.body.update(contact=('mailto:' + config.email,)))) + body=acc.regr.body.update(contact=acc_contacts))) account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 14cde27ee..8c9e24354 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1435,7 +1435,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mocked_storage = mock.MagicMock() mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] - mocked_det.return_value = (mock.MagicMock(), "foo") + mock_acc = mock.MagicMock() + mock_regr = mock_acc.regr + mocked_det.return_value = (mock_acc, "foo") cb_client = mock.MagicMock() mocked_client.Client.return_value = cb_client x = self._call_no_clientmock( @@ -1445,8 +1447,10 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(x[0] is None) # and we got supposedly did update the registration from # the server - self.assertTrue( - cb_client.acme.update_registration.called) + reg_arg = cb_client.acme.update_registration.call_args[0][0] + # Test the return value of .update() was used because + # the regr is immutable. + self.assertEqual(reg_arg, mock_regr.update()) # and we saved the updated registration on disk self.assertTrue(mocked_storage.save_regr.called) self.assertTrue( diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 9748befa3..e931e30f3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -191,7 +191,14 @@ for dir in $renewal_hooks_dirs; do exit 1 fi done -common register --update-registration --email example@example.org + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org common plugins --init --prepare | grep webroot From 8440d0814de346fb8beb2ca1497e1cc7803a19fe Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Tue, 22 May 2018 18:35:12 -0400 Subject: [PATCH 504/631] fixed dependency-requirements.txt (#6023) --- letsencrypt-auto-source/letsencrypt-auto | 18 ++++++++++++------ .../pieces/dependency-requirements.txt | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0b83b08a7..28281e20d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index a30a32b48..376e19deb 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -59,9 +59,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -116,7 +118,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -142,7 +145,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -170,9 +174,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ From deb5b072d9bfb10646df5a7de71f05ac6c9b9cd6 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Wed, 23 May 2018 12:59:49 -0400 Subject: [PATCH 505/631] Log cases when standalone fails to bind a port. (#5985) * Log cases when standalone fails to bind a port. * Fix linter + changed log message * Changed multiline string format * Fixed indentation in standalone.py --- acme/acme/standalone.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index a370501ee..ff9159933 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -83,9 +83,22 @@ class BaseDualNetworkedServers(object): new_address = (server_address[0],) + (port,) + server_address[2:] new_args = (new_address,) + remaining_args server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args - except socket.error: - logger.debug("Failed to bind to %s:%s using %s", new_address[0], + logger.debug( + "Successfully bound to %s:%s using %s", new_address[0], new_address[1], "IPv6" if ip_version else "IPv4") + except socket.error: + if self.servers: + # Already bound using IPv6. + logger.debug( + "Certbot wasn't able to bind to %s:%s using %s, this " + + "is often expected due to the dual stack nature of " + + "IPv6 socket implementations.", + new_address[0], new_address[1], + "IPv6" if ip_version else "IPv4") + else: + logger.debug( + "Failed to bind to %s:%s using %s", new_address[0], + new_address[1], "IPv6" if ip_version else "IPv4") else: self.servers.append(server) # If two servers are set up and port 0 was passed in, ensure we always From 4304ff0d623bc3512343c09f545579119887138e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 23 May 2018 11:33:21 -0700 Subject: [PATCH 506/631] Bring up just the boulder container. (#6031) Boulder recently added a "netaccess" container which may conflict. --- tests/boulder-fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index d513ec064..7a677ad12 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,7 +17,7 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -docker-compose up -d +docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder until curl http://localhost:4000/directory 2>/dev/null; do From 0b215366b1c66950195ddbbfb6f16b8657f4b198 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 May 2018 13:57:22 -0700 Subject: [PATCH 507/631] turn off cancel notifications (#5918) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 111ddb3d4..e3d964326 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,6 +107,7 @@ notifications: irc: channels: - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw=" + on_cancel: never on_success: never on_failure: always use_notice: true From a1f5dc27f28e46cc2aa92777f12f17b418fd6e7c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 23 May 2018 14:03:30 -0700 Subject: [PATCH 508/631] Add domain to error message when no matching server block found (#6034) --- certbot-nginx/certbot_nginx/configurator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 118699aa2..293f7378e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -334,7 +334,7 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost(port) + default_vhost = self._get_default_vhost(port, domain) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,7 +350,7 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port): + def _get_default_vhost(self, port, domain): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] @@ -367,7 +367,7 @@ class NginxConfigurator(common.Installer): # TODO: present a list of vhosts for user to choose from raise errors.MisconfigurationError("Could not automatically find a matching server" - " block. Set the `server_name` directive to use the Nginx installer.") + " block for %s. Set the `server_name` directive to use the Nginx installer." % domain) def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. From b1bcccb04bbd4d9d35bc95397f9ba1633b91242a Mon Sep 17 00:00:00 2001 From: Jeremy Gillula Date: Wed, 23 May 2018 20:40:34 -0700 Subject: [PATCH 509/631] Changing opt-in ask for emails (#6035) --- certbot/eff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/eff.py b/certbot/eff.py index b047c0b97..388ae986b 100644 --- a/certbot/eff.py +++ b/certbot/eff.py @@ -41,8 +41,8 @@ def _want_subscription(): 'Would you be willing to share your email address with the ' "Electronic Frontier Foundation, a founding partner of the Let's " 'Encrypt project and the non-profit organization that develops ' - "Certbot? We'd like to send you email about EFF and our work to " - 'encrypt the web, protect its users and defend digital rights.') + "Certbot? We'd like to send you email about our work encrypting " + "the web, EFF news, campaigns, and ways to support digital freedom. ") display = zope.component.getUtility(interfaces.IDisplay) return display.yesno(prompt, default=False) From a03c68fc8334cac84e8ee22f2b7b7fa7fc323ab9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 24 May 2018 10:53:21 -0700 Subject: [PATCH 510/631] Clean up boulder-fetch a bit. (#6032) The value for FAKE_DNS is now always the same because Boulder's docker-compose hardcodes it, so skip some sed. Set a time limit on how long we'll wait for boulder to come up. --- tests/boulder-fetch.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 7a677ad12..31e0f6b30 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -11,16 +11,20 @@ if [ ! -d ${BOULDERPATH} ]; then fi cd ${BOULDERPATH} -FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) -[ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 -sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml +sed -i "s/FAKE_DNS: .*/FAKE_DNS: 10.77.77.1/" docker-compose.yml docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder -until curl http://localhost:4000/directory 2>/dev/null; do - echo waiting for boulder - sleep 1 +for n in `seq 1 150` ; do + if curl http://localhost:4000/directory 2>/dev/null; then + break + else + sleep 1 + fi done + +if ! curl http://localhost:4000/directory 2>/dev/null; then + echo "timed out waiting for boulder to start" + exit 1 +fi From e48c653245bc08b7e517465aea32f678c5b9b64b Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 25 May 2018 21:00:37 +0300 Subject: [PATCH 511/631] Change GenericUpdater parameter to lineage (#6030) In order to give more flexibility for plugins using interfaces.GenericUpdater interface, lineage needs to be passed to the updater method instead of individual domains. All of the (present and potential) installers do not work on per domain basis, while the lineage does contain a list of them for installers which do. This also means that we don't unnecessarily run the updater method multiple times, potentially invoking expensive tooling up to $max_san_amount times. * Make GenericUpdater use lineage as parameter and get invoked only once per lineage --- certbot/interfaces.py | 18 +++++++++--------- certbot/tests/renewupdater_test.py | 11 ++++------- certbot/updater.py | 7 +++---- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 6233e3592..a5fb426e6 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -604,10 +604,10 @@ class IReporter(zope.interface.Interface): # When "certbot renew" is run, Certbot will iterate over each lineage and check # if the selected installer for that lineage is a subclass of each updater # class. If it is and the update of that type is configured to be run for that -# lineage, the relevant update function will be called for each domain in the -# lineage. These functions are never called for other subcommands, so if an -# installer wants to perform an update during the run or install subcommand, it -# should do so when :func:`IInstaller.deploy_cert` is called. +# lineage, the relevant update function will be called for it. These functions +# are never called for other subcommands, so if an installer wants to perform +# an update during the run or install subcommand, it should do so when +# :func:`IInstaller.deploy_cert` is called. @six.add_metaclass(abc.ABCMeta) class GenericUpdater(object): @@ -623,7 +623,7 @@ class GenericUpdater(object): """ @abc.abstractmethod - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): """Perform any update types defined by the installer. If an installer is a subclass of the class containing this method, this @@ -631,9 +631,10 @@ class GenericUpdater(object): update defined by the installer should be run conditionally, the installer needs to handle checking the conditions itself. - This method is called once for each domain. + This method is called once for each lineage. - :param str domain: domain to handle the updates for + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert """ @@ -661,8 +662,7 @@ class RenewDeployer(object): This method is called once for each lineage renewed - :param lineage: Certificate lineage object that is set if certificate - was renewed on this run. + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert """ diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index 9d0f8d515..ade8db390 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -19,7 +19,7 @@ class RenewUpdaterTest(unittest.TestCase): # pylint: disable=unused-argument self.restart = mock.MagicMock() self.callcounter = mock.MagicMock() - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): self.callcounter(*args, **kwargs) class MockInstallerRenewDeployer(interfaces.RenewDeployer): @@ -46,9 +46,7 @@ class RenewUpdaterTest(unittest.TestCase): def test_server_updates(self, _, mock_select, mock_getsave): config = self.get_config({"disable_renew_updates": False}) - lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] - mock_getsave.return_value = lineage + mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater @@ -59,14 +57,13 @@ class RenewUpdaterTest(unittest.TestCase): mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, lineage) - self.assertEqual(mock_generic_updater.callcounter.call_count, 2) + updater.run_generic_updaters(config, None, mock.MagicMock()) + self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] mock_deployer = self.renew_deployer updater.run_renewal_deployer(lineage, mock_deployer, config) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) diff --git a/certbot/updater.py b/certbot/updater.py index f822c55ee..1216f38a6 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -61,7 +61,6 @@ def _run_updaters(lineage, installer, config): :returns: `None` :rtype: None """ - for domain in lineage.names(): - if not config.disable_renew_updates: - if isinstance(installer, interfaces.GenericUpdater): - installer.generic_updates(domain) + if not config.disable_renew_updates: + if isinstance(installer, interfaces.GenericUpdater): + installer.generic_updates(lineage) From 9f6b147d6f06e534351377195b4f28c61270115d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sat, 26 May 2018 18:31:23 +0300 Subject: [PATCH 512/631] Do not call updaters and deployers when run with --dry-run (#6038) When Certbot is run with --dry-run, skip running GenericUpdater and RenewDeployer interface methods. This PR also makes the parameter order of updater.run_generic_updaters and updater.run_renewal_deployer consistent. Fixes #5927 * Do not call updaters and deployers when run with --dry-run * Use ConfigTestCase instead of mocking config objects manually --- certbot/main.py | 2 +- certbot/renewal.py | 4 ++-- certbot/tests/main_test.py | 3 ++- certbot/tests/renewupdater_test.py | 35 ++++++++++++++++++------------ certbot/updater.py | 20 ++++++++++++----- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index dad1b793e..8fa6ebfc6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1163,7 +1163,7 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. - updater.run_renewal_deployer(renewed_lineage, installer, config) + updater.run_renewal_deployer(config, renewed_lineage, installer) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) diff --git a/certbot/renewal.py b/certbot/renewal.py index 0a6568426..236330a85 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -431,8 +431,8 @@ def handle_renewal_request(config): renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, expiry.strftime("%Y-%m-%d"))) # Run updater interface methods - updater.run_generic_updaters(lineage_config, plugins, - renewal_candidate) + updater.run_generic_updaters(lineage_config, renewal_candidate, + plugins) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8c9e24354..36821cf53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1464,7 +1464,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met None, None, None) with mock.patch('certbot.updater.logger.warning') as mock_log: - updater.run_generic_updaters(None, None, None) + self.config.dry_run = False + updater.run_generic_updaters(self.config, None, None) self.assertTrue(mock_log.called) self.assertTrue("Could not choose appropriate plugin for updaters" in mock_log.call_args[0][0]) diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index ade8db390..bd1cd891e 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -9,10 +9,11 @@ from certbot import updater import certbot.tests.util as test_util -class RenewUpdaterTest(unittest.TestCase): +class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): + super(RenewUpdaterTest, self).setUp() class MockInstallerGenericUpdater(interfaces.GenericUpdater): """Mock class that implements GenericUpdater""" def __init__(self, *args, **kwargs): @@ -33,41 +34,47 @@ class RenewUpdaterTest(unittest.TestCase): self.generic_updater = MockInstallerGenericUpdater() self.renew_deployer = MockInstallerRenewDeployer() - def get_config(self, args): - """Get mock config from dict of parameters""" - config = mock.MagicMock() - for key in args.keys(): - config.__dict__[key] = args[key] - return config - @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') @test_util.patch_get_utility() def test_server_updates(self, _, mock_select, mock_getsave): - config = self.get_config({"disable_renew_updates": False}) - mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater mock_select.return_value = (mock_generic_updater, None) with mock.patch('certbot.main._init_le_client'): - main.renew_cert(config, None, mock.MagicMock()) + main.renew_cert(self.config, None, mock.MagicMock()) self.assertTrue(mock_generic_updater.restart.called) mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, mock.MagicMock()) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): - config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() mock_deployer = self.renew_deployer - updater.run_renewal_deployer(lineage, mock_deployer, config) + updater.run_renewal_deployer(self.config, lineage, mock_deployer) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + @mock.patch("certbot.updater.logger.debug") + def test_updater_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_generic_updaters(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping updaters in dry-run mode.") + + @mock.patch("certbot.updater.logger.debug") + def test_deployer_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_renewal_deployer(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping renewal deployer in dry-run mode.") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py index 1216f38a6..112cf06ef 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -8,21 +8,24 @@ from certbot.plugins import selection as plug_sel logger = logging.getLogger(__name__) -def run_generic_updaters(config, plugins, lineage): +def run_generic_updaters(config, lineage, plugins): """Run updaters that the plugin supports :param config: Configuration object :type config: interfaces.IConfig - :param plugins: List of plugins - :type plugins: `list` of `str` - :param lineage: Certificate lineage object :type lineage: storage.RenewableCert + :param plugins: List of plugins + :type plugins: `list` of `str` + :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping updaters in dry-run mode.") + return try: # installers are used in auth mode to determine domain names installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") @@ -31,10 +34,13 @@ def run_generic_updaters(config, plugins, lineage): return _run_updaters(lineage, installer, config) -def run_renewal_deployer(lineage, installer, config): +def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used installer plugin. + :param config: Configuration object + :type config: interfaces.IConfig + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert @@ -44,6 +50,10 @@ def run_renewal_deployer(lineage, installer, config): :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping renewal deployer in dry-run mode.") + return + if not config.disable_renew_updates and isinstance(installer, interfaces.RenewDeployer): installer.renew_deploy(lineage) From d53ef1f7c24c21d297a23349f3d566e24252b1fb Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 31 May 2018 13:57:23 -0700 Subject: [PATCH 513/631] Add mypy info to Certbot docs (#6033) * Add mypy info to Certbot docs * break up lines * link to mypy docs and links to https * Expand on import wording * be consistent about mypy styling --- docs/contributing.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ed986c562..58db251d4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,40 @@ Please: .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +Mypy type annotations +===================== + +Certbot uses the `mypy`_ static type checker. Python 3 natively supports official type annotations, +which can then be tested for consistency using mypy. Python 2 doesn’t, but type annotations can +be `added in comments`_. Mypy does some type checks even without type annotations; we can find +bugs in Certbot even without a fully annotated codebase. + +Certbot supports both Python 2 and 3, so we’re using Python 2-style annotations. + +Zulip wrote a `great guide`_ to using mypy. It’s useful, but you don’t have to read the whole thing +to start contributing to Certbot. + +To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. + +Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from +``acme.magic_typing`` and have to add some comments for pylint like this: + +.. code-block:: python + + from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module + +Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. +Those imports should look like this: + +.. code-block:: python + + from OpenSSL import crypto + from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 + +.. _mypy: https://mypy.readthedocs.io +.. _added in comments: https://mypy.readthedocs.io/en/latest/cheat_sheet.html +.. _great guide: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/ + Submitting a pull request ========================= From fb0d2ec3d60cf29d2c8aff09a3adbf50ba7f938f Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 1 Jun 2018 18:09:02 -0400 Subject: [PATCH 514/631] Include missing space (#6061) --- certbot-apache/certbot_apache/configurator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb82a9d3f..861fe4458 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -132,10 +132,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): default=cls.OS_DEFAULTS["challenge_location"], help="Directory path for challenge configuration.") add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you." + + help="Let installer handle enabling required modules for you. " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you." + + help="Let installer handle enabling sites for you. " + "(Only Ubuntu/Debian currently)") util.add_deprecated_argument(add, argument_name="ctl", nargs=1) util.add_deprecated_argument( From e2d6faa8a95b05e11369f7ae646fa8216b2b883a Mon Sep 17 00:00:00 2001 From: schoen Date: Fri, 1 Jun 2018 15:21:02 -0700 Subject: [PATCH 515/631] Add --reuse-key feature (#5901) * Initial work on new version of --reuse-key * Test for reuse_key * Make lint happier * Also test a non-dry-run reuse_key renewal * Test --reuse-key in boulder integration test * Better reuse-key integration testing * Log fact that key was reused * Test that the certificates themselves are different * Change "oldkeypath" to "old_keypath" * Simply appearance of new-key generation logic * Reorganize new-key logic * Move awk logic into TotalAndDistinctLines function * After refactor, there's now explicit None rather than missing param * Indicate for MyPy that key can be None * Actually import the Optional type * magic_typing is too magical for pylint * Remove --no-reuse-key option * Correct pylint test disable --- certbot/cli.py | 6 ++++++ certbot/client.py | 32 +++++++++++++++++++++++++++----- certbot/constants.py | 1 + certbot/renewal.py | 7 +++++-- certbot/tests/main_test.py | 24 +++++++++++++++++++++--- tests/boulder-integration.sh | 28 ++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 10 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 25319bbd8..05e316133 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1017,6 +1017,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "certificate already exists for the requested certificate name " "but does not match the requested domains, renew it now, " "regardless of whether it is near expiry.") + helpful.add( + "automation", "--reuse-key", dest="reuse_key", + action="store_true", default=flag_default("reuse_key"), + help="When renewing, use the same private key as the existing " + "certificate.") + helpful.add( ["automation", "renew", "certonly"], "--allow-subset-of-names", action="store_true", diff --git a/certbot/client.py b/certbot/client.py index dadc3a0f8..cc2f31d56 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -4,6 +4,7 @@ import logging import os import platform + from cryptography.hazmat.backends import default_backend # https://github.com/python/typeshed/blob/master/third_party/ # 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi @@ -16,6 +17,7 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import,no-name-in-module import certbot @@ -273,7 +275,7 @@ class Client(object): cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) return cert.encode(), chain.encode() - def obtain_certificate(self, domains): + def obtain_certificate(self, domains, old_keypath=None): """Obtains a certificate from the ACME server. `.register` must be called before `.obtain_certificate` @@ -286,16 +288,36 @@ class Client(object): :rtype: tuple """ + + # We need to determine the key path, key PEM data, CSR path, + # and CSR PEM data. For a dry run, the paths are None because + # they aren't permanently saved to disk. For a lineage with + # --reuse-key, the key path and PEM data are derived from an + # existing file. + + if old_keypath is not None: + # We've been asked to reuse a specific existing private key. + # Therefore, we'll read it now and not generate a new one in + # either case below. + with open(old_keypath, "r") as f: + keypath = old_keypath + keypem = f.read() + key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + logger.info("Reusing existing private key from %s.", old_keypath) + else: + # The key is set to None here but will be created below. + key = None + # Create CSR from names if self.config.dry_run: - key = util.Key(file=None, - pem=crypto_util.make_key(self.config.rsa_key_size)) + key = key or util.Key(file=None, + pem=crypto_util.make_key(self.config.rsa_key_size)) csr = util.CSR(file=None, form="pem", data=acme_crypto_util.make_csr( key.pem, domains, self.config.must_staple)) else: - key = crypto_util.init_save_key( - self.config.rsa_key_size, self.config.key_dir) + key = key or crypto_util.init_save_key(self.config.rsa_key_size, + self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) diff --git a/certbot/constants.py b/certbot/constants.py index 40557d287..1dd25e799 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,6 +64,7 @@ CLI_DEFAULTS = dict( pref_challs=[], validate_hooks=True, directory_hooks=True, + reuse_key=False, disable_renew_updates=False, # Subparsers diff --git a/certbot/renewal.py b/certbot/renewal.py index 236330a85..aa8c9722a 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -36,7 +36,7 @@ STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "pre_hook", "post_hook", "tls_sni_01_address", "http01_address"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names"] +BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key"] CONFIG_ITEMS = set(itertools.chain( BOOL_CONFIG_ITEMS, INT_CONFIG_ITEMS, STR_CONFIG_ITEMS, ('pref_challs',))) @@ -298,7 +298,10 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) + # The private key is the existing lineage private key if reuse_key is set. + # Otherwise, generate a fresh private key by passing None. + new_key = lineage.privkey if config.reuse_key else None + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 36821cf53..4b251c421 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1026,8 +1026,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False, expiry_date=datetime.datetime.now()): - # pylint: disable=too-many-locals,too-many-arguments + quiet_mode=False, expiry_date=datetime.datetime.now(), + reuse_key=False): + # pylint: disable=too-many-locals,too-many-arguments,too-many-branches cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path, @@ -1077,7 +1078,13 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met traceback.format_exc()) if should_renew: - mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) + if reuse_key: + # The location of the previous live privkey.pem is passed + # to obtain_certificate + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], + os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem")) + else: + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None) else: self.assertEqual(mock_client.obtain_certificate.call_count, 0) except: @@ -1127,6 +1134,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + def test_reuse_key(self): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + + @mock.patch('certbot.storage.RenewableCert.save_successor') + def test_reuse_key_no_dry_run(self, unused_save_successor): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + @mock.patch('certbot.renewal.should_renew') def test_renew_skips_recent_certs(self, should_renew): should_renew.return_value = False diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e931e30f3..ef611e743 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -166,6 +166,14 @@ CheckRenewHook() { CheckSavedRenewHook $1 } +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + # Cleanup coverage data coverage erase @@ -347,6 +355,26 @@ if common certificates | grep "fail\.dns1\.le\.wtf"; then exit 1 fi +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ From f19ebab44173b3ed70eb2aa81464fe1aa2fc0122 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Jun 2018 21:08:40 +0300 Subject: [PATCH 516/631] Make sure the pluginstorage file gets truncated when writing to it (#6062) --- certbot/plugins/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index 9472a1ebb..ae3ca1889 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -84,7 +84,8 @@ class PluginStorage(object): raise errors.PluginStorageError(errmsg) try: with os.fdopen(os.open(self._storagepath, - os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fh: + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600), 'w') as fh: fh.write(serialized) except IOError as e: errmsg = "Could not write PluginStorage data to file {0} : {1}".format( From 4151737e171b9a73923d02dc746cb593fb1f6d33 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 13:13:23 -0700 Subject: [PATCH 517/631] Read in bytes to fix --reuse-key on Python 3 (#6069) * Read bytes for now for compatibility * add clarifying comment --- certbot/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index cc2f31d56..d97de0571 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -299,7 +299,10 @@ class Client(object): # We've been asked to reuse a specific existing private key. # Therefore, we'll read it now and not generate a new one in # either case below. - with open(old_keypath, "r") as f: + # + # We read in bytes here because the type of `key.pem` + # created below is also bytes. + with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] From 15f1405fff7083bf5d4f599a58c54a43be499740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20D=C4=99bski?= Date: Mon, 4 Jun 2018 23:54:17 +0200 Subject: [PATCH 518/631] Implement TLS-ALPN-01 challenge and standalone TLS-ALPN server (#5894) The new challenge is described in https://github.com/rolandshoemaker/acme-tls-alpn. * TLS-ALPN tests * Implement TLS-ALPN challenge * Skip TLS-ALPN tests on old pyopenssl * make _selection methods private. --- acme/acme/challenges.py | 152 +++++++++++++++++++++++++++- acme/acme/challenges_test.py | 121 ++++++++++++++++++++++ acme/acme/crypto_util.py | 61 ++++++++--- acme/acme/crypto_util_test.py | 16 ++- acme/acme/standalone.py | 48 ++++++++- acme/acme/standalone_test.py | 57 +++++++++++ acme/acme/testdata/README | 6 +- acme/acme/testdata/rsa1024_cert.pem | 13 +++ 8 files changed, 453 insertions(+), 21 deletions(-) create mode 100644 acme/acme/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 674f2c38f..ce788e2cc 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,5 +1,6 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging @@ -7,7 +8,7 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import OpenSSL +from OpenSSL import crypto import requests import six @@ -411,8 +412,8 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ if key is None: - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) return crypto_util.gen_ss_cert(key, [ # z_domain is too big to fit into CN, hence first dummy domain 'dummy', self.z_domain.decode()], force_san=True), key @@ -507,6 +508,151 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) +@ChallengeResponse.register +class TLSALPN01Response(KeyAuthorizationChallengeResponse): + """ACME tls-alpn-01 challenge response.""" + typ = "tls-alpn-01" + + PORT = 443 + """Verification port as defined by the protocol. + + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` iff client's control of the domain has been + verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(cert, domain) + + +@Challenge.register # pylint: disable=too-many-ancestors +class TLSALPN01(KeyAuthorizationChallenge): + """ACME tls-alpn-01 challenge.""" + response_cls = TLSALPN01Response + typ = response_cls.typ + + def validation(self, account_key, **kwargs): + """Generate validation. + + :param JWK account_key: + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) + + @Challenge.register # pylint: disable=too-many-ancestors class DNS(_TokenChallenge): """ACME "dns" challenge.""" diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 834d569aa..b929d4939 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -393,6 +393,127 @@ class TLSSNI01Test(unittest.TestCase): mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) +class TLSALPN01ResponseTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) + self.jmsg = { + 'resource': 'challenge', + 'type': 'tls-alpn-01', + 'keyAuthorization': self.response.key_authorization, + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.response.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01Response + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01Response + hash(TLSALPN01Response.from_json(self.jmsg)) + + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, mock.sentinel.cert, self.domain) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + + +class TLSALPN01Test(unittest.TestCase): + + def setUp(self): + from acme.challenges import TLSALPN01 + self.msg = TLSALPN01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) + self.jmsg = { + 'type': 'tls-alpn-01', + 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.msg.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01 + self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01 + hash(TLSALPN01.from_json(self.jmsg)) + + def test_from_json_invalid_token_length(self): + from acme.challenges import TLSALPN01 + self.jmsg['token'] = jose.encode_b64jose(b'abcd') + self.assertRaises( + jose.DeserializationError, TLSALPN01.from_json, self.jmsg) + + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + + class DNSTest(unittest.TestCase): def setUp(self): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d0e203417..d25c2340b 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) _DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. @@ -38,12 +47,25 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_TLSSNI01_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -60,18 +82,19 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -96,6 +119,8 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -111,8 +136,9 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -124,6 +150,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -160,6 +188,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -251,12 +281,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -269,10 +301,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 36d62b324..b661e4e70 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -19,7 +19,6 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -34,7 +33,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -66,6 +66,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ff9159933..3bcb0b230 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -43,7 +43,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): # pylint: disable=missing-docstring self._wrap_sock() @@ -147,6 +154,45 @@ class TLSSNI01DualNetworkedServers(BaseDualNetworkedServers): BaseDualNetworkedServers.__init__(self, TLSSNI01Server, *args, **kwargs) +class BadALPNProtos(Exception): + """Error raised when cannot negotiate ALPN protocol.""" + pass + + +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + # Raising an exception causes openssl to terminate handshake and + # send fatal tls alert. + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + raise BadALPNProtos("Got: %s" % str(alpn_protos)) + + class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): """BaseRequestHandler with logging.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 1591187e5..aee592187 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -10,6 +10,7 @@ import unittest from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose import mock import requests @@ -119,6 +120,62 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@unittest.skipUnless( + hasattr(SSL.Connection, "set_alpn_protos") and + hasattr(SSL.Context, "set_alpn_select_callback"), + "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + #TODO: This is not implemented yet, see comments in standalone.py + #def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" diff --git a/acme/acme/testdata/README b/acme/acme/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/acme/testdata/README +++ b/acme/acme/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/acme/testdata/rsa1024_cert.pem b/acme/acme/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/acme/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- From 236f9630e074ad45aa2b834483cbf373aae18566 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 15:04:56 -0700 Subject: [PATCH 519/631] Remove unneeded sys import (#5873) * Remove unneeded sys import. Once upon a time we needed this in some of these setup.py files because we were using sys in the file, but we aren't anymore so let's remove the import. * use setuptools instead of distutils --- acme/setup.py | 2 -- certbot-apache/setup.py | 2 -- certbot-dns-cloudflare/setup.py | 2 -- certbot-dns-cloudxns/setup.py | 2 -- certbot-dns-digitalocean/setup.py | 2 -- certbot-dns-dnsimple/setup.py | 2 -- certbot-dns-dnsmadeeasy/setup.py | 2 -- certbot-dns-google/setup.py | 2 -- certbot-dns-luadns/setup.py | 2 -- certbot-dns-nsone/setup.py | 2 -- certbot-dns-rfc2136/setup.py | 2 -- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 2 -- letshelp-certbot/setup.py | 2 -- setup.py | 1 - 15 files changed, 1 insertion(+), 30 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e91c36b3d..f8196e953 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 0e4304300..59ca3ed4a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 8e1f9d28b..d80a1ae20 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 05998ee6a..3bb7457e3 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index cd3b0613e..1d7c34197 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 10ee710cd..9c678ec60 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a7f44b989..66f3edd55 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index c171e5014..64b332b04 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2c0e35308..8df4f1735 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 821a40655..4425ace16 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 21d9dec29..6ced18b5d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 083cd15ae..9ee587c4e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,6 +1,4 @@ -import sys - -from distutils.core import setup +from setuptools import setup from setuptools import find_packages version = '0.25.0.dev0' diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6889d06e4..067f280d4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b5be07a59..28ce0e962 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/setup.py b/setup.py index ee0470d3a..e32925583 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ import codecs import os import re -import sys from setuptools import setup from setuptools import find_packages From 8e4303af9f30f514fce04b88bff70b6bbb9d43c8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 4 Jun 2018 16:04:47 -0700 Subject: [PATCH 520/631] Reuse ACMEv1 accounts for ACMEv2 (#5902) * Reuse ACMEv1 accounts for ACMEv2 * Correct behavior * add unit tests * add _find_all_inner to comply with interface * acme-staging-v01 --> acme-staging * only create symlink to previous account if there is one there * recurse on server path * update tests and change internal use of load to use server_path * fail gracefully on corrupted account file by returning [] when rmdir fails * only reuse accounts in staging for now --- certbot/account.py | 42 ++++++++++++++++++++---- certbot/configuration.py | 6 +++- certbot/constants.py | 6 ++++ certbot/tests/account_test.py | 62 ++++++++++++++++++++++++++++++++++- 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index 70d9a7fc3..c4eeb1388 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -16,6 +16,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import constants from certbot import errors from certbot import interfaces from certbot import util @@ -142,7 +143,11 @@ class AccountFileStorage(interfaces.AccountStorage): self.config.strict_permissions) def _account_dir_path(self, account_id): - return os.path.join(self.config.accounts_dir, account_id) + return self._account_dir_path_for_server_path(account_id, self.config.server_path) + + def _account_dir_path_for_server_path(self, account_id, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + return os.path.join(accounts_dir, account_id) @classmethod def _regr_path(cls, account_dir_path): @@ -156,22 +161,44 @@ class AccountFileStorage(interfaces.AccountStorage): def _metadata_path(cls, account_dir_path): return os.path.join(account_dir_path, "meta.json") - def find_all(self): + def _find_all_for_server_path(self, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) try: - candidates = os.listdir(self.config.accounts_dir) + candidates = os.listdir(accounts_dir) except OSError: return [] accounts = [] for account_id in candidates: try: - accounts.append(self.load(account_id)) + accounts.append(self._load_for_server_path(account_id, server_path)) except errors.AccountStorageError: logger.debug("Account loading problem", exc_info=True) + + + if not accounts and server_path in constants.LE_REUSE_SERVERS: + # find all for the next link down + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_accounts = self._find_all_for_server_path(prev_server_path) + # if we found something, link to that + if prev_accounts: + if os.path.islink(accounts_dir): + os.unlink(accounts_dir) + else: + try: + os.rmdir(accounts_dir) + except OSError: + return [] + prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) + os.symlink(prev_account_dir, accounts_dir) + accounts = prev_accounts return accounts - def load(self, account_id): - account_dir_path = self._account_dir_path(account_id) + def find_all(self): + return self._find_all_for_server_path(self.config.server_path) + + def _load_for_server_path(self, account_id, server_path): + account_dir_path = self._account_dir_path_for_server_path(account_id, server_path) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) @@ -193,6 +220,9 @@ class AccountFileStorage(interfaces.AccountStorage): account_id, acc.id)) return acc + def load(self, account_id): + return self._load_for_server_path(account_id, self.config.server_path) + def save(self, account, acme): self._save(account, acme, regr_only=False) diff --git a/certbot/configuration.py b/certbot/configuration.py index 297795609..daf514be8 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -65,8 +65,12 @@ class NamespaceConfig(object): @property def accounts_dir(self): # pylint: disable=missing-docstring + return self.accounts_dir_for_server_path(self.server_path) + + def accounts_dir_for_server_path(self, server_path): + """Path to accounts directory based on server_path""" return os.path.join( - self.namespace.config_dir, constants.ACCOUNTS_DIR, self.server_path) + self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) @property def backup_dir(self): # pylint: disable=missing-docstring diff --git a/certbot/constants.py b/certbot/constants.py index 1dd25e799..c6746e888 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -158,6 +158,12 @@ CONFIG_DIRS_MODE = 0o755 ACCOUNTS_DIR = "accounts" """Directory where all accounts are saved.""" +LE_REUSE_SERVERS = { + 'acme-staging-v02.api.letsencrypt.org/directory': + 'acme-staging.api.letsencrypt.org/directory' +} +"""Servers that can reuse accounts from other servers.""" + BACKUP_DIR = "backups" """Directory (relative to `IConfig.work_dir`) where backups are kept.""" diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ebda56af..a8059fbcf 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -95,6 +95,7 @@ class AccountMemoryStorageTest(unittest.TestCase): class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot.account.AccountFileStorage.""" + #pylint: disable=too-many-public-methods def setUp(self): super(AccountFileStorageTest, self).setUp() @@ -159,7 +160,8 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertEqual([], self.storage.find_all()) def test_find_all_load_skips(self): - self.storage.load = mock.MagicMock( + # pylint: disable=protected-access + self.storage._load_for_server_path = mock.MagicMock( side_effect=["x", errors.AccountStorageError, "z"]) with mock.patch("certbot.account.os.listdir") as mock_listdir: mock_listdir.return_value = ["x", "y", "z"] @@ -175,6 +177,64 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.AccountStorageError, self.storage.load, "x" + self.acc.id) + def _set_server(self, server): + self.config.server = server + from certbot.account import AccountFileStorage + self.storage = AccountFileStorage(self.config) + + def test_find_all_neither_exists(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.assertEqual([], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + + def test_find_all_find_before_save(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + # we shouldn't have created a v1 account + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_save_before_find(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + self.assertTrue(os.path.isdir(self.config.accounts_dir)) + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_server_downgrade(self): + # don't use v2 accounts with a v1 url + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + + def test_upgrade_version(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([self.acc], self.storage.find_all()) + + @mock.patch('os.rmdir') + def test_corrupted_account(self, mock_rmdir): + # pylint: disable=protected-access + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + mock_rmdir.side_effect = OSError + self.storage._load_for_server_path = mock.MagicMock( + side_effect=errors.AccountStorageError) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() From 09a28c7a27fd01d1517e6381ffb37501bffca292 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 4 Jun 2018 17:44:51 -0700 Subject: [PATCH 521/631] Allow multiple add_headers directives (#6068) * fix(nginx-hsts): allow multiple add_headers * test(nginx): fix nginx tests --- certbot-nginx/certbot_nginx/parser.py | 2 +- .../certbot_nginx/tests/configurator_test.py | 15 ++++++++++++--- certbot-nginx/certbot_nginx/tests/parser_test.py | 5 +++-- .../testdata/etc_nginx/sites-enabled/headers.com | 4 ++++ 4 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 5bc7946dc..7d1da2e73 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -566,7 +566,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e88dcb8e0..0668a38c9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -47,7 +47,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(10, len(self.config.parser.parsed)) + self.assertEqual(11, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -91,7 +91,8 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) + "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", + "headers.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -548,6 +549,14 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[example_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_multiple_headers_hsts(self): + headers_conf = self.config.parser.abs_path('sites-enabled/headers.com') + self.config.enhance("headers.com", "ensure-http-header", + "Strict-Transport-Security") + expected = ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always'] + generated_conf = self.config.parser.parsed[headers_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_http_header_hsts_twice(self): self.config.enhance("www.example.com", "ensure-http-header", "Strict-Transport-Security") @@ -852,7 +861,7 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 5) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 5a37c9565..f6f28e42b 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -49,6 +49,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', 'sites-enabled/example.com', + 'sites-enabled/headers.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', 'sites-enabled/globalssl.com', @@ -77,7 +78,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(7, len( + self.assertEqual(8, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -160,7 +161,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(12, len(vhosts)) + self.assertEqual(13, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com new file mode 100644 index 000000000..6c032928c --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com @@ -0,0 +1,4 @@ +server { + server_name headers.com; + add_header X-Content-Type-Options nosniff; +} From d905886f4c63d1546ff723a909a088ab9334b91f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 5 Jun 2018 13:40:48 -0700 Subject: [PATCH 522/631] Automatically select among default vhosts if we have a port preference in nginx (#5944) * automatically select among default vhosts if we have a port preference * ports should be strings in the nginx plugin * clarify port vs preferred_port behavior by adding allow_port_mismatch flag * update all instances of default_vhosts to all_default_vhosts * require port * port should never be None in _get_default_vhost --- certbot-nginx/certbot_nginx/configurator.py | 31 ++++++++++++------- .../certbot_nginx/tests/configurator_test.py | 7 +++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 293f7378e..41b5124b8 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -289,7 +289,8 @@ class NginxConfigurator(common.Installer): if not vhosts: if create_if_no_match: # result will not be [None] because it errors on failure - vhosts = [self._vhost_from_duplicated_default(target_name)] + vhosts = [self._vhost_from_duplicated_default(target_name, True, + str(self.config.tls_sni_01_port))] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -332,9 +333,12 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, port=None): + def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + """if allow_port_mismatch is False, only server blocks with matching ports will be + used as a default server block template. + """ if self.new_vhost is None: - default_vhost = self._get_default_vhost(port, domain) + default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,19 +354,24 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port, domain): + def _get_default_vhost(self, domain, allow_port_mismatch, port): + """Helper method for _vhost_from_duplicated_default; see argument documentation there""" vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one - default_vhosts = [] + all_default_vhosts = [] + port_matching_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - if port is None or self._port_matches(port, addr.get_port()): - default_vhosts.append(vhost) - break + all_default_vhosts.append(vhost) + if self._port_matches(port, addr.get_port()): + port_matching_vhosts.append(vhost) + break - if len(default_vhosts) == 1: - return default_vhosts[0] + if len(port_matching_vhosts) == 1: + return port_matching_vhosts[0] + elif len(all_default_vhosts) == 1 and allow_port_mismatch: + return all_default_vhosts[0] # TODO: present a list of vhosts for user to choose from @@ -471,7 +480,7 @@ class NginxConfigurator(common.Installer): matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts def _port_matches(self, test_port, matching_port): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 0668a38c9..9386c3cd9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -731,6 +731,13 @@ class NginxConfiguratorTest(util.NginxTest): "www.nomatch.com", "example/cert.pem", "example/key.pem", "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_multiple_defaults_ok(self): + foo_conf = self.config.parser.abs_path('foo.conf') + self.config.parser.parsed[foo_conf][2][1][0][1][0][1] = '*:5001' + self.config.version = (1, 3, 1) + self.config.deploy_cert("www.nomatch.com", "example/cert.pem", "example/key.pem", + "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_add_redirect(self): default_conf = self.config.parser.abs_path('sites-enabled/default') foo_conf = self.config.parser.abs_path('foo.conf') From 868e5b831b27ac07cb4a73e04ed9e6fac5987fc7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Jun 2018 17:59:11 -0700 Subject: [PATCH 523/631] Make python setup.py test use pytest for acme (#6072) --- acme/setup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index f8196e953..0fb4d8fff 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,6 +1,7 @@ from setuptools import setup from setuptools import find_packages - +from setuptools.command.test import test as TestCommand +import sys version = '0.25.0.dev0' @@ -33,6 +34,19 @@ docs_extras = [ 'sphinx_rtd_theme', ] +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) setup( name='acme', @@ -65,5 +79,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, + tests_require=["pytest"], test_suite='acme', + cmdclass={"test": PyTest}, ) From 3cffe1449c4e9166b65eaed75022d73b7ad79328 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 07:58:50 -0700 Subject: [PATCH 524/631] Revert "switch signature verification to use pure cryptography (#6000)" (#6074) This reverts commit 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2. --- certbot/crypto_util.py | 36 +++++-------------- certbot/tests/crypto_util_test.py | 10 ------ .../tests/testdata/cert-nosans_nistp256.pem | 11 ------ .../tests/testdata/csr-nosans_nistp256.pem | 8 ----- certbot/tests/testdata/nistp256_key.pem | 5 --- 5 files changed, 9 insertions(+), 61 deletions(-) delete mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 71f6c990c..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,16 +12,11 @@ import os import pyrfc3339 import six import zope.component -from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA -from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey -from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 -from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore +from cryptography.hazmat.backends import default_backend +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -233,26 +228,13 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) + chain, _ = pyopenssl_load_certificate(chain_file.read()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) - pk = chain.public_key() - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") - except (IOError, ValueError, InvalidSignature) as e: + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) + hash_name = cert.signature_hash_algorithm.name + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index baf14b2ef..2fe0e3d30 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,9 +21,6 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') -P256_KEY = test_util.load_vector('nistp256_key.pem') -P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') -P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -220,13 +217,6 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) - def test_cert_sig_match_ec(self): - renewable_cert = mock.MagicMock() - renewable_cert.cert = P256_CERT_PATH - renewable_cert.chain = P256_CERT_PATH - renewable_cert.privkey = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) - def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem deleted file mode 100644 index 4ec3f24ce..000000000 --- a/certbot/tests/testdata/cert-nosans_nistp256.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER -MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD -RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 -MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG -A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY -QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK -BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O -Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM ------END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem deleted file mode 100644 index 2f0a671ed..000000000 --- a/certbot/tests/testdata/csr-nosans_nistp256.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ -BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl -LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw -GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg -ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ -A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== ------END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem deleted file mode 100644 index 4be37e49b..000000000 --- a/certbot/tests/testdata/nistp256_key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 -AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ -pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== ------END EC PRIVATE KEY----- From 4ae2390c441f5e2e276ffb0460f097af1c325846 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 13:50:30 -0700 Subject: [PATCH 525/631] Release 0.25.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 47 +++++++++++------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 29 +++++++---- letsencrypt-auto | 47 +++++++++++------- letsencrypt-auto-source/certbot-auto.asc | 16 +++--- letsencrypt-auto-source/letsencrypt-auto | 26 +++++----- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++----- 22 files changed, 124 insertions(+), 95 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0fb4d8fff..e24f4297b 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.25.0.dev0' +version = '0.25.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 59ca3ed4a..18e10223d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 0848080b3..c15a851db 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ @@ -1187,6 +1193,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # @@ -1199,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 50df2a56e..e739c4924 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d80a1ae20..861ad12d7 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3bb7457e3..52aebb374 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1d7c34197..c32a2d003 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9c678ec60..026eb3ed2 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 66f3edd55..799163d1d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 64b332b04..ef4ae0f8c 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 8df4f1735..838d323af 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4425ace16..75d3c25e4 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 6ced18b5d..6042365e7 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 9ee587c4e..b6e8293b0 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 067f280d4..be057a928 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 27c63e266..85b4d9fd8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.25.0.dev0' +__version__ = '0.25.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 259142e62..f40a9aa4c 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,9 +108,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.24.0 (certbot; - darwin 10.13.4) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags + "". (default: CertbotACMEClient/0.25.0 (certbot; + darwin 10.13.5) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -143,6 +143,8 @@ automation: certificate name but does not match the requested domains, renew it now, regardless of whether it is near expiry. (default: False) + --reuse-key When renewing, use the same private key as the + existing certificate. (default: False) --allow-subset-of-names When performing domain validation, do not consider it a failure if authorizations can not be obtained for a @@ -319,6 +321,13 @@ renew: disable it. (default: False) --no-directory-hooks Disable running executables found in Certbot's hook directories during renewal. (default: False) + --disable-renew-updates + Disable automatic updates to your server configuration + that would otherwise be done by the selected installer + plugin, and triggered when the user executes "certbot + renew", regardless of if the certificate is renewed. + This setting does not apply to important TLS + configuration updates. (default: False) certificates: List certificates managed by Certbot @@ -360,8 +369,9 @@ register: e-mail address, should be updated, rather than registering a new account. (default: False) -m EMAIL, --email EMAIL - Email used for registration and recovery contact. - (default: Ask) + Email used for registration and recovery contact. Use + comma to register multiple emails, ex: + u1@example.com,u2@example.com. (default: Ask). --eff-email Share your e-mail address with EFF (default: None) --no-eff-email Don't share your e-mail address with EFF (default: None) @@ -399,7 +409,7 @@ update_symlinks: changed them by hand or edited a renewal configuration file enhance: - Helps to harden the TLS configration by adding security enhancements to + Helps to harden the TLS configuration by adding security enhancements to already existing configuration. plugins: @@ -472,9 +482,9 @@ apache: /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: False) + you. (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES - Let installer handle enabling sites for you.(Only + Let installer handle enabling sites for you. (Only Ubuntu/Debian currently) (default: False) certbot-route53:auth: @@ -628,7 +638,8 @@ nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT - Nginx server root directory. (default: /etc/nginx) + Nginx server root directory. (default: + /usr/local/etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and retrieving nginx version number. (default: nginx) diff --git a/letsencrypt-auto b/letsencrypt-auto index 0848080b3..c15a851db 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ @@ -1187,6 +1193,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # @@ -1199,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 641ebaef8..11e9750b5 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlro/1AACgkQTRfJlc2X -dfLm5ggAxCrWU9dmYZKllcFzp7TFOdRap0pmarfL4gwSYj7B/bSceD7ysOyoQ8Ra -7UHuZKAQyurZn1seN49d88Kgor9KWZQ1jZiGkfiEpp8qAkdWzFR8UqYa2/CZtk2l -bExm8YQDwhuKvCObGLDGi3ydcIQpfg/rsBkSTphKYXN/Zebx9mAelZN4CgGRy03Y -3z2UqqnyqFPAg4wUGcNfCgUEbJ5bUPr733vQzjBS2IVUbDbu06/1Y8oYzurezXNS -6lEyvTfC5G8RGlSWupNu7yWviD14M4LnAo6WXWEVH+C+ssJaPrZVhZ6KfEt/Erg3 -k06WZSPDCtOm5EfhDm0Rumqm1owA2g== -=Bc4G +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsYSPUACgkQTRfJlc2X +dfKOzQgAhyWglFSdc2VTdqL5FDkg9yv5HzlpwUKp+Q3Q0Tq7/fecZeMybUnj8aJZ +6kiIQ0TE7vlQiKknxCg883hjoW/g0ZMemHsyVIAB6yi69Xltf/maxwwVdDPd+ens +db3/mRiefW+WE2tf5xPKc5xcch1Ej0H9bTIOyj7sgod/bFMuLEtyT/Y58Sb5gWIK +hIrfpWl0L3IP1EAFGSTbRhhxDPsMI6jveHMAQdh5uYgvDneUMVDwcuF4HW+qpvxG +lVQJqDTCSYpIg2bpzI8KHqsiHoe37Au5KbxewPb7Mmx91QE9u3deuPLM/jutHqQ7 +kGs5isnImrtQODSfAnWQbkq9BQocdA== +=EBXk -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 28281e20d..c15a851db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0.dev0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4a937e7e0c442056410a4d75bc6fc64fa7fc5ed8..12330ad162164927502aadb9e73375d8a40b3d1a 100644 GIT binary patch literal 256 zcmV+b0ssC$@8wI45>gS1>Vf-0?P)E}7cN~xWB;XyfZbYHf67R;8??@TB|A<6&Ymtv za~(Bb7VdlqKAz=5dK^kLhO%Pu%mcZ{cVC7weDiho<&%Nz(;*Xb*c(;*W-^&XC=|OD_~8fI^IFAUF~p|$&jhTt;E~D3eY}R9 z()EQT9^Wu3iJV-m%DTGS!1TdJ#YvLh-mJo<+p^7z6cnVGbv2Zai!|_#2965E&iQo& zOyxwu*TNkoH+8^qQvf_x;y1Sb&ZX_LL&2M~e^bphBFP!(!0Q2!3tl{Y_@sRtrU<`!&JJ7Yet?gj zP<-mLp*eOt8&aten_HT4Xaq=Kx#zQxgGH^wqLqmDbiNH^#Z>!uQ#lxbz9TmzQ``gA zepS_2=ZD+oI|Kl|@HhjznV9>uJqAGPzsGZ`_Tr$cOh5JZM+hOCqkF1gQK5rq?01Ha z$#L4=i}~HfWo>cf%K<$CERWB0i$y+n$+{zxu0TOGArrjm@c)#iwau&wz!kfWn^wMB GGm3kHFn Date: Wed, 6 Jun 2018 13:50:46 -0700 Subject: [PATCH 526/631] Bump version to 0.26.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e24f4297b..ecafac61a 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.25.0' +version = '0.26.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 18e10223d..7a0dc43bf 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e739c4924..bf86df5da 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 861ad12d7..055a8cffc 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 52aebb374..2c0c074be 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c32a2d003..bd5f2ff26 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 026eb3ed2..581c478b8 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 799163d1d..c5d310d71 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ef4ae0f8c..01d979c2b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 838d323af..44f67c0a7 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 75d3c25e4..87a7da4cc 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 6042365e7..c1fffc4a2 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index b6e8293b0..8e2821332 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index be057a928..3de257a5f 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 85b4d9fd8..3ae0e315b 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.25.0' +__version__ = '0.26.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c15a851db..cac73dcbd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.26.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From eec37f65a83ccf185a67378b6b8010b37f4fba1f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 19:01:55 -0700 Subject: [PATCH 527/631] Update changelog for 0.25.0 (#6076) --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 044e55250..5facc5380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,70 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.25.0 - 2018-06-06 + +### Added + +* Support for the ready status type was added to acme. Without this change, + Certbot and acme users will begin encountering errors when using Let's + Encrypt's ACMEv2 API starting on June 19th for the staging environment and + July 5th for production. See + https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more + information. +* Certbot now accepts the flag --reuse-key which will cause the same key to be + used in the certificate when the lineage is renewed rather than generating a + new key. +* You can now add multiple email addresses to your ACME account with Certbot by + providing a comma separated list of emails to the --email flag. +* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme. + For more information, see + https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1. +* acme now supports specifying the source address to bind to when sending + outgoing connections. You still cannot specify this address using Certbot. +* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't + already have an account registered at that server URL, Certbot will + automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint + if it exists. +* Interfaces were added to Certbot allowing plugins to be called at additional + points. The `GenericUpdater` interface allows plugins to perform actions + every time `certbot renew` is run, regardless of whether any certificates are + due for renewal, and the `RenewDeployer` interface allows plugins to perform + actions when a certificate is renewed. See `certbot.interfaces` for more + information. + +### Changed + +* When running Certbot with --dry-run and you don't already have a staging + account, the created account does not contain an email address even if one + was provided to avoid expiration emails from Let's Encrypt's staging server. +* certbot-nginx does a better job of automatically detecting the location of + Nginx's configuration files when run on BSD based systems. +* acme now requires and uses pytest when running tests with setuptools with + `python setup.py test`. +* `certbot config_changes` no longer waits for user input before exiting. + +### Fixed + +* Misleading log output that caused users to think that Certbot's standalone + plugin failed to bind to a port when performing a challenge has been + corrected. +* An issue where certbot-nginx would fail to enable HSTS if the server block + already had an `add_header` directive has been resolved. +* certbot-nginx now does a better job detecting the server block to base the + configuration for TLS-SNI challenges on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with functional changes were: + +* acme +* certbot +* certbot-apache +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/54?closed=1 + ## 0.24.0 - 2018-05-02 ### Added From da6320f4d1b816592991f144caa4d6197faafd14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:11:06 -0700 Subject: [PATCH 528/631] Stop testing against Debian 7. (#6077) Debian Wheezy is no longer supported (see https://wiki.debian.org/LTS) and Amazon shut down their Debian 7 mirrors so let's stop trying to use Debian 7 during testing. --- tests/letstest/targets.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 9c1aca24e..766b4ea09 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -22,24 +22,6 @@ targets: # #cloud-init # runcmd: # - [ apt-get, install, -y, curl ] - - ami: ami-e0efab88 - name: debian7.8.aws.1 - type: ubuntu - virt: hvm - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] - - ami: ami-e6eeaa8e - name: debian7.8.aws.1_32bit - type: ubuntu - virt: pv - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-60b6c60a From 780a1b3a26192f06c58fce157db5c2a32ecbdc23 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:43:45 -0700 Subject: [PATCH 529/631] Don't require festival during signing. (#6079) Festival isn't available via Homebrew and is only needed to read the hash aloud, so let's not make it a strict requirement that it's installed. You can simply read the hash from the terminal instead. --- tools/offline-sigrequest.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh index 7706796ef..6443ae8af 100755 --- a/tools/offline-sigrequest.sh +++ b/tools/offline-sigrequest.sh @@ -2,17 +2,17 @@ set -o errexit -if ! `which festival > /dev/null` ; then - echo Please install \'festival\'! - exit 1 -fi - function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do - cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ - echo -n '(SayText "'; \ - sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ - echo '")' ) | festival + if ! `which festival > /dev/null` ; then + echo \`festival\` is not installed! + echo Please install it to read the hash aloud + else + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ + echo -n '(SayText "'; \ + sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + echo '")' ) | festival + fi done echo 'Paste in the data from the QR code, then type Ctrl-D:' From 3a8de6d172d3d91348cb4b817586869a756cc2a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 07:50:36 -0700 Subject: [PATCH 530/631] Upgrade pinned twine version. (#6078) For the past couple of releases, twine has errored while trying to upload packages and this is fixed by upgrading to a newer version of twine. This commit updates our pinned version installed when using tools/venv.sh to the latest available version. pkginfo had to be upgraded as well to support the latest version of twine. --- tools/dev_constraints.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d965d4470..777222ffb 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -36,7 +36,7 @@ oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkginfo==1.4.1 +pkginfo==1.4.2 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 @@ -66,7 +66,7 @@ tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 -twine==1.9.1 +twine==1.11.0 typed-ast==1.1.0 typing==3.6.4 uritemplate==0.6 From afa7e3fb8266fb6af7c36d46eb8ee20e844d8107 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 14:45:23 -0700 Subject: [PATCH 531/631] Unrevert #6000 and silence deprecation warnings (#6082) * Revert "Revert "switch signature verification to use pure cryptography (#6000)" (#6074)" This reverts commit 3cffe1449c4e9166b65eaed75022d73b7ad79328. * Fixes #6073. This silences the deprecation warnings from cryptography. I looked into only silencing the cryptography warning specifically in the function, however, CryptographyDeprecationWarning doesn't seem to be publicly documented, so we probably shouldn't depend on it. --- certbot/crypto_util.py | 40 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 +++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 +++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 ++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..943e2c87f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -7,16 +7,21 @@ import hashlib import logging import os - +import warnings import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,28 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY----- From da028ca9c27acf31a6266ae5c0abe53922da86b4 Mon Sep 17 00:00:00 2001 From: Roland Bracewell Shoemaker Date: Mon, 11 Jun 2018 11:59:57 -0700 Subject: [PATCH 532/631] Wrap TLS-ALPN extension with ASN.1 (#6089) * Wrap TLS-ALPN extension with ASN.1 * Fix test --- acme/acme/challenges.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index ce788e2cc..30983e28f 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -546,7 +546,9 @@ class TLSALPN01Response(KeyAuthorizationChallengeResponse): key.generate_key(crypto.TYPE_RSA, bits) - der_value = b"DER:" + codecs.encode(self.h, 'hex') + # Instead of using a ASN.1 encoding library just append the OCTET STRING tag (0x04) + # and the length of the SHA256 hash (0x20) since both of these should never change + der_value = b"DER:0420" + codecs.encode(self.h, 'hex') acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, critical=True, value=der_value) @@ -592,7 +594,8 @@ class TLSALPN01Response(KeyAuthorizationChallengeResponse): # way to get full OID of an unknown extension from pyopenssl. if ext.get_short_name() == b'UNDEF': data = ext.get_data() - return data == self.h + # Add the ASN.1 tag/length prefix to the hash before comparison + return data == b'\x04\x20' + self.h return False From 95892cd4ab57636f39dfd5b31b69b810f42d7481 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jun 2018 17:24:19 -0700 Subject: [PATCH 533/631] Require acme>=0.25.0 for nginx (#6099) --- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 65f5a758e..e70ac0c7f 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 -e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3de257a5f..d9cb4a9c2 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -7,10 +7,7 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - # This plugin works with an older version of acme, but Certbot does not. - # 0.22.0 is specified here to work around - # https://github.com/pypa/pip/issues/988. - 'acme>0.21.1', + 'acme>=0.25.0', 'certbot>0.21.1', 'mock', 'PyOpenSSL', From 9f20fa0ef969811803042be4a60f8127f6883e34 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jun 2018 17:31:22 -0700 Subject: [PATCH 534/631] Fixes #6085. (#6091) The value of norecusedirs is the default in newer versions of pytest which is listed at https://docs.pytest.org/en/3.0.0/customize.html#confval-norecursedirs. --- acme/MANIFEST.in | 1 + acme/pytest.ini | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 acme/pytest.ini diff --git a/acme/MANIFEST.in b/acme/MANIFEST.in index 5367e484a..1619bef69 100644 --- a/acme/MANIFEST.in +++ b/acme/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE.txt include README.rst +include pytest.ini recursive-include docs * recursive-include examples * recursive-include acme/testdata * diff --git a/acme/pytest.ini b/acme/pytest.ini new file mode 100644 index 000000000..0c07ceac7 --- /dev/null +++ b/acme/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = .* build dist CVS _darcs {arch} *.egg From c9ae365f6678ae64134a9408185601e124cdf296 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jun 2018 14:20:15 -0700 Subject: [PATCH 535/631] 0.25.1 update for master (#6110) * Release 0.25.1 (cherry picked from commit 21b5e4eadb445d0a3dcd8cebb709a9fd15e18278) * Bump version to 0.26.0 --- certbot-auto | 26 +++++++++--------- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/certbot-auto b/certbot-auto index c15a851db..d2cfa672d 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.25.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f40a9aa4c..49821a7b5 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.0 (certbot; + "". (default: CertbotACMEClient/0.25.1 (certbot; darwin 10.13.5) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags encoded in the user agent are: --duplicate, --force- diff --git a/letsencrypt-auto b/letsencrypt-auto index c15a851db..d2cfa672d 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.25.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 11e9750b5..67bab66d4 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsYSPUACgkQTRfJlc2X -dfKOzQgAhyWglFSdc2VTdqL5FDkg9yv5HzlpwUKp+Q3Q0Tq7/fecZeMybUnj8aJZ -6kiIQ0TE7vlQiKknxCg883hjoW/g0ZMemHsyVIAB6yi69Xltf/maxwwVdDPd+ens -db3/mRiefW+WE2tf5xPKc5xcch1Ej0H9bTIOyj7sgod/bFMuLEtyT/Y58Sb5gWIK -hIrfpWl0L3IP1EAFGSTbRhhxDPsMI6jveHMAQdh5uYgvDneUMVDwcuF4HW+qpvxG -lVQJqDTCSYpIg2bpzI8KHqsiHoe37Au5KbxewPb7Mmx91QE9u3deuPLM/jutHqQ7 -kGs5isnImrtQODSfAnWQbkq9BQocdA== -=EBXk +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsgc/cACgkQTRfJlc2X +dfLjBgf/bHZn/q+Dqn34uBXHymRSce7UxQn17izcKAt7hZBl4j4sebQ9+0jjuNur +zrW8b0XJ0PsI10GG9qHR3ajC+04pWfRritnK1g4Ycb/pDcUkWo+8uRwr7skAVcvC +oa8ToBS3iUbd3csFl1mu1BGACUHLvVs2cYdDtMuJj8wjsVZ7KnWBGKULAskwmU4Z +VVUxeUrG9f+2kT35meEJUk91FS+4tmqNIVsVlBzf0Q0ZU1iQnV56dMwTqFRzdDJ2 +DBecE0GwuYnKXo2I7kIYaqACQmk9YFh55Sh0K9PbQxyv7YEZXZtkcdqFqyhxy3Nh +EJ2kurFaM3/VmLljc/rW8QW8B3QNbw== +=pkDz -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index cac73dcbd..dc9190630 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 12330ad162164927502aadb9e73375d8a40b3d1a..f266c93e99a6c6b451a0dd066cf33fc069cac53c 100644 GIT binary patch literal 256 zcmV+b0ssDtYs3=1`HBT)24SsR1u@$W7ZPSU$S&OOtBrFsR+98n78ySvPt!!G z;8VuT@C4ln966cf7@p1|^2x*#OsORv*dICX|C;T?NkAcYvB#;S?b|v(@U=jc^F-3_=^g!<2^ko94v~{-OR8 zs(jE6oLCW6yS;4jMUbxlVGl~J&StH_A-KuM!mMbFXeL^Jnpkr41{l6#rWYsfUp^(i zzO<}YgQBPwyO%n@J11S;c~Voa9yf?}P0YBYWV5x#$dd)Xb67iJaEyXJESY09t?}#b GsB~g1X@A53 literal 256 zcmV+b0ssC$@8wI45>gS1>Vf-0?P)E}7cN~xWB;XyfZbYHf67R;8??@TB|A<6&Ymtv za~(Bb7VdlqKAz=5dK^kLhO%Pu%mcZ{cVC7weDiho<&%Nz(;*Xb*c(;*W-^&XC=|OD_~8fI^IFAUF~p|$&jhTt;E~D3eY}R9 z()EQT9^Wu3iJV-m%DTGS!1TdJ#YvLh-mJo<+p^7z6cnVGbv2Zai!|_#2965E&iQo& zOyxwu*TNkoH+8^qQvf_x;y1Sb&ZX_LL&2M~e^bphBFP!(!0Q2!3tl{Y_@sR Date: Wed, 13 Jun 2018 14:20:43 -0700 Subject: [PATCH 536/631] add 0.25.1 changelog (#6111) --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5facc5380..88251e48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.25.1 - 2018-06-13 + +### Fixed + +* TLS-ALPN-01 support has been removed from our acme library. Using our current + dependencies, we are unable to provide a correct implementation of this + challenge so we decided to remove it from the library until we can provide + proper support. +* Issues causing test failures when running the tests in the acme package with + pytest<3.0 has been resolved. +* certbot-nginx now correctly depends on acme>=0.25.0. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* acme +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/56?closed=1 + ## 0.25.0 - 2018-06-06 ### Added From c4ae376279e90ef12b4461f958403d4e517e6348 Mon Sep 17 00:00:00 2001 From: r5d Date: Wed, 13 Jun 2018 18:24:51 -0400 Subject: [PATCH 537/631] Add autorenew option to `renew` subcommand (#5911) * Add autorenew option to `renew` subcommand. * Change default value for 'autorenew' cli option. * Update certbot.cli.prepare_and_parse_args (autorenew) Set `default` for --autorenew and --no-autorenew. * Update certbot.storage.RenewableCert.should_autorenew. - Remove `interactive` argument in RenewableCert.should_autorenew. - Update certbot.renewal.should_renew. * Move autorenew enable/disable check to certbot.storage. - Remove autorenew enable/disable check in `certbot.renewal.handle_renewal_request`. - Fix RenewableCert.autorenewal_is_enabled; autorenew is stored in 'renewalparams'. - Add autorenew enable/disable check in `RenewableCert.should_autorenew`. - Update tests test_time_interval_judgments, test_autorenewal_is_enabled, test_should_autorenew tests in storage_test.py * certbot: Update RenewableCert.should_autorenew Remove block that sets autorenew option in the renewal configuration file. * certbot: Update prepare_and_parse_args. Remove --autorenew option. * certbot: Update CLI_DEFAULTS. Set default of `autorenew` to True. * Remove unused imports in certbot.storage. --- certbot/cli.py | 4 ++++ certbot/constants.py | 1 + certbot/renewal.py | 5 +++-- certbot/storage.py | 12 ++++-------- certbot/tests/storage_test.py | 18 ++++++++++++------ 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 05e316133..24e0bddac 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,6 +1220,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis " when the user executes \"certbot renew\", regardless of if the certificate" " is renewed. This setting does not apply to important TLS configuration" " updates.") + helpful.add( + "renew", "--no-autorenew", action="store_false", + default=flag_default("autorenew"), dest="autorenew", + help="Disable auto renewal of certificates.") helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) diff --git a/certbot/constants.py b/certbot/constants.py index c6746e888..93bc269af 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -37,6 +37,7 @@ CLI_DEFAULTS = dict( expand=False, renew_by_default=False, renew_with_new_domains=False, + autorenew=True, allow_subset_of_names=False, tos=False, account=None, diff --git a/certbot/renewal.py b/certbot/renewal.py index aa8c9722a..f50131028 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -36,7 +36,8 @@ STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "pre_hook", "post_hook", "tls_sni_01_address", "http01_address"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key"] +BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key", + "autorenew"] CONFIG_ITEMS = set(itertools.chain( BOOL_CONFIG_ITEMS, INT_CONFIG_ITEMS, STR_CONFIG_ITEMS, ('pref_challs',))) @@ -261,7 +262,7 @@ def should_renew(config, lineage): if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True - if lineage.should_autorenew(interactive=True): + if lineage.should_autorenew(): logger.info("Cert is due for renewal, auto-renewing...") return True if config.dry_run: diff --git a/certbot/storage.py b/certbot/storage.py index c453e55b0..5b2293bd1 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -920,10 +920,10 @@ class RenewableCert(object): :rtype: bool """ - return ("autorenew" not in self.configuration or - self.configuration.as_bool("autorenew")) + return ("autorenew" not in self.configuration["renewalparams"] or + self.configuration["renewalparams"].as_bool("autorenew")) - def should_autorenew(self, interactive=False): + def should_autorenew(self): """Should we now try to autorenew the most recent cert version? This is a policy question and does not only depend on whether @@ -934,16 +934,12 @@ class RenewableCert(object): Note that this examines the numerically most recent cert version, not the currently deployed version. - :param bool interactive: set to True to examine the question - regardless of whether the renewal configuration allows - automated renewal (for interactive use). Default False. - :returns: whether an attempt should now be made to autorenew the most current cert version in this lineage :rtype: bool """ - if interactive or self.autorenewal_is_enabled(): + if self.autorenewal_is_enabled(): # Consider whether to attempt to autorenew this cert now # Renewals on the basis of revocation diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 09c752ebe..aa6c52ad4 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -383,8 +383,9 @@ class RenewableCertTests(BaseRenewableCertTest): os.unlink(self.test_rc.cert) self.assertRaises(errors.CertStorageError, self.test_rc.names) + @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.datetime") - def test_time_interval_judgments(self, mock_datetime): + def test_time_interval_judgments(self, mock_datetime, mock_cli): """Test should_autodeploy() and should_autorenew() on the basis of expiry time windows.""" test_cert = test_util.load_vector("cert_512.pem") @@ -399,6 +400,8 @@ class RenewableCertTests(BaseRenewableCertTest): f.write(test_cert) mock_datetime.timedelta = datetime.timedelta + mock_cli.set_by_cli.return_value = False + self.test_rc.configuration["renewalparams"] = {} for (current_time, interval, result) in [ # 2014-12-13 12:00:00+00:00 (about 5 days prior to expiry) @@ -451,22 +454,25 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(self.test_rc.should_autodeploy()) def test_autorenewal_is_enabled(self): + self.test_rc.configuration["renewalparams"] = {} self.assertTrue(self.test_rc.autorenewal_is_enabled()) - self.test_rc.configuration["autorenew"] = "1" + self.test_rc.configuration["renewalparams"]["autorenew"] = "True" self.assertTrue(self.test_rc.autorenewal_is_enabled()) - self.test_rc.configuration["autorenew"] = "0" + self.test_rc.configuration["renewalparams"]["autorenew"] = "False" self.assertFalse(self.test_rc.autorenewal_is_enabled()) + @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.RenewableCert.ocsp_revoked") - def test_should_autorenew(self, mock_ocsp): + def test_should_autorenew(self, mock_ocsp, mock_cli): """Test should_autorenew on the basis of reasons other than expiry time window.""" # pylint: disable=too-many-statements + mock_cli.set_by_cli.return_value = False # Autorenewal turned off - self.test_rc.configuration["autorenew"] = "0" + self.test_rc.configuration["renewalparams"] = {"autorenew": "False"} self.assertFalse(self.test_rc.should_autorenew()) - self.test_rc.configuration["autorenew"] = "1" + self.test_rc.configuration["renewalparams"]["autorenew"] = "True" for kind in ALL_FOUR: self._write_out_kind(kind, 12) # Mandatory renewal on the basis of OCSP revocation From 453eafb11e5a571e9c63f5e95f2f1bd81c39e89c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 01:41:38 -0700 Subject: [PATCH 538/631] Used packaged acme in oldest tests. (#6112) --- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-dns-route53/local-oldest-requirements.txt | 2 +- local-oldest-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 724b61d3f..4e4aadbd8 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt index 724b61d3f..4e4aadbd8 100644 --- a/certbot-dns-route53/local-oldest-requirements.txt +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 certbot[dev]==0.21.1 diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 2346300a3..1f449acae 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ --e acme[dev] +acme[dev]==0.25.0 From 8b16a56de830a6aca19846fa9d4f234aea87b2d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 01:43:48 -0700 Subject: [PATCH 539/631] remove comment about renewer (#6115) --- certbot/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 8fa6ebfc6..089eab0c3 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -473,8 +473,7 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None): def _determine_account(config): """Determine which account to use. - In order to make the renewer (configuration de/serialization) happy, - if ``config.account`` is ``None``, it will be updated based on the + If ``config.account`` is ``None``, it will be updated based on the user input. Same for ``config.email``. :param config: Configuration object From 3316eac178bd5a41ed75776159f79695a929aad3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 09:55:16 -0700 Subject: [PATCH 540/631] Separate integration coverage (#6113) * check coverage separately * Add coverage minimums for integration tests. --- certbot-nginx/tests/boulder-integration.sh | 2 ++ tests/boulder-integration.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index d6bd767ce..980b5d45a 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -62,3 +62,5 @@ test_deployment_and_rollback nginx6.wtf # note: not reached if anything above fails, hence "killall" at the # top nginx -c $nginx_root/nginx.conf -s stop + +coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ef611e743..2f9130489 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -486,11 +486,11 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi +coverage report --fail-under 65 --include 'certbot/*' --show-missing + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; then . ./certbot-nginx/tests/boulder-integration.sh fi - -coverage report --fail-under 67 -m From adc07ef933dea4526314f17a1b07b2f5c13715f3 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 15 Jun 2018 15:03:58 -0700 Subject: [PATCH 541/631] fix(display): alternate spaces and dashes (#6119) * fix(display): alternate spaces and dashes * add comment --- certbot/display/util.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/certbot/display/util.py b/certbot/display/util.py index 5e97bca4e..e157a1123 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -29,6 +29,10 @@ HELP = "help" ESC = "esc" """Display exit code when the user hits Escape (UNUSED)""" +# Display constants +SIDE_FRAME = ("- " * 39) + "-" +"""Display boundary (alternates spaces, so when copy-pasted, markdown doesn't interpret +it as a heading)""" def _wrap_lines(msg): """Format lines nicely to 80 chars. @@ -111,12 +115,11 @@ class FileDisplay(object): because it won't cause any workflow regressions """ - side_frame = "-" * 79 if wrap: message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( - line=os.linesep, frame=side_frame, msg=message)) + line=os.linesep, frame=SIDE_FRAME, msg=message)) self.outfile.flush() if pause: if self._can_interact(force_interactive): @@ -208,12 +211,10 @@ class FileDisplay(object): if self._return_default(message, default, cli_flag, force_interactive): return default - side_frame = ("-" * 79) + os.linesep - message = _wrap_lines(message) self.outfile.write("{0}{frame}{msg}{0}{frame}".format( - os.linesep, frame=side_frame, msg=message)) + os.linesep, frame=SIDE_FRAME + os.linesep, msg=message)) self.outfile.flush() while True: @@ -386,8 +387,7 @@ class FileDisplay(object): # Write out the message to the user self.outfile.write( "{new}{msg}{new}".format(new=os.linesep, msg=message)) - side_frame = ("-" * 79) + os.linesep - self.outfile.write(side_frame) + self.outfile.write(SIDE_FRAME + os.linesep) # Write out the menu choices for i, desc in enumerate(choices, 1): @@ -397,7 +397,7 @@ class FileDisplay(object): # Keep this outside of the textwrap self.outfile.write(os.linesep) - self.outfile.write(side_frame) + self.outfile.write(SIDE_FRAME + os.linesep) self.outfile.flush() def _get_valid_int_ans(self, max_): @@ -482,12 +482,11 @@ class NoninteractiveDisplay(object): :param bool wrap: Whether or not the application should wrap text """ - side_frame = "-" * 79 if wrap: message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( - line=os.linesep, frame=side_frame, msg=message)) + line=os.linesep, frame=SIDE_FRAME, msg=message)) self.outfile.flush() def menu(self, message, choices, ok_label=None, cancel_label=None, From 5025b4ea9614f87b8e3f8d01c90066fdde664df3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 16:08:47 -0700 Subject: [PATCH 542/631] Add certbot-postfix to tools pep8ify Delint cover++ test more_info() Refactor get_config_var Don't duplicate changes to Postfix config document instance variables Always clear save_notes on save Test deploy_cert and save and add MockPostfix. Move mock and call to InstallerTest Add getters and setters Use postfix getters and setters protect get_config_var bump cover to 100% bump required coverage to 100 s/config_dir/config_utility Decrease minimum version to Postfix 2.6. This is the minimum version that allows us to set ciphers to be used with opportunistic TLS and is the oldest version packaged in any major distro. Use tls_security_level instead of use_tls. smtpd_tls_security_level should be used instead according to Postfix documentation. Test smtpd_tls_security_level conditional make dunder method an under method refactor postconf usage add check_all_output test check_all_output Add and test verify_exe_exists Add PostfixUtilBase Add ReadOnlyMainMap Use _get_output instead of _call Fix split strip typo --- certbot-postfix/certbot_postfix/installer.py | 211 ++++++++---- .../certbot_postfix/installer_test.py | 313 +++++++++++++----- certbot-postfix/certbot_postfix/postconf.py | 62 ++++ certbot-postfix/certbot_postfix/util.py | 122 ++++++- certbot-postfix/certbot_postfix/util_test.py | 92 +++++ tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 9 files changed, 645 insertions(+), 163 deletions(-) create mode 100644 certbot-postfix/certbot_postfix/postconf.py diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 09e1cc18b..387f01051 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,9 +1,7 @@ """Certbot installer plugin for Postfix.""" import logging import os -import string import subprocess -import sys import zope.interface @@ -22,7 +20,15 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Installer): - """Certbot installer plugin for Postfix.""" + """Certbot installer plugin for Postfix. + + :ivar str config_dir: Postfix configuration directory to modify + :ivar dict proposed_changes: configuration parameters and values to + be written to the Postfix config when save() is called + :ivar list save_notes: documentation for proposed changes. This is + cleared and stored in Certbot checkpoints when save() is called + + """ description = "Configure TLS with the Postfix MTA" @@ -53,7 +59,7 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: when version is not supported """ - for param in ("ctl", "config_dir",): + for param in ("ctl", "config_utility",): self._verify_executable_is_available(param) self._set_config_dir() self._check_version() @@ -85,7 +91,7 @@ class Installer(plugins_common.Installer): """ if self.conf("config-dir") is None: - self.config_dir = self.get_config_var("config_directory") + self.config_dir = self._get_config_var("config_directory") else: self.config_dir = self.conf("config-dir") @@ -95,7 +101,7 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: if the version is unsupported """ - if self._get_version() < (2, 11, 0): + if self._get_version() < (2, 6,): raise errors.NotSupportedError('Postfix version is too old') def _lock_config_dir(self): @@ -138,7 +144,7 @@ class Installer(plugins_common.Installer): :raises .PluginError: Unable to find Postfix version. """ - mail_version = self.get_config_var("mail_version", default=True) + mail_version = self._get_config_default("mail_version") return tuple(int(i) for i in mail_version.split('.')) def get_all_names(self): @@ -147,7 +153,7 @@ class Installer(plugins_common.Installer): :rtype: `set` of `str` """ - return set(self.get_config_var(var) + return set(self._get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) def deploy_cert(self, domain, cert_path, @@ -164,12 +170,16 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ + # pylint: disable=unused-argument self.save_notes.append("Configuring TLS for {0}".format(domain)) self._set_config_var("smtpd_tls_cert_file", fullchain_path) self._set_config_var("smtpd_tls_key_file", key_path) self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - self._set_config_var("smtpd_use_tls", "yes") + + # Don't configure opportunistic TLS if it's currently mandatory + if self._get_config_var("smtpd_tls_security_level") != "encrypt": + self._set_config_var("smtpd_tls_security_level", "may") def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -178,6 +188,7 @@ class Installer(plugins_common.Installer): are currently supported """ + # pylint: disable=unused-argument raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) @@ -202,14 +213,13 @@ class Installer(plugins_common.Installer): :raises errors.PluginError: when save is unsuccessful """ - if not self.proposed_changes: - return + if self.proposed_changes: + save_files = set((os.path.join(self.config_dir, "main.cf"),)) + self.add_to_checkpoint(save_files, + "\n".join(self.save_notes), temporary) + self._write_config_changes() + self.proposed_changes.clear() - self.add_to_checkpoint(os.path.join(self.config_dir, "main.cf"), - "\n".join(self.save_notes), temporary) - self._write_config_changes() - - self.proposed_changes.clear() del self.save_notes[:] if title and not temporary: @@ -298,70 +308,67 @@ class Installer(plugins_common.Installer): util.check_call(cmd) - def get_config_var(self, name, default=False): + def _get_config_default(self, name): + """Return the default value of the specified config parameter. + + :param str name: name of the Postfix config default to return + + :returns: default for the specified configuration parameter if it + exists, otherwise, None + :rtype: str or types.NoneType + + :raises errors.PluginError: if an error occurs while running postconf + or parsing its output + + """ + try: + return self._get_value_from_postconf(("-d", name,)) + except (subprocess.CalledProcessError, errors.PluginError): + raise errors.PluginError("Unable to determine the default value of" + " the Postfix parameter {0}".format(name)) + + def _get_config_var(self, name): """Return the value of the specified Postfix config parameter. - :param str name: name of the Postfix config parameter to return - :param bool default: whether or not to return the default value - instead of the actual value + If there is an unsaved change modifying the value of the + specified config parameter, the value after this proposed change + is returned rather than the current value. If the value is + unset, `None` is returned. - :returns: value of the specified configuration parameter - :rtype: str + :param str name: name of the Postfix config parameter to return + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if an error occurs while running postconf + or parsing its output """ - cmd = self._build_cmd_for_config_var(name, default) + if name in self.proposed_changes: + return self.proposed_changes[name] try: - output = util.check_output(cmd) - except subprocess.CalledProcessError: - logger.debug("Encountered an error when running 'postconf'", - exc_info=True) - raise errors.PluginError( - "Unable to determine the value " - "of Postfix parameter {0}".format(name)) - - expected_prefix = name + " =" - if not output.startswith(expected_prefix): - raise errors.PluginError( - "Unexpected output '{0}' from '{1}'".format(output, - ' '.join(cmd))) - - return output[len(expected_prefix):].strip() - - def _build_cmd_for_config_var(self, name, default): - """Return a command to run to get a Postfix config parameter. - - :param str name: name of the Postfix config parameter to return - :param bool default: whether or not to return the default value - instead of the actual value - - :returns: command to run - :rtype: list - - """ - cmd = self._postconf_command_base() - - if default: - cmd.append("-d") - - cmd.append(name) - - return cmd + return self._get_value_from_postconf((name,)) + except (subprocess.CalledProcessError, errors.PluginError): + raise errors.PluginError("Unable to determine the value of" + " the Postfix parameter {0}".format(name)) def _set_config_var(self, name, value): """Set the Postfix config parameter name to value. This method only stores the requested change in memory. The Postfix configuration is not modified until save() is called. + If there's already an identical in progress change or the + Postfix configuration parameter already has the specified value, + no changes are made. :param str name: name of the Postfix config parameter :param str value: value to set the Postfix config parameter to """ - assert isinstance(name, str), "Invalid name value" - assert isinstance(value, str), "Invalid key value" - self.proposed_changes[name] = value - self.save_notes.append("\t* Set {0} to {1}".format(name, value)) + if self._get_config_var(name) != value: + self.proposed_changes[name] = value + self.save_notes.append("\t* Set {0} to {1}".format(name, value)) def _write_config_changes(self): """Write proposed changes to the Postfix config. @@ -369,21 +376,83 @@ class Installer(plugins_common.Installer): :raises errors.PluginError: if an error occurs """ - cmd = self._postconf_command_base() - cmd.extend("{0}={1}".format(name, value) - for name, value in self.proposed_changes.items()) - try: - util.check_call(cmd) + self._run_postconf_command( + "{0}={1}".format(name, value) + for name, value in self.proposed_changes.items()) except subprocess.CalledProcessError: raise errors.PluginError( "An error occurred while updating your Postfix config.") - def _postconf_command_base(self): - """Builds start of a postconf command using the selected config.""" - cmd = [self.conf("config-utility")] + def _get_value_from_postconf(self, postconf_args): + """Runs postconf and extracts the specified config value. + It is assumed that the name of the Postfix config parameter to + parse from the output is the last value in postconf_args. If the + value is unset, `None` is returned. If an error occurs, the + relevant information is logged before an exception is raised. + + :param collections.Iterable args: arguments to postconf + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if unable to parse postconf output + :raises subprocess.CalledProcessError: if postconf fails + + """ + name = postconf_args[-1] + output = self._run_postconf_command(postconf_args) + + try: + return self._parse_postconf_output(output, name) + except errors.PluginError: + logger.debug("An error occurred while parsing postconf output", + exc_info=True) + raise + + def _run_postconf_command(self, args): + """Runs a postconf command using the selected config. + + If postconf exits with a nonzero status, the error is logged + before an exception is raised. + + :param collections.Iterable args: additional arguments to postconf + + :returns: stdout output of postconf + :rtype: str + + :raises subprocess.CalledProcessError: if the command fails + + """ + + cmd = [self.conf("config-utility")] if self.conf("config-dir") is not None: cmd.extend(("-c", self.conf("config-dir"),)) + cmd.extend(args) - return cmd + return util.check_output(cmd) + + def _parse_postconf_output(self, output, name): + """Parses postconf output and returns the specified value. + + If the specified Postfix parameter is unset, `None` is returned. + It is assumed that most one configuration parameter will be + included in the given output. + + :param str output: output from postconf + :param str name: name of the Postfix config parameter to obtain + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if unable to parse postconf ouput + + """ + expected_prefix = name + " =" + if output.count("\n") != 1 or not output.startswith(expected_prefix): + raise errors.PluginError( + "Unexpected output '{0}' from postconf".format(output)) + + value = output[len(expected_prefix):].strip() + return value if value else None diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 69d999b31..c57eda746 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -1,36 +1,25 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - +"""Tests for certbot_postfix.installer.""" import functools -import logging import os import subprocess import unittest import mock -import six from certbot import errors from certbot.tests import util as certbot_test_util -# Fake Postfix Configs -names_only_config = """mydomain = fubard.org -myhostname = mail.fubard.org -myorigin = fubard.org""" - class InstallerTest(certbot_test_util.ConfigTestCase): + # pylint: disable=too-many-public-methods def setUp(self): super(InstallerTest, self).setUp() self.config.postfix_ctl = "postfix" self.config.postfix_config_dir = self.tempdir self.config.postfix_config_utility = "postconf" + self.mock_postfix = MockPostfix(self.tempdir, + {"mail_version": "3.1.4"}) def test_add_parser_arguments(self): options = set(('ctl', 'config-dir', 'config-utility',)) @@ -51,7 +40,29 @@ class InstallerTest(certbot_test_util.ConfigTestCase): with mock.patch(path_surgery_path, return_value=False): with mock.patch(exe_exists_path, return_value=False): - self.assertRaises(errors.NoInstallationError, installer.prepare) + self.assertRaises(errors.NoInstallationError, + installer.prepare) + + def test_postconf_error(self): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = subprocess.CalledProcessError(42, + "a") + with mock.patch(exe_exists_path, return_value=True): + self.assertRaises(errors.PluginError, installer.prepare) + + def test_unexpected_postconf(self): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.return_value = "foobar" + with mock.patch(exe_exists_path, return_value=True): + self.assertRaises(errors.PluginError, installer.prepare) def test_set_config_dir(self): self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") @@ -61,32 +72,100 @@ class InstallerTest(certbot_test_util.ConfigTestCase): expected = self.config.postfix_config_dir self.config.postfix_config_dir = None - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" + self.mock_postfix.set_value("config_directory", expected) exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = [ - "config_directory = " + expected, "mail_version = 3.1.4" - ] - with mock.patch(exe_exists_path, return_value=True): - with mock.patch(check_call_path): - installer.prepare() + with mock.patch(exe_exists_path, return_value=True): + self._mock_postfix_and_call(installer.prepare) self.assertEqual(installer.config_dir, expected) + @mock.patch("certbot_postfix.installer.certbot_util.exe_exists") + def test_old_version(self, mock_exe_exists): + installer = self._create_installer() + mock_exe_exists.return_value = True + self.mock_postfix.set_value("mail_version", "0.0.1") + self._mock_postfix_and_call( + self.assertRaises, errors.NotSupportedError, installer.prepare) + def test_lock_error(self): assert_raises = functools.partial(self.assertRaises, errors.PluginError, self._create_prepared_installer) certbot_test_util.lock_and_call(assert_raises, self.tempdir) - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_all_names(self, mock_check_output): + def test_more_info(self): installer = self._create_prepared_installer() - mock_check_output.side_effect = names_only_config.splitlines() + version = "3.1.2" + self.mock_postfix.set_value("mail_version", version) - result = installer.get_all_names() - self.assertTrue("fubard.org" in result) - self.assertTrue("mail.fubard.org" in result) + output = self._mock_postfix_and_call(installer.more_info) + self.assertTrue("Postfix" in output) + self.assertTrue(self.tempdir in output) + self.assertTrue(version in output) + + def test_get_all_names(self): + config = {"mydomain": "example.org", + "myhostname": "mail.example.org", + "myorigin": "example.org"} + for name, value in config.items(): + self.mock_postfix.set_value(name, value) + + installer = self._create_prepared_installer() + result = self._mock_postfix_and_call(installer.get_all_names) + self.assertEqual(result, set(config.values())) + + def test_deploy(self): + installer = self._create_prepared_installer() + + def deploy_cert(domain): + """Calls deploy_cert for the given domain. + + :param str domain: domain to deploy cert for + + """ + installer.deploy_cert(domain, "foo", "bar", "baz", "qux") + + self._mock_postfix_and_call(deploy_cert, "example.org") + # No calls to postconf are expected so mock isn't needed + deploy_cert("mail.example.org") + + def test_deploy_and_save(self): + self._test_deploy_and_save_common({"smtpd_tls_security_level": "may"}) + + def test_deploy_and_save2(self): + self.mock_postfix.set_value("smtpd_tls_security_level", "encrypt") + self._test_deploy_and_save_common({"smtpd_tls_security_level": + "encrypt"}) + + def _test_deploy_and_save_common(self, expected_config): + key_path = "key_path" + fullchain_path = "fullchain_path" + installer = self._create_prepared_installer() + + for i, domain in enumerate(("example.org", "mail.example.org",)): + self._mock_postfix_and_call( + installer.deploy_cert, domain, "unused", + key_path, "unused", fullchain_path) + if i: + # No mock because Postfix utilities aren't expected to be used + installer.save("noop") + else: + self._mock_postfix_and_call(installer.save, "real save") + + expected_config.setdefault("smtpd_tls_cert_file", fullchain_path) + expected_config.setdefault("smtpd_tls_key_file", key_path) + for key, value in expected_config.items(): + self.assertEqual(self.mock_postfix.get_value(key), value) + + def test_save_error(self): + installer = self._create_prepared_installer() + self._mock_postfix_and_call( + installer.deploy_cert, "example.org", "foo", "bar", "baz", "qux") + + check_call_path = "certbot_postfix.installer.util.check_output" + with mock.patch(check_call_path) as mock_check_call: + mock_check_call.side_effect = subprocess.CalledProcessError(42, + "foo") + self.assertRaises(errors.PluginError, installer.save) def test_enhance(self): self.assertRaises(errors.PluginError, @@ -111,10 +190,10 @@ class InstallerTest(certbot_test_util.ConfigTestCase): ] self.assertRaises(errors.PluginError, installer.restart) - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_reload_success(self, mock_check_call): - installer = self._create_prepared_installer() - installer.restart() + def test_postfix_reload_success(self): + with mock.patch("certbot_postfix.installer.subprocess.check_call"): + installer = self._create_prepared_installer() + installer.restart() @mock.patch("certbot_postfix.installer.subprocess.check_call") def test_postfix_start_failure(self, mock_check_call): @@ -130,52 +209,6 @@ class InstallerTest(certbot_test_util.ConfigTestCase): ] installer.restart() - def test_get_config_var_success(self): - self.config.postfix_config_dir = None - - command = self._test_get_config_var_success_common('foo', False) - self.assertFalse("-c" in command) - self.assertFalse("-d" in command) - - def test_get_config_var_success_with_config(self): - command = self._test_get_config_var_success_common('foo', False) - self.assertTrue("-c" in command) - self.assertFalse("-d" in command) - - def test_get_config_var_success_with_default(self): - self.config.postfix_config_dir = None - - command = self._test_get_config_var_success_common('foo', True) - self.assertFalse("-c" in command) - self.assertTrue("-d" in command) - - @mock.patch("certbot_postfix.installer.logger") - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_config_var_failure(self, mock_check_output, mock_logger): - mock_check_output.side_effect = subprocess.CalledProcessError(42, "foo") - installer = self._create_installer() - self.assertRaises(errors.PluginError, installer.get_config_var, "foo") - self.assertTrue(mock_logger.debug.call_args[1]["exc_info"]) - - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_config_var_unexpected_output(self, mock_check_output): - self.config.postfix_config_dir = None - mock_check_output.return_value = "foo" - - installer = self._create_installer() - self.assertRaises(errors.PluginError, installer.get_config_var, "foo") - - def _test_get_config_var_success_common(self, name, default): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - with mock.patch(check_output_path) as mock_check_output: - value = "bar" - mock_check_output.return_value = name + " = " + value - self.assertEqual(installer.get_config_var(name, default), value) - - return mock_check_output.call_args[0][0] - def _create_prepared_installer(self): """Creates and returns a new prepared Postfix Installer. @@ -188,14 +221,9 @@ class InstallerTest(certbot_test_util.ConfigTestCase): """ installer = self._create_installer() - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - with mock.patch(exe_exists_path, return_value=True): - with mock.patch(check_call_path): - mock_check_output.return_value = "mail_version = 3.1.4" - installer.prepare() + with mock.patch(exe_exists_path, return_value=True): + self._mock_postfix_and_call(installer.prepare) return installer @@ -211,6 +239,113 @@ class InstallerTest(certbot_test_util.ConfigTestCase): from certbot_postfix import installer return installer.Installer(self.config, name) + def _mock_postfix_and_call(self, func, *args, **kwargs): + """Calls func with mocked responses from Postfix utilities. + + :param callable func: function to call with mocked args + :param tuple args: positional arguments to func + :param dict kwargs: keyword arguments to func + + :returns: the return value of func + + """ + check_call_path = "certbot_postfix.installer.subprocess.check_call" + check_output_path = "certbot_postfix.installer.util.check_output" + + with mock.patch(check_call_path) as mock_check_call: + mock_check_call.side_effect = self.mock_postfix + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = self.mock_postfix + return func(*args, **kwargs) + + +class MockPostfix(object): + """A callable to mimic Postfix command line utilities. + + This is best used a side effect to a mock object. All calls to + 'postfix' are noops. For calls to 'postconf', values that are set in + the constructor or through mocked out runs of postconf are + remembered and properly returned if the installer attempts to fetch + the value. If the Postfix installer attempts to obtain a value that + hasn't yet been set, a dummy value is returned. + + :ivar str config_path: path to Postfix main.cf file + + """ + def __init__(self, config_dir, initial_values): + """Create Postfix configuration. + + :param str config_dir: path for Postfix config dir + :param dict initial_values: initial Postfix config values + + """ + initial_values["config_directory"] = config_dir + + self.config_path = os.path.join(config_dir, "main.cf") + self._write_config(initial_values) + + def __call__(self, args, *unused_args, **unused_kwargs): + cmd = os.path.basename(args[0]) + if cmd == "postfix": + return + elif cmd != "postconf": # pragma: no cover + assert False, "Unexpected command '{0}'".format(''.join(args)) + + output = [] + + skip = False + for arg in args[1:]: + if skip: + skip = False + elif arg[0] == "-": + if arg == "-c": + skip = True + elif "=" in arg: + name, _, value = arg.partition("=") + self.set_value(name, value) + else: + output.append("{0} = {1}\n".format(arg, self.get_value(arg))) + + return "\n".join(output) + + def get_value(self, name): + """Returns the value for the Postfix config parameter name. + + If the value isn't set, an empty string is returned. + + :param str name: name of the Postfix config parameter + + :returns: value of the named parameter + :rtype: str + + """ + return self._read_config().get(name, "") + + def set_value(self, name, value): + """Sets the value for a Postfix config parameter. + + :param str name: name of the Postfix config parameter + :param str value: value ot set the parameter to + + """ + config = self._read_config() + config[name] = value + self._write_config(config) + + def _read_config(self): + config = {} + with open(self.config_path) as f: + for line in f: + key, _, value = line.strip().partition(" = ") + config[key] = value + + return config + + def _write_config(self, config): + with open(self.config_path, "w") as f: + f.writelines("{0} = {1}\n".format(key, value) + for key, value in config.items()) + if __name__ == '__main__': - unittest.main() + unittest.main() # pragma: no cover diff --git a/certbot-postfix/certbot_postfix/postconf.py b/certbot-postfix/certbot_postfix/postconf.py new file mode 100644 index 000000000..7738562dd --- /dev/null +++ b/certbot-postfix/certbot_postfix/postconf.py @@ -0,0 +1,62 @@ +"""Classes that wrap the postconf command line utility. + +These classes allow you to interact with a Postfix config like it is a +dictionary, with the getting and setting of values in the config being +handled automatically by the class. + +""" +import collections + +from certbot_postfix import util + + +class ReadOnlyMainMap(util.PostfixUtilBase, collections.Mapping): + """A read-only view of a Postfix main.cf file.""" + + _modifiers = None + """An iterable containing additional CLI flags for postconf.""" + + def __getitem__(self, name): + return next(_parse_main_output(self._get_output([name])))[1] + + def __iter__(self): + for name, _ in _parse_main_output(self._get_output()): + yield name + + def __len__(self): + return sum(1 for _ in _parse_main_output(self._get_output())) + + def _call(self, extra_args=None): + """Runs Postconf and returns the result. + + If self._modifiers is set, it is provided on the command line to + postconf before any values in extra_args. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises subprocess.CalledProcessError: if the command fails + + """ + all_extra_args = [] + for args_list in (self._modifiers, extra_args,): + if args_list is not None: + all_extra_args.extend(args_list) + + return super(ReadOnlyMainMap, self)._call(all_extra_args) + + +def _parse_main_output(output): + """Parses the raw output from Postconf about main.cf. + + :param str output: data postconf wrote to stdout about main.cf + + :returns: generator providing key-value pairs from main.cf + :rtype: generator + + """ + for line in output.splitlines(): + name, _, value = line.partition(" =") + yield name, value.strip() diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index b65e4231e..57196017f 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -1,12 +1,129 @@ """Utility functions for use in the Postfix installer.""" - import logging import subprocess +from certbot import errors +from certbot import util as certbot_util +from certbot.plugins import util as plugins_util + logger = logging.getLogger(__name__) +class PostfixUtilBase(object): + """A base class for wrapping Postfix command line utilities.""" + + def __init__(self, executable, config_dir=None): + """Sets up the Postfix utility class. + + :param str executable: name or path of the Postfix utility + :param str config_dir: path to an alternative Postfix config + + :raises .NoInstallationError: when the executable isn't found + + """ + verify_exe_exists(executable) + + self._base_command = [executable] + if config_dir is not None: + self._base_command.extend(('-c', config_dir,)) + + def _call(self, extra_args=None): + """Runs the Postfix utility and returns the result. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises subprocess.CalledProcessError: if the command fails + + """ + args = list(self._base_command) + if extra_args is not None: + args.extend(extra_args) + return check_all_output(args) + + def _get_output(self, extra_args=None): + """Runs the Postfix utility and returns only stdout output. + + This function relies on self._call for running the utility. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout + :rtype: str + + :raises subprocess.CalledProcessError: if the command fails + + """ + return self._call(extra_args)[0] + + +def check_all_output(*args, **kwargs): + """A version of subprocess.check_output that also captures stderr. + + This is the same as :func:`subprocess.check_output` except output + written to stderr is also captured and returned to the caller. The + return value is a tuple of two strings (rather than byte strings). + To accomplish this, the caller cannot set the stdout, stderr, or + universal_newlines parameters to :class:`subprocess.Popen`. + + Additionally, if the command exits with a nonzero status, output is + not included in the raised :class:`subprocess.CalledProcessError` + because Python 2.6 does not support this. Instead, the failure + including the output is logged. + + :param tuple args: positional arguments for Popen + :param dict kwargs: keyword arguments for Popen + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises ValueError: if arguments are invalid + :raises subprocess.CalledProcessError: if the command fails + + """ + for keyword in ('stdout', 'stderr', 'universal_newlines',): + if keyword in kwargs: + raise ValueError( + keyword + ' argument not allowed, it will be overridden.') + + kwargs['stdout'] = subprocess.PIPE + kwargs['stderr'] = subprocess.PIPE + kwargs['universal_newlines'] = True + + process = subprocess.Popen(*args, **kwargs) + output, err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get('args') + if cmd is None: + cmd = args[0] + logger.debug( + "'%s' exited with %d. stdout output was:\n%s\nstderr output was:\n%s", + cmd, retcode, output, err) + raise subprocess.CalledProcessError(retcode, cmd) + return (output, err) + + +def verify_exe_exists(exe): + """Ensures an executable with the given name is available. + + If an executable isn't found for the given path or name, extra + directories are added to the user's PATH to help find system + utilities that may not be available in the default cron PATH. + + :param str exe: executable path or name + + :raises .NoInstallationError: when the executable isn't found + + """ + if not (certbot_util.exe_exists(exe) or plugins_util.path_surgery(exe)): + raise errors.NoInstallationError( + "Cannot find executable '{0}'.".format(exe)) + + def check_call(*args, **kwargs): """A simple wrapper of subprocess.check_call that logs errors. @@ -63,7 +180,8 @@ def check_output(*args, **kwargs): if retcode: cmd = _get_cmd(*args, **kwargs) logger.debug( - "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) + "'%s' exited with %d. Output was:\n%s", + cmd, retcode, output, exc_info=True) raise subprocess.CalledProcessError(retcode, cmd) return output diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py index 95253e1fd..4a014ca9b 100644 --- a/certbot-postfix/certbot_postfix/util_test.py +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -5,6 +5,98 @@ import unittest import mock +from certbot import errors + + +class PostfixUtilBaseTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostfixUtilBase.""" + + @classmethod + def _create_object(cls, *args, **kwargs): + from certbot_postfix.util import PostfixUtilBase + return PostfixUtilBase(*args, **kwargs) + + @mock.patch('certbot_postfix.util.verify_exe_exists') + def test_no_exe(self, mock_verify): + expected_error = errors.NoInstallationError + mock_verify.side_effect = expected_error + self.assertRaises(expected_error, self._create_object, 'nonexistent') + + def test_object_creation(self): + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self._create_object('existent') + + +class CheckAllOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_all_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_all_output + return check_all_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + err = 'baz' + + mock_popen().communicate.return_value = (output, err) + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + log_args = mock_logger.debug.call_args[0] + for value in (command, retcode, output, err,): + self.assertTrue(value in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + expected = ('bar', '') + mock_popen().communicate.return_value = expected + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), expected) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_stderr_error(self): + self.assertRaises(ValueError, self._call, stderr=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +class VerifyExeExistsTest(unittest.TestCase): + """Tests for certbot_postfix.util.verify_exe_exists.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import verify_exe_exists + return verify_exe_exists(*args, **kwargs) + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_failure(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = mock_path_surgery.return_value = False + self.assertRaises(errors.NoInstallationError, self._call, 'foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + def test_simple_success(self, mock_exe_exists): + mock_exe_exists.return_value = True + self._call('foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = False + mock_path_surgery.return_value = True + self._call('foo') + + class CheckCallTest(unittest.TestCase): """Tests for certbot_postfix.util.check_call.""" diff --git a/tools/venv.sh b/tools/venv.sh index 1533f0e1f..a623ec529 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -25,5 +25,6 @@ fi -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-nginx \ + -e certbot-postfix \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tools/venv3.sh b/tools/venv3.sh index da56c2249..602118004 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,5 +23,6 @@ fi -e certbot-dns-nsone \ -e certbot-dns-route53 \ -e certbot-nginx \ + -e certbot-postfix \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tox.cover.sh b/tox.cover.sh index fc0c9f476..4f6ea2dab 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -43,6 +43,8 @@ cover () { min=99 elif [ "$1" = "certbot_nginx" ]; then min=97 + elif [ "$1" = "certbot_postfix" ]; then + min=100 elif [ "$1" = "letshelp_certbot" ]; then min=100 else diff --git a/tox.ini b/tox.ini index dee14b8b3..87eb77338 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ py26_packages = certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ + certbot-postfix \ letshelp-certbot non_py26_packages = certbot-dns-cloudxns \ @@ -55,6 +56,7 @@ source_paths = certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-nginx/certbot_nginx + certbot-postfix/certbot_postfix letshelp-certbot/letshelp_certbot tests/lock_test.py From 4ba153949de4fb6c4cfeda3d6400bf2bd6a33a73 Mon Sep 17 00:00:00 2001 From: Sydney Li Date: Wed, 14 Feb 2018 16:20:16 -0800 Subject: [PATCH 543/631] Fixing up postfix plugin - Finishing refactor of postconf/postfix command-line utilities - Plugin uses starttls_policy plugin to specify per-domain policies Cleaning up TLS policy code. Print warning when setting configuration parameter that is overridden by master. Update client to use new policy API Cleanup and test fixes Documentation fix smaller fixes Policy is now an enhancement and reverting works Added a README, and small documentation fixes throughout Moving testing infra from starttls repo to certbot-postfix fixing tests and lint Changes against new policy API starttls-everywhere => starttls-policy testing(postfix): Added more varieties of certificates to test against. Moar fixes against policy API. Address comments on README and setup.py Address small comments on postconf and util Address comments in installer Python 3 fixes and Postconf tester extends TempDir test class Mock out postconf calls from tests and test coverage for master overrides More various fixes. Everything minus testing done Remove STARTTLS policy enhancement from this branch. sphinx quickstart 99% test coverage some cleanup and testfixing cleanup leftover files Remove print statement testfix for python 3.4 Revert dockerfile change mypy fix fix(postfix): brad's comments test(postfix): coverage to 100 test(postfix): mypy import mypy types fix(postfix docs): add .rst files and fix build fix(postfix): tls_only and server_only params behave nicely together some cleanup lint fix more comments bump version number --- certbot-postfix/MANIFEST.in | 2 + certbot-postfix/README.rst | 9 + certbot-postfix/certbot_postfix/constants.py | 63 +++ certbot-postfix/certbot_postfix/installer.py | 434 ++++++------------ .../certbot_postfix/installer_test.py | 351 -------------- certbot-postfix/certbot_postfix/postconf.py | 172 +++++-- .../certbot_postfix/tests/__init__.py | 1 + .../certbot_postfix/tests/installer_test.py | 314 +++++++++++++ .../certbot_postfix/tests/postconf_test.py | 107 +++++ .../certbot_postfix/tests/util_test.py | 205 +++++++++ certbot-postfix/certbot_postfix/util.py | 231 +++++++--- certbot-postfix/certbot_postfix/util_test.py | 165 ------- certbot-postfix/docs/.gitignore | 1 + certbot-postfix/docs/Makefile | 20 + certbot-postfix/docs/api.rst | 8 + certbot-postfix/docs/api/installer.rst | 5 + certbot-postfix/docs/api/postconf.rst | 5 + certbot-postfix/docs/conf.py | 190 ++++++++ certbot-postfix/docs/index.rst | 28 ++ certbot-postfix/docs/make.bat | 36 ++ certbot-postfix/local-oldest-requirements.txt | 2 + certbot-postfix/setup.py | 24 +- certbot/constants.py | 2 +- certbot/plugins/disco.py | 1 + 24 files changed, 1438 insertions(+), 938 deletions(-) create mode 100644 certbot-postfix/certbot_postfix/constants.py delete mode 100644 certbot-postfix/certbot_postfix/installer_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/__init__.py create mode 100644 certbot-postfix/certbot_postfix/tests/installer_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/postconf_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/util_test.py delete mode 100644 certbot-postfix/certbot_postfix/util_test.py create mode 100644 certbot-postfix/docs/.gitignore create mode 100644 certbot-postfix/docs/Makefile create mode 100644 certbot-postfix/docs/api.rst create mode 100644 certbot-postfix/docs/api/installer.rst create mode 100644 certbot-postfix/docs/api/postconf.rst create mode 100644 certbot-postfix/docs/conf.py create mode 100644 certbot-postfix/docs/index.rst create mode 100644 certbot-postfix/docs/make.bat create mode 100644 certbot-postfix/local-oldest-requirements.txt diff --git a/certbot-postfix/MANIFEST.in b/certbot-postfix/MANIFEST.in index 97e2ad3df..273381403 100644 --- a/certbot-postfix/MANIFEST.in +++ b/certbot-postfix/MANIFEST.in @@ -1,2 +1,4 @@ include LICENSE.txt include README.rst +recursive-include certbot_postfix/testdata * +recursive-include certbot_postfix/docs * diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index ee88648d3..78fd9a421 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -1 +1,10 @@ +========================== Postfix plugin for Certbot +========================== + +To install your certs with this plugin, run: + +``certbot install --installer postfix --cert-path --key-path -d `` + +And there you go! If you'd like to obtain these certificates via certbot, there's more documentation on how to do this `here `_. + diff --git a/certbot-postfix/certbot_postfix/constants.py b/certbot-postfix/certbot_postfix/constants.py new file mode 100644 index 000000000..40a263a53 --- /dev/null +++ b/certbot-postfix/certbot_postfix/constants.py @@ -0,0 +1,63 @@ +"""Postfix plugin constants.""" + +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Tuple, Union +# pylint: enable=unused-import, no-name-in-module + +MINIMUM_VERSION = (2, 11,) + +# If the value of a default VAR is a tuple, then the values which +# come LATER in the tuple are more strict/more secure. +# Certbot will default to the first value in the tuple, but will +# not override "more secure" settings. + +ACCEPTABLE_SERVER_SECURITY_LEVELS = ("may", "encrypt") +ACCEPTABLE_CLIENT_SECURITY_LEVELS = ("may", "encrypt", + "dane", "dane-only", + "fingerprint", + "verify", "secure") +ACCEPTABLE_CIPHER_LEVELS = ("medium", "high") + +# Exporting certain ciphers to prevent logjam: https://weakdh.org/sysadmin.html +EXCLUDE_CIPHERS = ("aNULL, eNULL, EXPORT, DES, RC4, MD5, PSK, aECDH, " + "EDH-DSS-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA, KRB5-DES, CBC3-SHA",) + + +TLS_VERSIONS = ("SSLv2", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2") +# Should NOT use SSLv2/3. +ACCEPTABLE_TLS_VERSIONS = ("TLSv1", "TLSv1.1", "TLSv1.2") + +# Variables associated with enabling opportunistic TLS. +TLS_SERVER_VARS = { + "smtpd_tls_security_level": ACCEPTABLE_SERVER_SECURITY_LEVELS, +} # type:Dict[str, Tuple[str, ...]] +TLS_CLIENT_VARS = { + "smtp_tls_security_level": ACCEPTABLE_CLIENT_SECURITY_LEVELS, +} # type:Dict[str, Tuple[str, ...]] +# Default variables for a secure MTA server [receiver]. +DEFAULT_SERVER_VARS = { + "smtpd_tls_auth_only": ("yes",), + "smtpd_tls_mandatory_protocols": ("!SSLv2, !SSLv3",), + "smtpd_tls_protocols": ("!SSLv2, !SSLv3",), + "smtpd_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtpd_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtpd_tls_exclude_ciphers": EXCLUDE_CIPHERS, + "smtpd_tls_eecdh_grade": ("strong",), +} # type:Dict[str, Tuple[str, ...]] + +# Default variables for a secure MTA client [sender]. +DEFAULT_CLIENT_VARS = { + "smtp_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtp_tls_exclude_ciphers": EXCLUDE_CIPHERS, + "smtp_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS, +} # type:Dict[str, Tuple[str, ...]] + +CLI_DEFAULTS = dict( + config_dir="/etc/postfix", + ctl="postfix", + config_utility="postconf", + tls_only=False, + ignore_master_overrides=False, + server_only=False, +) +"""CLI defaults.""" diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 387f01051..9ba92ef8f 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,127 +1,139 @@ -"""Certbot installer plugin for Postfix.""" +"""certbot installer plugin for postfix.""" import logging import os -import subprocess import zope.interface +import zope.component +import six from certbot import errors from certbot import interfaces from certbot import util as certbot_util from certbot.plugins import common as plugins_common -from certbot.plugins import util as plugins_util +from certbot_postfix import constants +from certbot_postfix import postconf from certbot_postfix import util +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Dict, List +# pylint: enable=unused-import, no-name-in-module logger = logging.getLogger(__name__) - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Installer): """Certbot installer plugin for Postfix. :ivar str config_dir: Postfix configuration directory to modify - :ivar dict proposed_changes: configuration parameters and values to - be written to the Postfix config when save() is called :ivar list save_notes: documentation for proposed changes. This is cleared and stored in Certbot checkpoints when save() is called + :ivar postconf: Wrapper for Postfix configuration command-line tool. + :type postconf: :class: `certbot_postfix.postconf.ConfigMain` + :ivar postfix: Wrapper for Postfix command-line tool. + :type postfix: :class: `certbot_postfix.util.PostfixUtil` """ description = "Configure TLS with the Postfix MTA" @classmethod def add_parser_arguments(cls, add): - add("ctl", default="postfix", + add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the 'postfix' control program.") - add("config-dir", help="Path to the directory containing the " + # This directory points to Postfix's configuration directory. + add("config-dir", default=constants.CLI_DEFAULTS["config_dir"], + help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " "default configuration paths.") - add("config-utility", default="postconf", + add("config-utility", default=constants.CLI_DEFAULTS["config_utility"], help="Path to the 'postconf' executable.") + add("tls-only", action="store_true", default=constants.CLI_DEFAULTS["tls_only"], + help="Only set params to enable opportunistic TLS and install certificates.") + add("server-only", action="store_true", default=constants.CLI_DEFAULTS["server_only"], + help="Only set server params (prefixed with smtpd*)") + add("ignore-master-overrides", action="store_true", + default=constants.CLI_DEFAULTS["ignore_master_overrides"], + help="Ignore errors reporting overridden TLS parameters in master.cf.") def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) - self.config_dir = None - self.proposed_changes = {} - self.save_notes = [] + # Wrapper around postconf commands + self.postfix = None + self.postconf = None + + # Files to save + self.save_notes = [] # type: List[str] + + self._enhance_func = {} # type: Dict[str, Callable[[str, str], None]] + # Since we only need to enable TLS once for all domains, + # keep track of whether this enhancement was already called. + self._tls_enabled = False def prepare(self): """Prepare the installer. - Finish up any additional initialization. - :raises errors.PluginError: when an unexpected error occurs :raises errors.MisconfigurationError: when the config is invalid :raises errors.NoInstallationError: when can't find installation :raises errors.NotSupportedError: when version is not supported - """ + # Verify postfix and postconf are installed for param in ("ctl", "config_utility",): - self._verify_executable_is_available(param) - self._set_config_dir() - self._check_version() - self.config_test() - self._lock_config_dir() - - def _verify_executable_is_available(self, config_name): - """Asserts the program in the specified config param is found. - - :param str config_name: name of the config param - - :raises .NoInstallationError: when the executable isn't found - - """ - if not certbot_util.exe_exists(self.conf(config_name)): - if not plugins_util.path_surgery(self.conf(config_name)): - raise errors.NoInstallationError( + util.verify_exe_exists(self.conf(param), "Cannot find executable '{0}'. You can provide the " "path to this command with --{1}".format( - self.conf(config_name), - self.option_name(config_name))) + self.conf(param), + self.option_name(param))) - def _set_config_dir(self): - """Ensure self.config_dir is set to the correct path. + # Set up CLI tools + self.postfix = util.PostfixUtil(self.conf('config-dir')) + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) - If the configuration directory to use was set by the user, we'll - use that value, otherwise, we'll find the default path using - 'postconf'. + # Ensure current configuration is valid. + self.config_test() + # Check Postfix version + self._check_version() + self._lock_config_dir() + self.install_ssl_dhparams() + + def config_test(self): + """Test to see that the current Postfix configuration is valid. + + :raises errors.MisconfigurationError: If the configuration is invalid. """ - if self.conf("config-dir") is None: - self.config_dir = self._get_config_var("config_directory") - else: - self.config_dir = self.conf("config-dir") + self.postfix.test() def _check_version(self): """Verifies that the installed Postfix version is supported. :raises errors.NotSupportedError: if the version is unsupported - """ - if self._get_version() < (2, 6,): - raise errors.NotSupportedError('Postfix version is too old') + if self._get_version() < constants.MINIMUM_VERSION: + version_string = '.'.join([str(n) for n in constants.MINIMUM_VERSION]) + raise errors.NotSupportedError('Postfix version must be at least %s' % version_string) def _lock_config_dir(self): """Stop two Postfix plugins from modifying the config at once. :raises .PluginError: if unable to acquire the lock - """ try: - certbot_util.lock_dir_until_exit(self.config_dir) + certbot_util.lock_dir_until_exit(self.conf('config-dir')) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( - "Unable to lock %s", self.config_dir) + "Unable to lock %s" % self.conf('config-dir')) def more_info(self): - """Human-readable string to help the user. - Should describe the steps taken and any relevant info to help the user - decide which plugin to use. - :rtype str: + """Human-readable string to help the user. Describes steps taken and any relevant + info to help the user decide which plugin to use. + + :rtype: str """ return ( "Configures Postfix to try to authenticate mail servers, use " @@ -129,22 +141,19 @@ class Installer(plugins_common.Installer): "Server root: {root}{0}" "Version: {version}".format( os.linesep, - root=self.config_dir, + root=self.conf('config-dir'), version='.'.join([str(i) for i in self._get_version()])) ) def _get_version(self): - """Return the mail version of Postfix. - - Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + """Return the version of Postfix, as a tuple. (e.g. '2.11.3' is (2, 11, 3)) :returns: version :rtype: tuple - :raises .PluginError: Unable to find Postfix version. - + :raises errors.PluginError: Unable to find Postfix version. """ - mail_version = self._get_config_default("mail_version") + mail_version = self.postconf.get_default("mail_version") return tuple(int(i) for i in mail_version.split('.')) def get_all_names(self): @@ -153,9 +162,34 @@ class Installer(plugins_common.Installer): :rtype: `set` of `str` """ - return set(self._get_config_var(var) + return certbot_util.get_filtered_names(self.postconf.get(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def _set_vars(self, var_dict): + """Sets all parameters in var_dict to config file. If current value is already set + as more secure (acceptable), then don't set/overwrite it. + """ + for param, acceptable in six.iteritems(var_dict): + if not util.is_acceptable_value(param, self.postconf.get(param), acceptable): + self.postconf.set(param, acceptable[0], acceptable) + + def _confirm_changes(self): + """Confirming outstanding updates for configuration parameters. + + :raises errors.PluginError: when user rejects the configuration changes. + """ + updates = self.postconf.get_changes() + output_string = "Postfix TLS configuration parameters to update in main.cf:\n" + for name, value in six.iteritems(updates): + output_string += "{0} = {1}\n".format(name, value) + output_string += "Is this okay?\n" + if not zope.component.getUtility(interfaces.IDisplay).yesno(output_string, + force_interactive=True, default=True): + raise errors.PluginError( + "Manually rejected configuration changes.\n" + "Try using --tls-only or --server-only to change a particular" + "subset of configuration parameters.") + def deploy_cert(self, domain, cert_path, key_path, chain_path, fullchain_path): """Configure the Postfix SMTP server to use the given TLS cert. @@ -171,22 +205,26 @@ class Installer(plugins_common.Installer): """ # pylint: disable=unused-argument + if self._tls_enabled: + return + self._tls_enabled = True self.save_notes.append("Configuring TLS for {0}".format(domain)) - self._set_config_var("smtpd_tls_cert_file", fullchain_path) - self._set_config_var("smtpd_tls_key_file", key_path) - self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") - self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - - # Don't configure opportunistic TLS if it's currently mandatory - if self._get_config_var("smtpd_tls_security_level") != "encrypt": - self._set_config_var("smtpd_tls_security_level", "may") + self.postconf.set("smtpd_tls_cert_file", cert_path) + self.postconf.set("smtpd_tls_key_file", key_path) + self._set_vars(constants.TLS_SERVER_VARS) + if not self.conf('server_only'): + self._set_vars(constants.TLS_CLIENT_VARS) + if not self.conf('tls_only'): + self._set_vars(constants.DEFAULT_SERVER_VARS) + if not self.conf('server_only'): + self._set_vars(constants.DEFAULT_CLIENT_VARS) + # Despite the name, this option also supports 2048-bit DH params. + # http://www.postfix.org/FORWARD_SECRECY_README.html#server_fs + self.postconf.set("smtpd_tls_dh1024_param_file", self.ssl_dhparams) + self._confirm_changes() def enhance(self, domain, enhancement, options=None): - """Raises an exception for request for unsupported enhancement. - - :raises .PluginError: this is always raised as no enhancements - are currently supported - + """Raises an exception since this installer doesn't support any enhancements. """ # pylint: disable=unused-argument raise errors.PluginError( @@ -211,248 +249,40 @@ class Installer(plugins_common.Installer): be quickly reversed in the future (challenges) :raises errors.PluginError: when save is unsuccessful - """ - if self.proposed_changes: - save_files = set((os.path.join(self.config_dir, "main.cf"),)) - self.add_to_checkpoint(save_files, - "\n".join(self.save_notes), temporary) - self._write_config_changes() - self.proposed_changes.clear() + save_files = set((os.path.join(self.conf('config-dir'), "main.cf"),)) + self.add_to_checkpoint(save_files, + "\n".join(self.save_notes), temporary) + self.postconf.flush() del self.save_notes[:] if title and not temporary: self.finalize_checkpoint(title) - def config_test(self): - """Make sure the configuration is valid. + def recovery_routine(self): + super(Installer, self).recovery_routine() + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) - :raises .MisconfigurationError: if the config is invalid + def rollback_checkpoints(self, rollback=1): + """Rollback saved checkpoints. + :param int rollback: Number of checkpoints to revert + + :raises .errors.PluginError: If there is a problem with the input or + the function is unable to correctly revert the configuration """ - try: - self._run_postfix_subcommand("check") - except subprocess.CalledProcessError: - raise errors.MisconfigurationError( - "Postfix failed internal configuration check.") + super(Installer, self).rollback_checkpoints(rollback) + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted - """ - logger.info("Reloading Postfix configuration...") - if self._is_postfix_running(): - self._reload() - else: - self._start() + self.postfix.restart() - def _is_postfix_running(self): - """Is Postfix currently running? - - Uses the 'postfix status' command to determine if Postfix is - currently running using the specified configuration files. - - :returns: True if Postfix is running, otherwise, False - :rtype: bool - - """ - try: - self._run_postfix_subcommand("status") - except subprocess.CalledProcessError: - return False - return True - - def _reload(self): - """Instructions Postfix to reload its configuration. - - If Postfix isn't currently running, this method will fail. - - :raises .PluginError: when Postfix cannot reload - - """ - try: - self._run_postfix_subcommand("reload") - except subprocess.CalledProcessError: - raise errors.PluginError( - "Postfix failed to reload its configuration.") - - def _start(self): - """Instructions Postfix to start running. - - :raises .PluginError: when Postfix cannot start - - """ - try: - self._run_postfix_subcommand("start") - except subprocess.CalledProcessError: - raise errors.PluginError("Postfix failed to start") - - def _run_postfix_subcommand(self, subcommand): - """Runs a subcommand of the 'postfix' control program. - - If the command fails, the exception is logged at the DEBUG - level. - - :param str subcommand: subcommand to run - - :raises subprocess.CalledProcessError: if the command fails - - """ - cmd = [self.conf("ctl")] - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) - cmd.append(subcommand) - - util.check_call(cmd) - - def _get_config_default(self, name): - """Return the default value of the specified config parameter. - - :param str name: name of the Postfix config default to return - - :returns: default for the specified configuration parameter if it - exists, otherwise, None - :rtype: str or types.NoneType - - :raises errors.PluginError: if an error occurs while running postconf - or parsing its output - - """ - try: - return self._get_value_from_postconf(("-d", name,)) - except (subprocess.CalledProcessError, errors.PluginError): - raise errors.PluginError("Unable to determine the default value of" - " the Postfix parameter {0}".format(name)) - - def _get_config_var(self, name): - """Return the value of the specified Postfix config parameter. - - If there is an unsaved change modifying the value of the - specified config parameter, the value after this proposed change - is returned rather than the current value. If the value is - unset, `None` is returned. - - :param str name: name of the Postfix config parameter to return - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if an error occurs while running postconf - or parsing its output - - """ - if name in self.proposed_changes: - return self.proposed_changes[name] - - try: - return self._get_value_from_postconf((name,)) - except (subprocess.CalledProcessError, errors.PluginError): - raise errors.PluginError("Unable to determine the value of" - " the Postfix parameter {0}".format(name)) - - def _set_config_var(self, name, value): - """Set the Postfix config parameter name to value. - - This method only stores the requested change in memory. The - Postfix configuration is not modified until save() is called. - If there's already an identical in progress change or the - Postfix configuration parameter already has the specified value, - no changes are made. - - :param str name: name of the Postfix config parameter - :param str value: value to set the Postfix config parameter to - - """ - if self._get_config_var(name) != value: - self.proposed_changes[name] = value - self.save_notes.append("\t* Set {0} to {1}".format(name, value)) - - def _write_config_changes(self): - """Write proposed changes to the Postfix config. - - :raises errors.PluginError: if an error occurs - - """ - try: - self._run_postconf_command( - "{0}={1}".format(name, value) - for name, value in self.proposed_changes.items()) - except subprocess.CalledProcessError: - raise errors.PluginError( - "An error occurred while updating your Postfix config.") - - def _get_value_from_postconf(self, postconf_args): - """Runs postconf and extracts the specified config value. - - It is assumed that the name of the Postfix config parameter to - parse from the output is the last value in postconf_args. If the - value is unset, `None` is returned. If an error occurs, the - relevant information is logged before an exception is raised. - - :param collections.Iterable args: arguments to postconf - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if unable to parse postconf output - :raises subprocess.CalledProcessError: if postconf fails - - """ - name = postconf_args[-1] - output = self._run_postconf_command(postconf_args) - - try: - return self._parse_postconf_output(output, name) - except errors.PluginError: - logger.debug("An error occurred while parsing postconf output", - exc_info=True) - raise - - def _run_postconf_command(self, args): - """Runs a postconf command using the selected config. - - If postconf exits with a nonzero status, the error is logged - before an exception is raised. - - :param collections.Iterable args: additional arguments to postconf - - :returns: stdout output of postconf - :rtype: str - - :raises subprocess.CalledProcessError: if the command fails - - """ - - cmd = [self.conf("config-utility")] - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) - cmd.extend(args) - - return util.check_output(cmd) - - def _parse_postconf_output(self, output, name): - """Parses postconf output and returns the specified value. - - If the specified Postfix parameter is unset, `None` is returned. - It is assumed that most one configuration parameter will be - included in the given output. - - :param str output: output from postconf - :param str name: name of the Postfix config parameter to obtain - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if unable to parse postconf ouput - - """ - expected_prefix = name + " =" - if output.count("\n") != 1 or not output.startswith(expected_prefix): - raise errors.PluginError( - "Unexpected output '{0}' from postconf".format(output)) - - value = output[len(expected_prefix):].strip() - return value if value else None diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py deleted file mode 100644 index c57eda746..000000000 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ /dev/null @@ -1,351 +0,0 @@ -"""Tests for certbot_postfix.installer.""" -import functools -import os -import subprocess -import unittest - -import mock - -from certbot import errors -from certbot.tests import util as certbot_test_util - - -class InstallerTest(certbot_test_util.ConfigTestCase): - # pylint: disable=too-many-public-methods - - def setUp(self): - super(InstallerTest, self).setUp() - self.config.postfix_ctl = "postfix" - self.config.postfix_config_dir = self.tempdir - self.config.postfix_config_utility = "postconf" - self.mock_postfix = MockPostfix(self.tempdir, - {"mail_version": "3.1.4"}) - - def test_add_parser_arguments(self): - options = set(('ctl', 'config-dir', 'config-utility',)) - mock_add = mock.MagicMock() - - from certbot_postfix import installer - installer.Installer.add_parser_arguments(mock_add) - - for call in mock_add.call_args_list: - self.assertTrue(call[0][0] in options) - - def test_no_postconf_prepare(self): - installer = self._create_installer() - - installer_path = "certbot_postfix.installer" - exe_exists_path = installer_path + ".certbot_util.exe_exists" - path_surgery_path = installer_path + ".plugins_util.path_surgery" - - with mock.patch(path_surgery_path, return_value=False): - with mock.patch(exe_exists_path, return_value=False): - self.assertRaises(errors.NoInstallationError, - installer.prepare) - - def test_postconf_error(self): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = subprocess.CalledProcessError(42, - "a") - with mock.patch(exe_exists_path, return_value=True): - self.assertRaises(errors.PluginError, installer.prepare) - - def test_unexpected_postconf(self): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.return_value = "foobar" - with mock.patch(exe_exists_path, return_value=True): - self.assertRaises(errors.PluginError, installer.prepare) - - def test_set_config_dir(self): - self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") - os.mkdir(self.config.postfix_config_dir) - installer = self._create_installer() - - expected = self.config.postfix_config_dir - self.config.postfix_config_dir = None - - self.mock_postfix.set_value("config_directory", expected) - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(exe_exists_path, return_value=True): - self._mock_postfix_and_call(installer.prepare) - self.assertEqual(installer.config_dir, expected) - - @mock.patch("certbot_postfix.installer.certbot_util.exe_exists") - def test_old_version(self, mock_exe_exists): - installer = self._create_installer() - mock_exe_exists.return_value = True - self.mock_postfix.set_value("mail_version", "0.0.1") - self._mock_postfix_and_call( - self.assertRaises, errors.NotSupportedError, installer.prepare) - - def test_lock_error(self): - assert_raises = functools.partial(self.assertRaises, - errors.PluginError, - self._create_prepared_installer) - certbot_test_util.lock_and_call(assert_raises, self.tempdir) - - def test_more_info(self): - installer = self._create_prepared_installer() - version = "3.1.2" - self.mock_postfix.set_value("mail_version", version) - - output = self._mock_postfix_and_call(installer.more_info) - self.assertTrue("Postfix" in output) - self.assertTrue(self.tempdir in output) - self.assertTrue(version in output) - - def test_get_all_names(self): - config = {"mydomain": "example.org", - "myhostname": "mail.example.org", - "myorigin": "example.org"} - for name, value in config.items(): - self.mock_postfix.set_value(name, value) - - installer = self._create_prepared_installer() - result = self._mock_postfix_and_call(installer.get_all_names) - self.assertEqual(result, set(config.values())) - - def test_deploy(self): - installer = self._create_prepared_installer() - - def deploy_cert(domain): - """Calls deploy_cert for the given domain. - - :param str domain: domain to deploy cert for - - """ - installer.deploy_cert(domain, "foo", "bar", "baz", "qux") - - self._mock_postfix_and_call(deploy_cert, "example.org") - # No calls to postconf are expected so mock isn't needed - deploy_cert("mail.example.org") - - def test_deploy_and_save(self): - self._test_deploy_and_save_common({"smtpd_tls_security_level": "may"}) - - def test_deploy_and_save2(self): - self.mock_postfix.set_value("smtpd_tls_security_level", "encrypt") - self._test_deploy_and_save_common({"smtpd_tls_security_level": - "encrypt"}) - - def _test_deploy_and_save_common(self, expected_config): - key_path = "key_path" - fullchain_path = "fullchain_path" - installer = self._create_prepared_installer() - - for i, domain in enumerate(("example.org", "mail.example.org",)): - self._mock_postfix_and_call( - installer.deploy_cert, domain, "unused", - key_path, "unused", fullchain_path) - if i: - # No mock because Postfix utilities aren't expected to be used - installer.save("noop") - else: - self._mock_postfix_and_call(installer.save, "real save") - - expected_config.setdefault("smtpd_tls_cert_file", fullchain_path) - expected_config.setdefault("smtpd_tls_key_file", key_path) - for key, value in expected_config.items(): - self.assertEqual(self.mock_postfix.get_value(key), value) - - def test_save_error(self): - installer = self._create_prepared_installer() - self._mock_postfix_and_call( - installer.deploy_cert, "example.org", "foo", "bar", "baz", "qux") - - check_call_path = "certbot_postfix.installer.util.check_output" - with mock.patch(check_call_path) as mock_check_call: - mock_check_call.side_effect = subprocess.CalledProcessError(42, - "foo") - self.assertRaises(errors.PluginError, installer.save) - - def test_enhance(self): - self.assertRaises(errors.PluginError, - self._create_prepared_installer().enhance, - "example.org", "redirect") - - def test_supported_enhancements(self): - self.assertEqual( - self._create_prepared_installer().supported_enhancements(), []) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_config_test_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") - self.assertRaises(errors.MisconfigurationError, installer.config_test) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_reload_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = [ - None, subprocess.CalledProcessError(42, "foo") - ] - self.assertRaises(errors.PluginError, installer.restart) - - def test_postfix_reload_success(self): - with mock.patch("certbot_postfix.installer.subprocess.check_call"): - installer = self._create_prepared_installer() - installer.restart() - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_start_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") - self.assertRaises(errors.PluginError, installer.restart) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_start_success(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = [ - subprocess.CalledProcessError(42, "foo"), None - ] - installer.restart() - - def _create_prepared_installer(self): - """Creates and returns a new prepared Postfix Installer. - - Calls in prepare() are mocked out so the Postfix version check - is successful. - - :returns: a prepared Postfix installer - :rtype: certbot_postfix.installer.Installer - - """ - installer = self._create_installer() - - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(exe_exists_path, return_value=True): - self._mock_postfix_and_call(installer.prepare) - - return installer - - def _create_installer(self): - """Creates and returns a new Postfix Installer. - - :returns: a new Postfix installer - :rtype: certbot_postfix.installer.Installer - - """ - name = "postfix" - - from certbot_postfix import installer - return installer.Installer(self.config, name) - - def _mock_postfix_and_call(self, func, *args, **kwargs): - """Calls func with mocked responses from Postfix utilities. - - :param callable func: function to call with mocked args - :param tuple args: positional arguments to func - :param dict kwargs: keyword arguments to func - - :returns: the return value of func - - """ - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" - - with mock.patch(check_call_path) as mock_check_call: - mock_check_call.side_effect = self.mock_postfix - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = self.mock_postfix - return func(*args, **kwargs) - - -class MockPostfix(object): - """A callable to mimic Postfix command line utilities. - - This is best used a side effect to a mock object. All calls to - 'postfix' are noops. For calls to 'postconf', values that are set in - the constructor or through mocked out runs of postconf are - remembered and properly returned if the installer attempts to fetch - the value. If the Postfix installer attempts to obtain a value that - hasn't yet been set, a dummy value is returned. - - :ivar str config_path: path to Postfix main.cf file - - """ - def __init__(self, config_dir, initial_values): - """Create Postfix configuration. - - :param str config_dir: path for Postfix config dir - :param dict initial_values: initial Postfix config values - - """ - initial_values["config_directory"] = config_dir - - self.config_path = os.path.join(config_dir, "main.cf") - self._write_config(initial_values) - - def __call__(self, args, *unused_args, **unused_kwargs): - cmd = os.path.basename(args[0]) - if cmd == "postfix": - return - elif cmd != "postconf": # pragma: no cover - assert False, "Unexpected command '{0}'".format(''.join(args)) - - output = [] - - skip = False - for arg in args[1:]: - if skip: - skip = False - elif arg[0] == "-": - if arg == "-c": - skip = True - elif "=" in arg: - name, _, value = arg.partition("=") - self.set_value(name, value) - else: - output.append("{0} = {1}\n".format(arg, self.get_value(arg))) - - return "\n".join(output) - - def get_value(self, name): - """Returns the value for the Postfix config parameter name. - - If the value isn't set, an empty string is returned. - - :param str name: name of the Postfix config parameter - - :returns: value of the named parameter - :rtype: str - - """ - return self._read_config().get(name, "") - - def set_value(self, name, value): - """Sets the value for a Postfix config parameter. - - :param str name: name of the Postfix config parameter - :param str value: value ot set the parameter to - - """ - config = self._read_config() - config[name] = value - self._write_config(config) - - def _read_config(self): - config = {} - with open(self.config_path) as f: - for line in f: - key, _, value = line.strip().partition(" = ") - config[key] = value - - return config - - def _write_config(self, config): - with open(self.config_path, "w") as f: - f.writelines("{0} = {1}\n".format(key, value) - for key, value in config.items()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/certbot-postfix/certbot_postfix/postconf.py b/certbot-postfix/certbot_postfix/postconf.py index 7738562dd..466e0e63e 100644 --- a/certbot-postfix/certbot_postfix/postconf.py +++ b/certbot-postfix/certbot_postfix/postconf.py @@ -1,62 +1,152 @@ """Classes that wrap the postconf command line utility. - -These classes allow you to interact with a Postfix config like it is a -dictionary, with the getting and setting of values in the config being -handled automatically by the class. - """ -import collections - +import six +from certbot import errors from certbot_postfix import util +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, List, Tuple +# pylint: enable=unused-import, no-name-in-module -class ReadOnlyMainMap(util.PostfixUtilBase, collections.Mapping): - """A read-only view of a Postfix main.cf file.""" +class ConfigMain(util.PostfixUtilBase): + """A parser for Postfix's main.cf file.""" - _modifiers = None - """An iterable containing additional CLI flags for postconf.""" - - def __getitem__(self, name): - return next(_parse_main_output(self._get_output([name])))[1] - - def __iter__(self): - for name, _ in _parse_main_output(self._get_output()): - yield name - - def __len__(self): - return sum(1 for _ in _parse_main_output(self._get_output())) - - def _call(self, extra_args=None): - """Runs Postconf and returns the result. - - If self._modifiers is set, it is provided on the command line to - postconf before any values in extra_args. - - :param list extra_args: additional arguments for the command - - :returns: data written to stdout and stderr - :rtype: `tuple` of `str` - - :raises subprocess.CalledProcessError: if the command fails + def __init__(self, executable, ignore_master_overrides=False, config_dir=None): + super(ConfigMain, self).__init__(executable, config_dir) + # Whether to ignore overrides from master. + self._ignore_master_overrides = ignore_master_overrides + # List of all current Postfix parameters, from `postconf` command. + self._db = {} # type: Dict[str, str] + # List of current master.cf overrides from Postfix config. Dictionary + # of parameter name => list of tuples (service name, paramter value) + # Note: We should never modify master without explicit permission. + self._master_db = {} # type: Dict[str, List[Tuple[str, str]]] + # List of all changes requested to the Postfix parameters as they are now + # in _db. These changes are flushed to `postconf` on `flush`. + self._updated = {} # type: Dict[str, str] + self._read_from_conf() + def _read_from_conf(self): + """Reads initial parameter state from `main.cf` into this object. """ - all_extra_args = [] - for args_list in (self._modifiers, extra_args,): - if args_list is not None: - all_extra_args.extend(args_list) + out = self._get_output() + for name, value in _parse_main_output(out): + self._db[name] = value + out = self._get_output_master() + for name, value in _parse_main_output(out): + service, param_name = name.rsplit("/", 1) + if param_name not in self._master_db: + self._master_db[param_name] = [] + self._master_db[param_name].append((service, value)) - return super(ReadOnlyMainMap, self)._call(all_extra_args) + def _get_output_master(self): + """Retrieves output for `master.cf` parameters.""" + return self._get_output('-P') + def get_default(self, name): + """Retrieves default value of parameter `name` from postfix parameters. + + :param str name: The name of the parameter to fetch. + :returns: The default value of parameter `name`. + :rtype: str + """ + out = self._get_output(['-d', name]) + _, value = next(_parse_main_output(out), (None, None)) + return value + + def get(self, name): + """Retrieves working value of parameter `name` from postfix parameters. + + :param str name: The name of the parameter to fetch. + :returns: The value of parameter `name`. + :rtype: str + """ + if name in self._updated: + return self._updated[name] + return self._db[name] + + def get_master_overrides(self, name): + """Retrieves list of overrides for parameter `name` in postfix's Master config + file. + + :returns: List of tuples (service, value), meaning that parameter `name` + is overridden as `value` for `service`. + :rtype: `list` of `tuple` of `str` + """ + if name in self._master_db: + return self._master_db[name] + return None + + def set(self, name, value, acceptable_overrides=None): + """Sets parameter `name` to `value`. If `name` is overridden by a particular service in + `master.cf`, reports any of these parameter conflicts as long as + `ignore_master_overrides` was not set. + + .. note:: that this function does not flush these parameter values to main.cf; + To do that, use `flush`. + + :param str name: The name of the parameter to set. + :param str value: The value of the parameter. + :param tuple acceptable_overrides: If the master configuration file overrides `value` + with a value in acceptable_overrides. + """ + if name not in self._db: + raise KeyError("Parameter name %s is not a valid Postfix parameter name.", name) + # Check to see if this parameter is overridden by master. + overrides = self.get_master_overrides(name) + if not self._ignore_master_overrides and overrides is not None: + util.report_master_overrides(name, overrides, acceptable_overrides) + if value != self._db[name]: + # _db contains the "original" state of parameters. We only care about + # writes if they cause a delta from the original state. + self._updated[name] = value + elif name in self._updated: + # If this write reverts a previously updated parameter back to the + # original DB's state, we don't have to keep track of it in _updated. + del self._updated[name] + + def flush(self): + """Flushes all parameter changes made using `self.set`, to `main.cf` + + :raises error.PluginError: When flush to main.cf fails for some reason. + """ + if len(self._updated) == 0: + return + args = ['-e'] + for name, value in six.iteritems(self._updated): + args.append('{0}={1}'.format(name, value)) + try: + self._get_output(args) + except IOError as e: + raise errors.PluginError("Unable to save to Postfix config: %v", e) + for name, value in six.iteritems(self._updated): + self._db[name] = value + self._updated = {} + + def get_changes(self): + """ Return queued changes to main.cf. + + :rtype: dict[str, str] + """ + return self._updated def _parse_main_output(output): """Parses the raw output from Postconf about main.cf. + Expects the output to look like: + + .. code-block:: none + + name1 = value1 + name2 = value2 + :param str output: data postconf wrote to stdout about main.cf :returns: generator providing key-value pairs from main.cf - :rtype: generator - + :rtype: Iterator[tuple(str, str)] """ for line in output.splitlines(): name, _, value = line.partition(" =") yield name, value.strip() + + diff --git a/certbot-postfix/certbot_postfix/tests/__init__.py b/certbot-postfix/certbot_postfix/tests/__init__.py new file mode 100644 index 000000000..7316b5888 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/__init__.py @@ -0,0 +1 @@ +""" Certbot Postfix Tests """ diff --git a/certbot-postfix/certbot_postfix/tests/installer_test.py b/certbot-postfix/certbot_postfix/tests/installer_test.py new file mode 100644 index 000000000..1bdd2c8b3 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/installer_test.py @@ -0,0 +1,314 @@ +"""Tests for certbot_postfix.installer.""" +from contextlib import contextmanager +import copy +import functools +import os +import pkg_resources +import six +import unittest + +import mock + +from certbot import errors +from certbot.tests import util as certbot_test_util + +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Tuple, Union +# pylint: enable=unused-import, no-name-in-module + +DEFAULT_MAIN_CF = { + "smtpd_tls_cert_file": "", + "smtpd_tls_key_file": "", + "smtpd_tls_dh1024_param_file": "", + "smtpd_tls_security_level": "none", + "smtpd_tls_auth_only": "", + "smtpd_tls_mandatory_protocols": "", + "smtpd_tls_protocols": "", + "smtpd_tls_ciphers": "", + "smtpd_tls_exclude_ciphers": "", + "smtpd_tls_mandatory_ciphers": "", + "smtpd_tls_eecdh_grade": "medium", + "smtp_tls_security_level": "", + "smtp_tls_ciphers": "", + "smtp_tls_exclude_ciphers": "", + "smtp_tls_mandatory_ciphers": "", + "mail_version": "3.2.3" +} + +def _main_cf_with(obj): + main_cf = copy.copy(DEFAULT_MAIN_CF) + main_cf.update(obj) + return main_cf + +class InstallerTest(certbot_test_util.ConfigTestCase): + # pylint: disable=too-many-public-methods + + def setUp(self): + super(InstallerTest, self).setUp() + _config_file = pkg_resources.resource_filename("certbot_postfix.tests", + os.path.join("testdata", "config.json")) + self.config.postfix_ctl = "postfix" + self.config.postfix_config_dir = self.tempdir + self.config.postfix_config_utility = "postconf" + self.config.postfix_tls_only = False + self.config.postfix_server_only = False + self.config.config_dir = self.tempdir + + @mock.patch("certbot_postfix.installer.util.is_acceptable_value") + def test_set_vars(self, mock_is_acceptable_value): + mock_is_acceptable_value.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + mock_is_acceptable_value.return_value = False + + @mock.patch("certbot_postfix.installer.util.is_acceptable_value") + def test_acceptable_value(self, mock_is_acceptable_value): + mock_is_acceptable_value.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + mock_is_acceptable_value.return_value = False + + @certbot_test_util.patch_get_utility() + def test_confirm_changes_no_raises_error(self, mock_util): + mock_util().yesno.return_value = False + with create_installer(self.config) as installer: + installer.prepare() + self.assertRaises(errors.PluginError, installer.deploy_cert, + "example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + + @certbot_test_util.patch_get_utility() + def test_save(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.postconf.flush = mock.Mock() + installer.reverter = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.save() + self.assertEqual(installer.save_notes, []) + self.assertEqual(installer.postconf.flush.call_count, 1) + self.assertEqual(installer.reverter.add_to_checkpoint.call_count, 1) + + @certbot_test_util.patch_get_utility() + def test_save_with_title(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.postconf.flush = mock.Mock() + installer.reverter = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.save(title="new_file!") + self.assertEqual(installer.reverter.finalize_checkpoint.call_count, 1) + + @certbot_test_util.patch_get_utility() + def test_rollback_checkpoints_resets_postconf(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.rollback_checkpoints() + self.assertEqual(installer.postconf.get_changes(), {}) + + @certbot_test_util.patch_get_utility() + def test_recovery_routine_resets_postconf(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.recovery_routine() + self.assertEqual(installer.postconf.get_changes(), {}) + + def test_restart(self): + with create_installer(self.config) as installer: + installer.prepare() + installer.restart() + self.assertEqual(installer.postfix.restart.call_count, 1) + + def test_add_parser_arguments(self): + options = set(("ctl", "config-dir", "config-utility", + "tls-only", "server-only", "ignore-master-overrides")) + mock_add = mock.MagicMock() + + from certbot_postfix import installer + installer.Installer.add_parser_arguments(mock_add) + + for call in mock_add.call_args_list: + self.assertTrue(call[0][0] in options) + + def test_no_postconf_prepare(self): + with create_installer(self.config) as installer: + installer_path = "certbot_postfix.installer" + exe_exists_path = installer_path + ".certbot_util.exe_exists" + path_surgery_path = "certbot_postfix.util.plugins_util.path_surgery" + with mock.patch(path_surgery_path, return_value=False): + with mock.patch(exe_exists_path, return_value=False): + self.assertRaises(errors.NoInstallationError, + installer.prepare) + + def test_old_version(self): + with create_installer(self.config, main_cf=_main_cf_with({"mail_version": "0.0.1"}))\ + as installer: + self.assertRaises(errors.NotSupportedError, installer.prepare) + + def test_lock_error(self): + with create_installer(self.config) as installer: + assert_raises = functools.partial(self.assertRaises, + errors.PluginError, + installer.prepare) + certbot_test_util.lock_and_call(assert_raises, self.tempdir) + + + @mock.patch('certbot.util.lock_dir_until_exit') + def test_dir_locked(self, lock_dir): + with create_installer(self.config) as installer: + lock_dir.side_effect = errors.LockError + self.assertRaises(errors.PluginError, installer.prepare) + + def test_more_info(self): + with create_installer(self.config) as installer: + installer.prepare() + output = installer.more_info() + self.assertTrue("Postfix" in output) + self.assertTrue(self.tempdir in output) + self.assertTrue(DEFAULT_MAIN_CF["mail_version"] in output) + + def test_get_all_names(self): + config = {"mydomain": "example.org", + "myhostname": "mail.example.org", + "myorigin": "example.org"} + with create_installer(self.config, main_cf=_main_cf_with(config)) as installer: + installer.prepare() + result = installer.get_all_names() + self.assertEqual(result, set(config.values())) + + @certbot_test_util.patch_get_utility() + def test_deploy(self, mock_util): + mock_util().yesno.return_value = True + from certbot_postfix import constants + with create_installer(self.config) as installer: + installer.prepare() + + # pylint: disable=protected-access + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + changes = installer.postconf.get_changes() + expected = {} # type: Dict[str, Tuple[str, ...]] + expected.update(constants.TLS_SERVER_VARS) + expected.update(constants.DEFAULT_SERVER_VARS) + expected.update(constants.DEFAULT_CLIENT_VARS) + self.assertEqual(changes["smtpd_tls_key_file"], "key_path") + self.assertEqual(changes["smtpd_tls_cert_file"], "cert_path") + for name, value in six.iteritems(expected): + self.assertEqual(changes[name], value[0]) + + @certbot_test_util.patch_get_utility() + def test_tls_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: x == "tls_only" + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 4) + + @certbot_test_util.patch_get_utility() + def test_server_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: x == "server_only" + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 11) + + @certbot_test_util.patch_get_utility() + def test_tls_and_server_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: True + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 3) + + @certbot_test_util.patch_get_utility() + def test_deploy_twice(self, mock_util): + # Deploying twice on the same installer shouldn't do anything! + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + from certbot_postfix.postconf import ConfigMain + with mock.patch.object(ConfigMain, "set", wraps=installer.postconf.set) as fake_set: + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(fake_set.call_count, 15) + fake_set.reset_mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + fake_set.assert_not_called() + + @certbot_test_util.patch_get_utility() + def test_deploy_already_secure(self, mock_util): + # Should not overwrite "more-secure" parameters + mock_util().yesno.return_value = True + more_secure = { + "smtpd_tls_security_level": "encrypt", + "smtpd_tls_protocols": "!SSLv3, !SSLv2, !TLSv1", + "smtpd_tls_eecdh_grade": "strong" + } + with create_installer(self.config,\ + main_cf=_main_cf_with(more_secure)) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + for param in more_secure.keys(): + self.assertFalse(param in installer.postconf.get_changes()) + + def test_enhance(self): + with create_installer(self.config) as installer: + installer.prepare() + self.assertRaises(errors.PluginError, + installer.enhance, + "example.org", "redirect") + + def test_supported_enhancements(self): + with create_installer(self.config) as installer: + installer.prepare() + self.assertEqual(installer.supported_enhancements(), []) + +@contextmanager +def create_installer(config, main_cf=DEFAULT_MAIN_CF): +# pylint: disable=dangerous-default-value + """Creates a Postfix installer with calls to `postconf` and `postfix` mocked out. + + In particular, creates a ConfigMain object that does regular things, but seeds it + with values from `main_cf` and `master_cf` dicts. + """ + from certbot_postfix.postconf import ConfigMain + from certbot_postfix import installer + def _mock_init_postconf(postconf, executable, ignore_master_overrides=False, config_dir=None): + # pylint: disable=protected-access,unused-argument + postconf._ignore_master_overrides = ignore_master_overrides + postconf._db = main_cf + postconf._master_db = {} + postconf._updated = {} + # override get_default to get from main + postconf.get_default = lambda name: main_cf[name] + with mock.patch.object(ConfigMain, "__init__", _mock_init_postconf): + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(exe_exists_path, return_value=True): + with mock.patch("certbot_postfix.installer.util.PostfixUtil", + return_value=mock.Mock()): + yield installer.Installer(config, "postfix") + +if __name__ == "__main__": + unittest.main() # pragma: no cover + diff --git a/certbot-postfix/certbot_postfix/tests/postconf_test.py b/certbot-postfix/certbot_postfix/tests/postconf_test.py new file mode 100644 index 000000000..91617d410 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/postconf_test.py @@ -0,0 +1,107 @@ +"""Tests for certbot_postfix.postconf.""" + +import mock +import unittest + +from certbot import errors + +class PostConfTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostConf.""" + def setUp(self): + from certbot_postfix.postconf import ConfigMain + super(PostConfTest, self).setUp() + with mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') as mock_call: + with mock.patch('certbot_postfix.postconf.ConfigMain._get_output_master') as \ + mock_master_call: + with mock.patch('certbot_postfix.postconf.util.verify_exe_exists') as verify_exe: + verify_exe.return_value = True + mock_call.return_value = ('default_parameter = value\n' + 'extra_param =\n' + 'overridden_by_master = default\n') + mock_master_call.return_value = ( + 'service/type/overridden_by_master = master_value\n' + 'service2/type/overridden_by_master = master_value2\n' + ) + self.config = ConfigMain('postconf', False) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + @mock.patch('certbot_postfix.postconf.util.verify_exe_exists') + def test_get_output_master(self, mock_verify_exe, mock_get_output): + from certbot_postfix.postconf import ConfigMain + mock_verify_exe.return_value = True + ConfigMain('postconf', lambda x, y, z: None) + mock_get_output.assert_called_with('-P') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_read_default(self, mock_get_output): + mock_get_output.return_value = 'param = default_value' + self.assertEqual(self.config.get_default('param'), 'default_value') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_set(self, mock_call): + self.config.set('extra_param', 'other_value') + self.assertEqual(self.config.get('extra_param'), 'other_value') + self.config.flush() + mock_call.assert_called_with(['-e', 'extra_param=other_value']) + + def test_set_bad_param_name(self): + self.assertRaises(KeyError, self.config.set, 'nonexistent_param', 'some_value') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_write_revert(self, mock_call): + self.config.set('default_parameter', 'fake_news') + # revert config set + self.config.set('default_parameter', 'value') + self.config.flush() + mock_call.assert_not_called() + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_write_default(self, mock_call): + self.config.set('default_parameter', 'value') + self.config.flush() + mock_call.assert_not_called() + + def test_master_overrides(self): + self.assertEqual(self.config.get_master_overrides('overridden_by_master'), + [('service/type', 'master_value'), + ('service2/type', 'master_value2')]) + + def test_set_check_override(self): + self.assertRaises(errors.PluginError, self.config.set, + 'overridden_by_master', 'new_value') + + def test_ignore_check_override(self): + # pylint: disable=protected-access + self.config._ignore_master_overrides = True + self.config.set('overridden_by_master', 'new_value') + + def test_check_acceptable_overrides(self): + self.config.set('overridden_by_master', 'new_value', + ('master_value', 'master_value2')) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush(self, mock_out): + self.config.set('default_parameter', 'new_value') + self.config.set('extra_param', 'another_value') + self.config.flush() + arguments = mock_out.call_args_list[-1][0][0] + self.assertEquals('-e', arguments[0]) + self.assertTrue('default_parameter=new_value' in arguments) + self.assertTrue('extra_param=another_value' in arguments) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush_updates_object(self, mock_out): + self.config.set('default_parameter', 'new_value') + self.config.flush() + mock_out.reset_mock() + self.config.set('default_parameter', 'new_value') + mock_out.assert_not_called() + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush_throws_error_on_fail(self, mock_out): + mock_out.side_effect = [IOError("oh no!")] + self.config.set('default_parameter', 'new_value') + self.assertRaises(errors.PluginError, self.config.flush) + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/certbot-postfix/certbot_postfix/tests/util_test.py b/certbot-postfix/certbot_postfix/tests/util_test.py new file mode 100644 index 000000000..fa38f83ab --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/util_test.py @@ -0,0 +1,205 @@ +"""Tests for certbot_postfix.util.""" + +import subprocess +import unittest + +import mock + +from certbot import errors + + +class PostfixUtilBaseTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostfixUtilBase.""" + + @classmethod + def _create_object(cls, *args, **kwargs): + from certbot_postfix.util import PostfixUtilBase + return PostfixUtilBase(*args, **kwargs) + + @mock.patch('certbot_postfix.util.verify_exe_exists') + def test_no_exe(self, mock_verify): + expected_error = errors.NoInstallationError + mock_verify.side_effect = expected_error + self.assertRaises(expected_error, self._create_object, 'nonexistent') + + def test_object_creation(self): + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self._create_object('existent') + + @mock.patch('certbot_postfix.util.check_all_output') + def test_call_extends_args(self, mock_output): + # pylint: disable=protected-access + with mock.patch('certbot_postfix.util.verify_exe_exists'): + mock_output.return_value = 'expected' + postfix = self._create_object('executable') + postfix._call(['many', 'extra', 'args']) + mock_output.assert_called_with(['executable', 'many', 'extra', 'args']) + postfix._call() + mock_output.assert_called_with(['executable']) + + def test_create_with_config(self): + # pylint: disable=protected-access + with mock.patch('certbot_postfix.util.verify_exe_exists'): + postfix = self._create_object('exec', 'config_dir') + self.assertEqual(postfix._base_command, ['exec', '-c', 'config_dir']) + +class PostfixUtilTest(unittest.TestCase): + def setUp(self): + # pylint: disable=protected-access + from certbot_postfix.util import PostfixUtil + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self.postfix = PostfixUtil() + self.postfix._call = mock.Mock() + self.mock_call = self.postfix._call + + def test_test(self): + self.postfix.test() + self.mock_call.assert_called_with(['check']) + + def test_test_raises_error_when_check_fails(self): + self.mock_call.side_effect = [subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.MisconfigurationError, self.postfix.test) + self.mock_call.assert_called_with(['check']) + + def test_restart_while_running(self): + self.mock_call.side_effect = [subprocess.CalledProcessError(1, ""), None] + self.postfix.restart() + self.mock_call.assert_called_with(['start']) + + def test_restart_while_not_running(self): + self.postfix.restart() + self.mock_call.assert_called_with(['reload']) + + def test_restart_raises_error_when_reload_fails(self): + self.mock_call.side_effect = [None, subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.PluginError, self.postfix.restart) + self.mock_call.assert_called_with(['reload']) + + def test_restart_raises_error_when_start_fails(self): + self.mock_call.side_effect = [ + subprocess.CalledProcessError(1, ""), + subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.PluginError, self.postfix.restart) + self.mock_call.assert_called_with(['start']) + +class CheckAllOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_all_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_all_output + return check_all_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + err = 'baz' + + mock_popen().communicate.return_value = (output, err) + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + log_args = mock_logger.debug.call_args[0] + for value in (command, retcode, output, err,): + self.assertTrue(value in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + expected = ('bar', '') + mock_popen().communicate.return_value = expected + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), expected) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_stderr_error(self): + self.assertRaises(ValueError, self._call, stderr=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +class VerifyExeExistsTest(unittest.TestCase): + """Tests for certbot_postfix.util.verify_exe_exists.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import verify_exe_exists + return verify_exe_exists(*args, **kwargs) + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_failure(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = mock_path_surgery.return_value = False + self.assertRaises(errors.NoInstallationError, self._call, 'foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + def test_simple_success(self, mock_exe_exists): + mock_exe_exists.return_value = True + self._call('foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = False + mock_path_surgery.return_value = True + self._call('foo') + +class TestUtils(unittest.TestCase): + """ Testing random utility functions in util.py + """ + def test_report_master_overrides(self): + from certbot_postfix.util import report_master_overrides + self.assertRaises(errors.PluginError, report_master_overrides, 'name', + [('service/type', 'value')]) + # Shouldn't raise error + report_master_overrides('name', [('service/type', 'value')], + acceptable_overrides=('value',)) + + def test_no_acceptable_value(self): + from certbot_postfix.util import is_acceptable_value + self.assertFalse(is_acceptable_value('name', 'value', None)) + + def test_is_acceptable_value(self): + from certbot_postfix.util import is_acceptable_value + self.assertTrue(is_acceptable_value('name', 'value', ('value',))) + self.assertFalse(is_acceptable_value('name', 'bad', ('value',))) + + def test_is_acceptable_tuples(self): + from certbot_postfix.util import is_acceptable_value + self.assertTrue(is_acceptable_value('name', 'value', ('value', 'value1'))) + self.assertFalse(is_acceptable_value('name', 'bad', ('value', 'value1'))) + + def test_is_acceptable_protocols(self): + from certbot_postfix.util import is_acceptable_value + # SSLv2 and SSLv3 are both not supported, unambiguously + self.assertFalse(is_acceptable_value('tls_mandatory_protocols_lol', + 'SSLv2, SSLv3', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'SSLv2, SSLv3', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + '!SSLv2, !TLSv1', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + '!SSLv2, SSLv3, !SSLv3, ', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + '!SSLv2, !SSLv3', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + '!SSLv3, !TLSv1, !SSLv2', None)) + # TLSv1.2 is supported unambiguously + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'TLSv1, TLSv1.1,', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'TLSv1.2, !TLSv1.2,', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + 'TLSv1.2, ', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + 'TLSv1, TLSv1.1, TLSv1.2', None)) + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index 57196017f..f06989903 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -1,14 +1,16 @@ """Utility functions for use in the Postfix installer.""" import logging +import re import subprocess from certbot import errors from certbot import util as certbot_util from certbot.plugins import util as plugins_util - +from certbot_postfix import constants logger = logging.getLogger(__name__) +COMMAND = "postfix" class PostfixUtilBase(object): """A base class for wrapping Postfix command line utilities.""" @@ -22,9 +24,13 @@ class PostfixUtilBase(object): :raises .NoInstallationError: when the executable isn't found """ + self.executable = executable verify_exe_exists(executable) + self._set_base_command(config_dir) + self.config_dir = None - self._base_command = [executable] + def _set_base_command(self, config_dir): + self._base_command = [self.executable] if config_dir is not None: self._base_command.extend(('-c', config_dir,)) @@ -59,6 +65,77 @@ class PostfixUtilBase(object): """ return self._call(extra_args)[0] +class PostfixUtil(PostfixUtilBase): + """Wrapper around Postfix CLI tool. + """ + + def __init__(self, config_dir=None): + super(PostfixUtil, self).__init__(COMMAND, config_dir) + + def test(self): + """Make sure the configuration is valid. + + :raises .MisconfigurationError: if the config is invalid + """ + try: + self._call(["check"]) + except subprocess.CalledProcessError as e: + logger.debug("Could not check postfix configuration:\n%s", e) + raise errors.MisconfigurationError( + "Postfix failed internal configuration check.") + + def restart(self): + """Restart or refresh the server content. + + :raises .PluginError: when server cannot be restarted + + """ + logger.info("Reloading Postfix configuration...") + if self._is_running(): + self._reload() + else: + self._start() + + + def _is_running(self): + """Is Postfix currently running? + + Uses the 'postfix status' command to determine if Postfix is + currently running using the specified configuration files. + + :returns: True if Postfix is running, otherwise, False + :rtype: bool + + """ + try: + self._call(["status"]) + except subprocess.CalledProcessError: + return False + return True + + def _start(self): + """Instructions Postfix to start running. + + :raises .PluginError: when Postfix cannot start + + """ + try: + self._call(["start"]) + except subprocess.CalledProcessError: + raise errors.PluginError("Postfix failed to start") + + def _reload(self): + """Instructs Postfix to reload its configuration. + + If Postfix isn't currently running, this method will fail. + + :raises .PluginError: when Postfix cannot reload + """ + try: + self._call(["reload"]) + except subprocess.CalledProcessError: + raise errors.PluginError( + "Postfix failed to reload its configuration") def check_all_output(*args, **kwargs): """A version of subprocess.check_output that also captures stderr. @@ -107,7 +184,7 @@ def check_all_output(*args, **kwargs): return (output, err) -def verify_exe_exists(exe): +def verify_exe_exists(exe, message=None): """Ensures an executable with the given name is available. If an executable isn't found for the given path or name, extra @@ -115,83 +192,101 @@ def verify_exe_exists(exe): utilities that may not be available in the default cron PATH. :param str exe: executable path or name + :param str message: Error message to print. :raises .NoInstallationError: when the executable isn't found """ + if message is None: + message = "Cannot find executable '{0}'.".format(exe) if not (certbot_util.exe_exists(exe) or plugins_util.path_surgery(exe)): - raise errors.NoInstallationError( - "Cannot find executable '{0}'.".format(exe)) + raise errors.NoInstallationError(message) +def report_master_overrides(name, overrides, acceptable_overrides=None): + """If the value for a parameter `name` is overridden by other services, + report a warning to notify the user. If `parameter` is a TLS version parameter + (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then + `acceptable_overrides` isn't used each value in overrides is inspected for secure TLS + versions. -def check_call(*args, **kwargs): - """A simple wrapper of subprocess.check_call that logs errors. - - :param tuple args: positional arguments to subprocess.check_call - :param dict kargs: keyword arguments to subprocess.check_call - - :raises subprocess.CalledProcessError: if the call fails - + :param str name: The name of the parameter that is being overridden. + :param list overrides: The values that other services are setting for `name`. + Each override is a tuple: (service name, value) + :param tuple acceptable_overrides: Override values that are acceptable. For instance, if + another service is overriding our parameter with a more secure option, we don't have + to warn. If this is set to None, errors are raised for *any* overrides of `name`! """ - try: - subprocess.check_call(*args, **kwargs) - except subprocess.CalledProcessError: - cmd = _get_cmd(*args, **kwargs) - logger.debug("%s exited with a non-zero status.", - "".join(cmd), exc_info=True) - raise + error_string = "" + for override in overrides: + service, value = override + # If this override is acceptable: + if acceptable_overrides is not None and \ + is_acceptable_value(name, value, acceptable_overrides): + continue + error_string += " {0}: {1}\n".format(service, value) + if error_string: + raise errors.PluginError("{0} is overridden with less secure options by the " + "following services in master.cf:\n".format(name) + error_string) -def check_output(*args, **kwargs): - """Backported version of subprocess.check_output for Python 2.6+. - - This is the same as subprocess.check_output from newer versions of - Python, except: - - 1. The return value is a string rather than a byte string. To - accomplish this, the caller cannot set the parameter - universal_newlines. - 2. If the command exits with a nonzero status, output is not - included in the raised subprocess.CalledProcessError because - subprocess.CalledProcessError on Python 2.6 does not support this. - Instead, the failure including the output is logged. - - :param tuple args: positional arguments for Popen - :param dict kwargs: keyword arguments for Popen - - :returns: data printed to stdout - :rtype: str - - :raises ValueError: if arguments are invalid - :raises subprocess.CalledProcessError: if the command fails +def is_acceptable_value(parameter, value, acceptable=None): + """ Returns whether the `value` for this `parameter` is acceptable, + given a tuple of `acceptable` values. If `parameter` is a TLS version parameter + (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then + `acceptable` isn't used and `value` is inspected for secure TLS versions. + :param str parameter: The name of the parameter being set. + :param str value: Proposed new value for parameter. + :param tuple acceptable: List of acceptable values for parameter. """ - for keyword in ('stdout', 'universal_newlines',): - if keyword in kwargs: - raise ValueError( - keyword + ' argument not allowed, it will be overridden.') - - kwargs['stdout'] = subprocess.PIPE - kwargs['universal_newlines'] = True - - process = subprocess.Popen(*args, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = _get_cmd(*args, **kwargs) - logger.debug( - "'%s' exited with %d. Output was:\n%s", - cmd, retcode, output, exc_info=True) - raise subprocess.CalledProcessError(retcode, cmd) - return output + # Check if param value is a comma-separated list of protocols. + # Otherwise, just check whether the value is in the acceptable list. + if 'tls_protocols' in parameter or 'tls_mandatory_protocols' in parameter: + return _has_acceptable_tls_versions(value) + if acceptable is not None: + return value in acceptable + return False -def _get_cmd(*args, **kwargs): - """Return the command from Popen args. - - :param tuple args: Popen args - :param dict kwargs: Popen kwargs - +def _has_acceptable_tls_versions(parameter_string): """ - cmd = kwargs.get('args') - return args[0] if cmd is None else cmd + Checks to see if the list of TLS protocols is acceptable. + This requires that TLSv1.2 is supported, and neither SSLv2 nor SSLv3 are supported. + + Should be a string of protocol names delimited by commas, spaces, or colons. + + Postfix's documents suggest listing protocols to exclude, like "!SSLv2, !SSLv3". + Listing the protocols to include, like "TLSv1, TLSv1.1, TLSv1.2" is okay as well, + though not recommended + + When these two modes are interspersed, the presence of a single non-negated protocol name + (i.e. "TLSv1" rather than "!TLSv1") automatically excludes all other unnamed protocols. + + In addition, the presence of both a protocol name inclusion and exclusion isn't explicitly + documented, so this method should return False if it encounters contradicting statements + about TLSv1.2, SSLv2, or SSLv3. (for instance, "SSLv3, !SSLv3"). + """ + if not parameter_string: + return False + bad_versions = list(constants.TLS_VERSIONS) + for version in constants.ACCEPTABLE_TLS_VERSIONS: + del bad_versions[bad_versions.index(version)] + supported_version_list = re.split("[, :]+", parameter_string) + # The presence of any non-"!" protocol listing excludes the others by default. + inclusion_list = False + for version in supported_version_list: + if not version: + continue + if version in bad_versions: # short-circuit if we recognize any bad version + return False + if version[0] != "!": + inclusion_list = True + if inclusion_list: # For any inclusion list, we still require TLS 1.2. + if "TLSv1.2" not in supported_version_list or "!TLSv1.2" in supported_version_list: + return False + else: + for bad_version in bad_versions: + if "!" + bad_version not in supported_version_list: + return False + return True + diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py deleted file mode 100644 index 4a014ca9b..000000000 --- a/certbot-postfix/certbot_postfix/util_test.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Tests for certbot_postfix.util.""" - -import subprocess -import unittest - -import mock - -from certbot import errors - - -class PostfixUtilBaseTest(unittest.TestCase): - """Tests for certbot_postfix.util.PostfixUtilBase.""" - - @classmethod - def _create_object(cls, *args, **kwargs): - from certbot_postfix.util import PostfixUtilBase - return PostfixUtilBase(*args, **kwargs) - - @mock.patch('certbot_postfix.util.verify_exe_exists') - def test_no_exe(self, mock_verify): - expected_error = errors.NoInstallationError - mock_verify.side_effect = expected_error - self.assertRaises(expected_error, self._create_object, 'nonexistent') - - def test_object_creation(self): - with mock.patch('certbot_postfix.util.verify_exe_exists'): - self._create_object('existent') - - -class CheckAllOutputTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_all_output.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_all_output - return check_all_output(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_command_error(self, mock_popen, mock_logger): - command = 'foo' - retcode = 42 - output = 'bar' - err = 'baz' - - mock_popen().communicate.return_value = (output, err) - mock_popen().poll.return_value = 42 - - self.assertRaises(subprocess.CalledProcessError, self._call, command) - log_args = mock_logger.debug.call_args[0] - for value in (command, retcode, output, err,): - self.assertTrue(value in log_args) - - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_success(self, mock_popen): - command = 'foo' - expected = ('bar', '') - mock_popen().communicate.return_value = expected - mock_popen().poll.return_value = 0 - - self.assertEqual(self._call(command), expected) - - def test_stdout_error(self): - self.assertRaises(ValueError, self._call, stdout=None) - - def test_stderr_error(self): - self.assertRaises(ValueError, self._call, stderr=None) - - def test_universal_newlines_error(self): - self.assertRaises(ValueError, self._call, universal_newlines=False) - - -class VerifyExeExistsTest(unittest.TestCase): - """Tests for certbot_postfix.util.verify_exe_exists.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import verify_exe_exists - return verify_exe_exists(*args, **kwargs) - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - @mock.patch('certbot_postfix.util.plugins_util.path_surgery') - def test_failure(self, mock_exe_exists, mock_path_surgery): - mock_exe_exists.return_value = mock_path_surgery.return_value = False - self.assertRaises(errors.NoInstallationError, self._call, 'foo') - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - def test_simple_success(self, mock_exe_exists): - mock_exe_exists.return_value = True - self._call('foo') - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - @mock.patch('certbot_postfix.util.plugins_util.path_surgery') - def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): - mock_exe_exists.return_value = False - mock_path_surgery.return_value = True - self._call('foo') - - -class CheckCallTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_call.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_call - return check_call(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.check_call') - def test_failure(self, mock_check_call, mock_logger): - cmd = "postconf smtpd_use_tls=yes".split() - mock_check_call.side_effect = subprocess.CalledProcessError(42, cmd) - self.assertRaises(subprocess.CalledProcessError, self._call, cmd) - self.assertTrue(mock_logger.method_calls) - - @mock.patch('certbot_postfix.util.subprocess.check_call') - def test_success(self, mock_check_call): - cmd = "postconf smtpd_use_tls=yes".split() - self._call(cmd) - mock_check_call.assert_called_once_with(cmd) - - -class CheckOutputTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_output.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_output - return check_output(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_command_error(self, mock_popen, mock_logger): - command = 'foo' - retcode = 42 - output = 'bar' - - mock_popen().communicate.return_value = (output, '') - mock_popen().poll.return_value = 42 - - self.assertRaises(subprocess.CalledProcessError, self._call, command) - - log_args = mock_logger.debug.call_args[0] - self.assertTrue(command in log_args) - self.assertTrue(retcode in log_args) - self.assertTrue(output in log_args) - - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_success(self, mock_popen): - command = 'foo' - output = 'bar' - mock_popen().communicate.return_value = (output, '') - mock_popen().poll.return_value = 0 - - self.assertEqual(self._call(command), output) - - def test_stdout_error(self): - self.assertRaises(ValueError, self._call, stdout=None) - - def test_universal_newlines_error(self): - self.assertRaises(ValueError, self._call, universal_newlines=False) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/certbot-postfix/docs/.gitignore b/certbot-postfix/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-postfix/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-postfix/docs/Makefile b/certbot-postfix/docs/Makefile new file mode 100644 index 000000000..717ff654f --- /dev/null +++ b/certbot-postfix/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-postfix +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-postfix/docs/api.rst b/certbot-postfix/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-postfix/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-postfix/docs/api/installer.rst b/certbot-postfix/docs/api/installer.rst new file mode 100644 index 000000000..121d58d5b --- /dev/null +++ b/certbot-postfix/docs/api/installer.rst @@ -0,0 +1,5 @@ +:mod:`certbot_postfix.installer` +-------------------------------------- + +.. automodule:: certbot_postfix.installer + :members: diff --git a/certbot-postfix/docs/api/postconf.rst b/certbot-postfix/docs/api/postconf.rst new file mode 100644 index 000000000..917150e45 --- /dev/null +++ b/certbot-postfix/docs/api/postconf.rst @@ -0,0 +1,5 @@ +:mod:`certbot_postfix.postconf` +------------------------------- + +.. automodule:: certbot_postfix.postconf + :members: diff --git a/certbot-postfix/docs/conf.py b/certbot-postfix/docs/conf.py new file mode 100644 index 000000000..51d99aab5 --- /dev/null +++ b/certbot-postfix/docs/conf.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'certbot-postfix' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The short X.Y version +version = u'0' +# The full version, including alpha/beta/rc tags +release = u'0' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', +] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = u'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-postfixdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-postfix.tex', u'certbot-postfix Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-postfix', u'certbot-postfix Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-postfix', u'certbot-postfix Documentation', + author, 'certbot-postfix', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/certbot-postfix/docs/index.rst b/certbot-postfix/docs/index.rst new file mode 100644 index 000000000..3d6697bcb --- /dev/null +++ b/certbot-postfix/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-postfix documentation master file, created by + sphinx-quickstart on Wed May 2 16:01:06 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-postfix's documentation! +=========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: certbot_postfix + :members: + +.. toctree:: + :maxdepth: 1 + + api + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-postfix/docs/make.bat b/certbot-postfix/docs/make.bat new file mode 100644 index 000000000..23fbdc93c --- /dev/null +++ b/certbot-postfix/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-postfix + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-postfix/local-oldest-requirements.txt b/certbot-postfix/local-oldest-requirements.txt new file mode 100644 index 000000000..bc0cdbf00 --- /dev/null +++ b/certbot-postfix/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.25.0 +certbot[dev]==0.23.0 diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 2fdcd46aa..cde1ac7c2 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -1,21 +1,23 @@ -import sys - from setuptools import setup from setuptools import find_packages -version = '0.18.0.dev0' +version = '0.26.0.dev0' install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'acme>=0.25.0', + 'certbot>0.23.0', + 'setuptools', 'six', + 'zope.component', 'zope.interface', ] +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + setup( name='certbot-postfix', version=version, @@ -24,6 +26,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -32,10 +35,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -50,6 +51,9 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, entry_points={ 'certbot.plugins': [ 'postfix = certbot_postfix:Installer', diff --git a/certbot/constants.py b/certbot/constants.py index dfdfcc0e8..cd77fb72d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,7 +64,7 @@ RENEWER_DEFAULTS = dict( """Defaults for renewer script.""" -ENHANCEMENTS = ["redirect", "http-header", "ocsp-stapling", "spdy"] +ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling", "spdy", "starttls-policy"] """List of possible :class:`certbot.interfaces.IInstaller` enhancements. diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..01b6ad5e9 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -39,6 +39,7 @@ class PluginEntryPoint(object): "certbot-dns-rfc2136", "certbot-dns-route53", "certbot-nginx", + "certbot-postfix", ] """Distributions for which prefix will be omitted.""" From 3877af6619f690e64fa750f83e12c7a744336a25 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 21 Jun 2018 17:27:19 +0300 Subject: [PATCH 544/631] Gradually increasing HSTS max-age (#5912) This PR adds the functionality to enhance Apache configuration to include HTTP Strict Transport Security header with a low initial max-age value. The max-age value will get increased on every (scheduled) run of certbot renew regardless of the certificate actually getting renewed, if the last increase took place longer than ten hours ago. The increase steps are visible in constants.AUTOHSTS_STEPS. Upon the first actual renewal after reaching the maximum increase step, the max-age value will be made "permanent" and will get value of one year. To achieve accurate VirtualHost discovery on subsequent runs, a comment with unique id string will be added to each enhanced VirtualHost. * AutoHSTS code rebased on master * Fixes to match the changes in master * Make linter happy with metaclass registration * Address small review comments * Use new enhancement interfaces * New style enhancement changes * Do not allow --hsts and --auto-hsts simultaneuously * MyPy annotation fixes and added test * Change oldest requrements to point to local certbot core version * Enable new style enhancements for run and install verbs * Test refactor * New test class for main.install tests * Move a test to a correct test class --- certbot-apache/certbot_apache/apache_util.py | 6 + certbot-apache/certbot_apache/configurator.py | 312 +++++++++++++++++- certbot-apache/certbot_apache/constants.py | 13 + certbot-apache/certbot_apache/parser.py | 32 ++ .../certbot_apache/tests/autohsts_test.py | 181 ++++++++++ .../certbot_apache/tests/configurator_test.py | 15 + .../certbot_apache/tests/parser_test.py | 7 + certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 2 +- certbot/cli.py | 8 + certbot/constants.py | 1 + certbot/main.py | 44 ++- certbot/plugins/enhancements.py | 159 +++++++++ certbot/plugins/enhancements_test.py | 65 ++++ certbot/tests/main_test.py | 82 ++++- certbot/tests/renewupdater_test.py | 87 +++-- certbot/updater.py | 46 +++ 17 files changed, 1027 insertions(+), 35 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/autohsts_test.py create mode 100644 certbot/plugins/enhancements.py create mode 100644 certbot/plugins/enhancements_test.py diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index f03c9da87..62342004f 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -1,4 +1,5 @@ """ Utility functions for certbot-apache plugin """ +import binascii import os from certbot import util @@ -98,3 +99,8 @@ def parse_define_file(filepath, varname): var_parts = v[2:].partition("=") return_vars[var_parts[0]] = var_parts[2] return return_vars + + +def unique_id(): + """ Returns an unique id to be used as a VirtualHost identifier""" + return binascii.hexlify(os.urandom(16)).decode("utf-8") diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 861fe4458..ab83a5332 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -13,7 +13,7 @@ import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict, Dict, List, Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, DefaultDict, Dict, List, Set, Union # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces @@ -22,6 +22,7 @@ from certbot import util from certbot.achallenges import KeyAuthorizationAnnotatedChallenge # pylint: disable=unused-import from certbot.plugins import common from certbot.plugins.util import path_surgery +from certbot.plugins.enhancements import AutoHSTSEnhancement from certbot_apache import apache_util from certbot_apache import augeas_configurator @@ -160,6 +161,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + # Temporary state for AutoHSTS enhancement + self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] # These will be set in the prepare function self.parser = None @@ -1472,6 +1475,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if need_to_save: self.save() + def find_vhost_by_id(self, id_str): + """ + Searches through VirtualHosts and tries to match the id in a comment + + :param str id_str: Id string for matching + + :returns: The matched VirtualHost or None + :rtype: :class:`~certbot_apache.obj.VirtualHost` or None + + :raises .errors.PluginError: If no VirtualHost is found + """ + + for vh in self.vhosts: + if self._find_vhost_id(vh) == id_str: + return vh + msg = "No VirtualHost with ID {} was found.".format(id_str) + logger.warning(msg) + raise errors.PluginError(msg) + + def _find_vhost_id(self, vhost): + """Tries to find the unique ID from the VirtualHost comments. This is + used for keeping track of VirtualHost directive over time. + + :param vhost: Virtual host to add the id + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :returns: The unique ID or None + :rtype: str or None + """ + + # Strip the {} off from the format string + search_comment = constants.MANAGED_COMMENT_ID.format("") + + id_comment = self.parser.find_comments(search_comment, vhost.path) + if id_comment: + # Use the first value, multiple ones shouldn't exist + comment = self.parser.get_arg(id_comment[0]) + return comment.split(" ")[-1] + return None + + def add_vhost_id(self, vhost): + """Adds an unique ID to the VirtualHost as a comment for mapping back + to it on later invocations, as the config file order might have changed. + If ID already exists, returns that instead. + + :param vhost: Virtual host to add or find the id + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :returns: The unique ID for vhost + :rtype: str or None + """ + + vh_id = self._find_vhost_id(vhost) + if vh_id: + return vh_id + + id_string = apache_util.unique_id() + comment = constants.MANAGED_COMMENT_ID.format(id_string) + self.parser.add_comment(vhost.path, comment) + return id_string + def _escape(self, fp): fp = fp.replace(",", "\\,") fp = fp.replace("[", "\\[") @@ -1531,6 +1595,78 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.warning("Failed %s for %s", enhancement, domain) raise + def _autohsts_increase(self, vhost, id_str, nextstep): + """Increase the AutoHSTS max-age value + + :param vhost: Virtual host object to modify + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :param str id_str: The unique ID string of VirtualHost + + :param int nextstep: Next AutoHSTS max-age value index + + """ + nextstep_value = constants.AUTOHSTS_STEPS[nextstep] + self._autohsts_write(vhost, nextstep_value) + self._autohsts[id_str] = {"laststep": nextstep, "timestamp": time.time()} + + def _autohsts_write(self, vhost, nextstep_value): + """ + Write the new HSTS max-age value to the VirtualHost file + """ + + hsts_dirpath = None + header_path = self.parser.find_dir("Header", None, vhost.path) + if header_path: + pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)' + for match in header_path: + if re.search(pat, self.aug.get(match).lower()): + hsts_dirpath = match + if not hsts_dirpath: + err_msg = ("Certbot was unable to find the existing HSTS header " + "from the VirtualHost at path {0}.").format(vhost.filep) + raise errors.PluginError(err_msg) + + # Prepare the HSTS header value + hsts_maxage = "\"max-age={0}\"".format(nextstep_value) + + # Update the header + # Our match statement was for string strict-transport-security, but + # we need to update the value instead. The next index is for the value + hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]") + self.aug.set(hsts_dirpath, hsts_maxage) + note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost " + "in {1}\n".format(nextstep_value, vhost.filep)) + logger.debug(note_msg) + self.save_notes += note_msg + self.save(note_msg) + + def _autohsts_fetch_state(self): + """ + Populates the AutoHSTS state from the pluginstorage + """ + try: + self._autohsts = self.storage.fetch("autohsts") + except KeyError: + self._autohsts = dict() + + def _autohsts_save_state(self): + """ + Saves the state of AutoHSTS object to pluginstorage + """ + self.storage.put("autohsts", self._autohsts) + self.storage.save() + + def _autohsts_vhost_in_lineage(self, vhost, lineage): + """ + Searches AutoHSTS managed VirtualHosts that belong to the lineage. + Matches the private key path. + """ + + return bool( + self.parser.find_dir("SSLCertificateKeyFile", + lineage.key_path, vhost.path)) + def _enable_ocsp_stapling(self, ssl_vhost, unused_options): """Enables OCSP Stapling @@ -2158,3 +2294,177 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + + def enable_autohsts(self, _unused_lineage, domains): + """ + Enable the AutoHSTS enhancement for defined domains + + :param _unused_lineage: Certificate lineage object, unused + :type _unused_lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + """ + + self._autohsts_fetch_state() + _enhanced_vhosts = [] + for d in domains: + matched_vhosts = self.choose_vhosts(d, create_if_no_ssl=False) + # We should be handling only SSL vhosts for AutoHSTS + vhosts = [vhost for vhost in matched_vhosts if vhost.ssl] + + if not vhosts: + msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a " + "domain {0} for enabling AutoHSTS enhancement.") + msg = msg_tmpl.format(d) + logger.warning(msg) + raise errors.PluginError(msg) + for vh in vhosts: + try: + self._enable_autohsts_domain(vh) + _enhanced_vhosts.append(vh) + except errors.PluginEnhancementAlreadyPresent: + if vh in _enhanced_vhosts: + continue + msg = ("VirtualHost for domain {0} in file {1} has a " + + "String-Transport-Security header present, exiting.") + raise errors.PluginEnhancementAlreadyPresent( + msg.format(d, vh.filep)) + if _enhanced_vhosts: + note_msg = "Enabling AutoHSTS" + self.save(note_msg) + logger.info(note_msg) + self.restart() + + # Save the current state to pluginstorage + self._autohsts_save_state() + + def _enable_autohsts_domain(self, ssl_vhost): + """Do the initial AutoHSTS deployment to a vhost + + :param ssl_vhost: The VirtualHost object to deploy the AutoHSTS + :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost` or None + + :raises errors.PluginEnhancementAlreadyPresent: When already enhanced + + """ + # This raises the exception + self._verify_no_matching_http_header(ssl_vhost, + "Strict-Transport-Security") + + if "headers_module" not in self.parser.modules: + self.enable_mod("headers") + # Prepare the HSTS header value + hsts_header = constants.HEADER_ARGS["Strict-Transport-Security"][:-1] + initial_maxage = constants.AUTOHSTS_STEPS[0] + hsts_header.append("\"max-age={0}\"".format(initial_maxage)) + + # Add ID to the VirtualHost for mapping back to it later + uniq_id = self.add_vhost_id(ssl_vhost) + self.save_notes += "Adding unique ID {0} to VirtualHost in {1}\n".format( + uniq_id, ssl_vhost.filep) + # Add the actual HSTS header + self.parser.add_dir(ssl_vhost.path, "Header", hsts_header) + note_msg = ("Adding gradually increasing HSTS header with initial value " + "of {0} to VirtualHost in {1}\n".format( + initial_maxage, ssl_vhost.filep)) + self.save_notes += note_msg + + # Save the current state to pluginstorage + self._autohsts[uniq_id] = {"laststep": 0, "timestamp": time.time()} + + def update_autohsts(self, _unused_domain): + """ + Increase the AutoHSTS values of VirtualHosts that the user has enabled + this enhancement for. + + :param _unused_domain: Not currently used + :type _unused_domain: Not Available + + """ + self._autohsts_fetch_state() + if not self._autohsts: + # No AutoHSTS enabled for any domain + return + curtime = time.time() + save_and_restart = False + for id_str, config in list(self._autohsts.items()): + if config["timestamp"] + constants.AUTOHSTS_FREQ > curtime: + # Skip if last increase was < AUTOHSTS_FREQ ago + continue + nextstep = config["laststep"] + 1 + if nextstep < len(constants.AUTOHSTS_STEPS): + # Have not reached the max value yet + try: + vhost = self.find_vhost_by_id(id_str) + except errors.PluginError: + msg = ("Could not find VirtualHost with ID {0}, disabling " + "AutoHSTS for this VirtualHost").format(id_str) + logger.warning(msg) + # Remove the orphaned AutoHSTS entry from pluginstorage + self._autohsts.pop(id_str) + continue + self._autohsts_increase(vhost, id_str, nextstep) + msg = ("Increasing HSTS max-age value for VirtualHost with id " + "{0}").format(id_str) + self.save_notes += msg + save_and_restart = True + + if save_and_restart: + self.save("Increased HSTS max-age values") + self.restart() + + self._autohsts_save_state() + + def deploy_autohsts(self, lineage): + """ + Checks if autohsts vhost has reached maximum auto-increased value + and changes the HSTS max-age to a high value. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + self._autohsts_fetch_state() + if not self._autohsts: + # No autohsts enabled for any vhost + return + + vhosts = [] + affected_ids = [] + # Copy, as we are removing from the dict inside the loop + for id_str, config in list(self._autohsts.items()): + if config["laststep"]+1 >= len(constants.AUTOHSTS_STEPS): + # max value reached, try to make permanent + try: + vhost = self.find_vhost_by_id(id_str) + except errors.PluginError: + msg = ("VirtualHost with id {} was not found, unable to " + "make HSTS max-age permanent.").format(id_str) + logger.warning(msg) + self._autohsts.pop(id_str) + continue + if self._autohsts_vhost_in_lineage(vhost, lineage): + vhosts.append(vhost) + affected_ids.append(id_str) + + save_and_restart = False + for vhost in vhosts: + self._autohsts_write(vhost, constants.AUTOHSTS_PERMANENT) + msg = ("Strict-Transport-Security max-age value for " + "VirtualHost in {0} was made permanent.").format(vhost.filep) + logger.debug(msg) + self.save_notes += msg+"\n" + save_and_restart = True + + if save_and_restart: + self.save("Made HSTS max-age permanent") + self.restart() + + for id_str in affected_ids: + self._autohsts.pop(id_str) + + # Update AutoHSTS storage (We potentially removed vhosts from managed) + self._autohsts_save_state() + + +AutoHSTSEnhancement.register(ApacheConfigurator) # pylint: disable=no-member diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index fd6a9eb11..23a7b7afd 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -48,3 +48,16 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy", HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, "Upgrade-Insecure-Requests": UIR_ARGS} + +AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400] +"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h""" + +AUTOHSTS_PERMANENT = 31536000 +"""Value for the last max-age of HSTS""" + +AUTOHSTS_FREQ = 172800 +"""Minimum time since last increase to perform a new one: 48h""" + +MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot" +MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}" +"""Managed by Certbot comments and the VirtualHost identification template""" diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 43878eda2..02337f8d4 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) class ApacheParser(object): + # pylint: disable=too-many-public-methods """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... @@ -350,6 +351,37 @@ class ApacheParser(object): else: self.aug.set(first_dir + "/arg", args) + def add_comment(self, aug_conf_path, comment): + """Adds the comment to the augeas path + + :param str aug_conf_path: Augeas configuration path to add directive + :param str comment: Comment content + + """ + self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment) + + def find_comments(self, arg, start=None): + """Finds a comment with specified content from the provided DOM path + + :param str arg: Comment content to search + :param str start: Beginning Augeas path to begin looking + + :returns: List of augeas paths containing the comment content + :rtype: list + + """ + if not start: + start = get_aug_path(self.root) + + comments = self.aug.match("%s//*[label() = '#comment']" % start) + + results = [] + for comment in comments: + c_content = self.aug.get(comment) + if c_content and arg in c_content: + results.append(comment) + return results + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py new file mode 100644 index 000000000..86d985079 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -0,0 +1,181 @@ +# pylint: disable=too-many-public-methods,too-many-lines +"""Test for certbot_apache.configurator AutoHSTS functionality""" +import re +import unittest +import mock +# six is used in mock.patch() +import six # pylint: disable=unused-import + +from certbot import errors +from certbot_apache import constants +from certbot_apache.tests import util + + +class AutoHSTSTest(util.ApacheTest): + """Tests for AutoHSTS feature""" + # pylint: disable=protected-access + + def setUp(self): # pylint: disable=arguments-differ + super(AutoHSTSTest, self).setUp() + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config.parser.modules.add("headers_module") + self.config.parser.modules.add("mod_headers.c") + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + + self.vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multiple_vhosts") + + def get_autohsts_value(self, vh_path): + """ Get value from Strict-Transport-Security header """ + header_path = self.config.parser.find_dir("Header", None, vh_path) + if header_path: + pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)' + for head in header_path: + if re.search(pat, self.config.parser.aug.get(head).lower()): + return self.config.parser.aug.get(head.replace("arg[3]", + "arg[4]")) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_autohsts_enable_headers_mod(self, mock_enable, _restart): + self.config.parser.modules.discard("headers_module") + self.config.parser.modules.discard("mod_header.c") + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + self.assertTrue(mock_enable.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_deploy_already_exists(self, _restart): + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + self.assertRaises(errors.PluginEnhancementAlreadyPresent, + self.config.enable_autohsts, + mock.MagicMock(), ["ocspvhost.com"]) + + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_increase(self, _mock_restart): + maxage = "\"max-age={0}\"" + initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) + inc_val = maxage.format(constants.AUTOHSTS_STEPS[1]) + + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Verify initial value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + initial_val) + # Increase + self.config.update_autohsts(mock.MagicMock()) + # Verify increased value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + inc_val) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase") + def test_autohsts_increase_noop(self, mock_increase, _restart): + maxage = "\"max-age={0}\"" + initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Verify initial value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + initial_val) + + self.config.update_autohsts(mock.MagicMock()) + # Freq not patched, so value shouldn't increase + self.assertFalse(mock_increase.called) + + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + def test_autohsts_increase_no_header(self, _restart): + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Remove the header + dir_locs = self.config.parser.find_dir("Header", None, + self.vh_truth[7].path) + dir_loc = "/".join(dir_locs[0].split("/")[:-1]) + self.config.parser.aug.remove(dir_loc) + self.assertRaises(errors.PluginError, + self.config.update_autohsts, + mock.MagicMock()) + + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_increase_and_make_permanent(self, _mock_restart): + maxage = "\"max-age={0}\"" + max_val = maxage.format(constants.AUTOHSTS_PERMANENT) + mock_lineage = mock.MagicMock() + mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem" + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + for i in range(len(constants.AUTOHSTS_STEPS)-1): + # Ensure that value is not made permanent prematurely + self.config.deploy_autohsts(mock_lineage) + self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path), + max_val) + self.config.update_autohsts(mock.MagicMock()) + # Value should match pre-permanent increment step + cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + cur_val) + # Make permanent + self.config.deploy_autohsts(mock_lineage) + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + max_val) + + def test_autohsts_update_noop(self): + with mock.patch("time.time") as mock_time: + # Time mock is used to make sure that the execution does not + # continue when no autohsts entries exist in pluginstorage + self.config.update_autohsts(mock.MagicMock()) + self.assertFalse(mock_time.called) + + def test_autohsts_make_permanent_noop(self): + self.config.storage.put = mock.MagicMock() + self.config.deploy_autohsts(mock.MagicMock()) + # Make sure that the execution does not continue when no entries in store + self.assertFalse(self.config.storage.put.called) + + @mock.patch("certbot_apache.display_ops.select_vhost") + def test_autohsts_no_ssl_vhost(self, mock_select): + mock_select.return_value = self.vh_truth[0] + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.assertRaises(errors.PluginError, + self.config.enable_autohsts, + mock.MagicMock(), "invalid.example.com") + self.assertTrue( + "Certbot was not able to find SSL" in mock_log.call_args[0][0]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id") + def test_autohsts_dont_enhance_twice(self, mock_id, _restart): + mock_id.return_value = "1234567" + self.config.enable_autohsts(mock.MagicMock(), + ["ocspvhost.com", "ocspvhost.com"]) + self.assertEquals(mock_id.call_count, 1) + + def test_autohsts_remove_orphaned(self): + # pylint: disable=protected-access + self.config._autohsts_fetch_state() + self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0} + + self.config._autohsts_save_state() + self.config.update_autohsts(mock.MagicMock()) + self.assertFalse("orphan_id" in self.config._autohsts) + # Make sure it's removed from the pluginstorage file as well + self.config._autohsts = None + self.config._autohsts_fetch_state() + self.assertFalse(self.config._autohsts) + + def test_autohsts_make_permanent_vhost_not_found(self): + # pylint: disable=protected-access + self.config._autohsts_fetch_state() + self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0} + self.config._autohsts_save_state() + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.config.deploy_autohsts(mock.MagicMock()) + self.assertTrue(mock_log.called) + self.assertTrue( + "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0]) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 23c1ee82b..350262634 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1487,6 +1487,21 @@ class MultipleVhostsTest(util.ApacheTest): "Upgrade-Insecure-Requests") self.assertTrue(mock_choose.called) + def test_add_vhost_id(self): + for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]: + vh_id = self.config.add_vhost_id(vh) + self.assertEqual(vh, self.config.find_vhost_by_id(vh_id)) + + def test_find_vhost_by_id_404(self): + self.assertRaises(errors.PluginError, + self.config.find_vhost_by_id, + "nonexistent") + + def test_add_vhost_id_already_exists(self): + first_id = self.config.add_vhost_id(self.vh_truth[0]) + second_id = self.config.add_vhost_id(self.vh_truth[0]) + self.assertEqual(first_id, second_id) + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index 4496781c9..f95f1b346 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -299,6 +299,13 @@ class BasicParserTest(util.ParserTest): errors.MisconfigurationError, self.parser.update_runtime_variables) + def test_add_comment(self): + from certbot_apache.parser import get_aug_path + self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456") + comm = self.parser.find_comments("123456") + self.assertEquals(len(comm), 1) + self.assertTrue(self.parser.loc["name"] in comm[0]) + class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 4e4aadbd8..e70ac0c7f 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.25.0 -certbot[dev]==0.21.1 +-e .[dev] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7a0dc43bf..8d15e7971 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -8,7 +8,7 @@ version = '0.26.0.dev0' # acme/certbot version. install_requires = [ 'acme>0.24.0', - 'certbot>=0.21.1', + 'certbot>0.25.1', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot/cli.py b/certbot/cli.py index 24e0bddac..81e926e2f 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -32,6 +32,7 @@ from certbot import util from certbot.display import util as display_util from certbot.plugins import disco as plugins_disco +import certbot.plugins.enhancements as enhancements import certbot.plugins.selection as plugin_selection logger = logging.getLogger(__name__) @@ -627,6 +628,10 @@ class HelpfulArgumentParser(object): raise errors.Error("Using --allow-subset-of-names with a" " wildcard domain is not supported.") + if parsed_args.hsts and parsed_args.auto_hsts: + raise errors.Error( + "Parameters --hsts and --auto-hsts cannot be used simultaneously.") + possible_deprecation_warning(parsed_args) return parsed_args @@ -1228,6 +1233,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) + # Populate the command line parameters for new style enhancements + enhancements.populate_cli(helpful.add) + _create_subparsers(helpful) _paths_parser(helpful) # _plugins_parsing should be the last thing to act upon the main diff --git a/certbot/constants.py b/certbot/constants.py index 93bc269af..7047424e5 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -58,6 +58,7 @@ CLI_DEFAULTS = dict( rsa_key_size=2048, must_staple=False, redirect=None, + auto_hsts=False, hsts=None, uir=None, staple=None, diff --git a/certbot/main.py b/certbot/main.py index 089eab0c3..a457919d4 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -36,7 +36,7 @@ from certbot import util from certbot.display import util as display_util, ops as display_ops from certbot.plugins import disco as plugins_disco from certbot.plugins import selection as plug_sel - +from certbot.plugins import enhancements USER_CANCELLED = ("User chose to cancel the operation and may " "reinvoke the client.") @@ -750,8 +750,8 @@ def _install_cert(config, le_client, domains, lineage=None): :param le_client: Client object :type le_client: client.Client - :param plugins: List of domains - :type plugins: `list` of `str` + :param domains: List of domains + :type domains: `list` of `str` :param lineage: Certificate lineage object. Defaults to `None` :type lineage: storage.RenewableCert @@ -789,11 +789,19 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") # If cert-path is defined, populate missing (ie. not overridden) values. # Unfortunately this can't be done in argument parser, as certificate # manager needs the access to renewal directory paths if config.certname: config = _populate_from_certname(config) + elif enhancements.are_requested(config): + # Preflight config check + raise errors.ConfigurationError("One or more of the requested enhancements " + "require --cert-name to be provided") + if config.key_path and config.cert_path: _check_certificate_and_key(config) domains, _ = _find_domains_or_certname(config, installer) @@ -804,6 +812,11 @@ def install(config, plugins): "If your certificate is managed by Certbot, please use --cert-name " "to define which certificate you would like to install.") + if enhancements.are_requested(config): + # In the case where we don't have certname, we have errored out already + lineage = cert_manager.lineage_for_certname(config, config.certname) + enhancements.enable(lineage, domains, installer, config) + def _populate_from_certname(config): """Helper function for install to populate missing config values from lineage defined by --cert-name.""" @@ -881,7 +894,8 @@ def enhance(config, plugins): """ supported_enhancements = ["hsts", "redirect", "uir", "staple"] # Check that at least one enhancement was requested on command line - if not any([getattr(config, enh) for enh in supported_enhancements]): + oldstyle_enh = any([getattr(config, enh) for enh in supported_enhancements]) + if not enhancements.are_requested(config) and not oldstyle_enh: msg = ("Please specify one or more enhancement types to configure. To list " "the available enhancement types, run:\n\n%s --help enhance\n") logger.warning(msg, sys.argv[0]) @@ -892,6 +906,10 @@ def enhance(config, plugins): except errors.PluginSelectionError as e: return str(e) + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") + certname_question = ("Which certificate would you like to use to enhance " "your configuration?") config.certname = cert_manager.get_certnames( @@ -907,11 +925,15 @@ def enhance(config, plugins): if not domains: raise errors.Error("User cancelled the domain selection. No domains " "defined, exiting.") + + lineage = cert_manager.lineage_for_certname(config, config.certname) if not config.chain_path: - lineage = cert_manager.lineage_for_certname(config, config.certname) config.chain_path = lineage.chain_path - le_client = _init_le_client(config, authenticator=None, installer=installer) - le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + if oldstyle_enh: + le_client = _init_le_client(config, authenticator=None, installer=installer) + le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + if enhancements.are_requested(config): + enhancements.enable(lineage, domains, installer, config) def rollback(config, plugins): @@ -1073,6 +1095,11 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals except errors.PluginSelectionError as e: return str(e) + # Preflight check for enhancement support by the selected installer + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") + # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) @@ -1091,6 +1118,9 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals _install_cert(config, le_client, domains, new_lineage) + if enhancements.are_requested(config) and new_lineage: + enhancements.enable(new_lineage, domains, installer, config) + if lineage is None or not should_get_cert: display_ops.success_installation(domains) else: diff --git a/certbot/plugins/enhancements.py b/certbot/plugins/enhancements.py new file mode 100644 index 000000000..506abe433 --- /dev/null +++ b/certbot/plugins/enhancements.py @@ -0,0 +1,159 @@ +"""New interface style Certbot enhancements""" +import abc +import six + +from certbot import constants + +from acme.magic_typing import Dict, List, Any # pylint: disable=unused-import, no-name-in-module + +def enabled_enhancements(config): + """ + Generator to yield the enabled new style enhancements. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + for enh in _INDEX: + if getattr(config, enh["cli_dest"]): + yield enh + +def are_requested(config): + """ + Checks if one or more of the requested enhancements are those of the new + enhancement interfaces. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + return any(enabled_enhancements(config)) + +def are_supported(config, installer): + """ + Checks that all of the requested enhancements are supported by the + installer. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: If all the requested enhancements are supported by the installer + :rtype: bool + """ + for enh in enabled_enhancements(config): + if not isinstance(installer, enh["class"]): + return False + return True + +def enable(lineage, domains, installer, config): + """ + Run enable method for each requested enhancement that is supported. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + for enh in enabled_enhancements(config): + getattr(installer, enh["enable_function"])(lineage, domains) + +def populate_cli(add): + """ + Populates the command line flags for certbot.cli.HelpfulParser + + :param add: Add function of certbot.cli.HelpfulParser + :type add: func + """ + for enh in _INDEX: + add(enh["cli_groups"], enh["cli_flag"], action=enh["cli_action"], + dest=enh["cli_dest"], default=enh["cli_flag_default"], + help=enh["cli_help"]) + + +@six.add_metaclass(abc.ABCMeta) +class AutoHSTSEnhancement(object): + """ + Enhancement interface that installer plugins can implement in order to + provide functionality that configures the software to have a + 'Strict-Transport-Security' with initially low max-age value that will + increase over time. + + The plugins implementing new style enhancements are responsible of handling + the saving of configuration checkpoints as well as calling possible restarts + of managed software themselves. + + Methods: + enable_autohsts is called when the header is initially installed using a + low max-age value. + + update_autohsts is called every time when Certbot is run using 'renew' + verb. The max-age value should be increased over time using this method. + + deploy_autohsts is called for every lineage that has had its certificate + renewed. A long HSTS max-age value should be set here, as we should be + confident that the user is able to automatically renew their certificates. + + + """ + + @abc.abstractmethod + def update_autohsts(self, lineage, *args, **kwargs): + """ + Gets called for each lineage every time Certbot is run with 'renew' verb. + Implementation of this method should increase the max-age value. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + + @abc.abstractmethod + def deploy_autohsts(self, lineage, *args, **kwargs): + """ + Gets called for a lineage when its certificate is successfully renewed. + Long max-age value should be set in implementation of this method. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + + @abc.abstractmethod + def enable_autohsts(self, lineage, domains, *args, **kwargs): + """ + Enables the AutoHSTS enhancement, installing + Strict-Transport-Security header with a low initial value to be increased + over the subsequent runs of Certbot renew. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + """ + +# This is used to configure internal new style enhancements in Certbot. These +# enhancement interfaces need to be defined in this file. Please do not modify +# this list from plugin code. +_INDEX = [ + { + "name": "AutoHSTS", + "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ + "Security security header", + "cli_flag": "--auto-hsts", + "cli_flag_default": constants.CLI_DEFAULTS["auto_hsts"], + "cli_groups": ["security", "enhance"], + "cli_dest": "auto_hsts", + "cli_action": "store_true", + "class": AutoHSTSEnhancement, + "updater_function": "update_autohsts", + "deployer_function": "deploy_autohsts", + "enable_function": "enable_autohsts" + } +] # type: List[Dict[str, Any]] diff --git a/certbot/plugins/enhancements_test.py b/certbot/plugins/enhancements_test.py new file mode 100644 index 000000000..b69dc9836 --- /dev/null +++ b/certbot/plugins/enhancements_test.py @@ -0,0 +1,65 @@ +"""Tests for new style enhancements""" +import unittest +import mock + +from certbot.plugins import enhancements +from certbot.plugins import null + +import certbot.tests.util as test_util + + +class EnhancementTest(test_util.ConfigTestCase): + """Tests for new style enhancements in certbot.plugins.enhancements""" + + def setUp(self): + super(EnhancementTest, self).setUp() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) + + + @test_util.patch_get_utility() + def test_enhancement_enabled_enhancements(self, _): + FAKEINDEX = [ + { + "name": "autohsts", + "cli_dest": "auto_hsts", + }, + { + "name": "somethingelse", + "cli_dest": "something", + } + ] + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + self.config.auto_hsts = True + self.config.something = True + enabled = list(enhancements.enabled_enhancements(self.config)) + self.assertEqual(len(enabled), 2) + self.assertTrue([i for i in enabled if i["name"] == "autohsts"]) + self.assertTrue([i for i in enabled if i["name"] == "somethingelse"]) + + def test_are_requested(self): + self.assertEquals( + len([i for i in enhancements.enabled_enhancements(self.config)]), 0) + self.assertFalse(enhancements.are_requested(self.config)) + self.config.auto_hsts = True + self.assertEquals( + len([i for i in enhancements.enabled_enhancements(self.config)]), 1) + self.assertTrue(enhancements.are_requested(self.config)) + + def test_are_supported(self): + self.config.auto_hsts = True + unsupported = null.Installer(self.config, "null") + self.assertTrue(enhancements.are_supported(self.config, self.mockinstaller)) + self.assertFalse(enhancements.are_supported(self.config, unsupported)) + + def test_enable(self): + self.config.auto_hsts = True + domains = ["example.com", "www.example.com"] + lineage = "lineage" + enhancements.enable(lineage, domains, self.mockinstaller, self.config) + self.assertTrue(self.mockinstaller.enable_autohsts.called) + self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0], + (lineage, domains)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 4b251c421..24d059e53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -29,7 +29,9 @@ from certbot import updater from certbot import util from certbot.plugins import disco +from certbot.plugins import enhancements from certbot.plugins import manual +from certbot.plugins import null import certbot.tests.util as test_util @@ -52,10 +54,11 @@ class TestHandleIdenticalCerts(unittest.TestCase): self.assertEqual(ret, ("reinstall", mock_lineage)) -class RunTest(unittest.TestCase): +class RunTest(test_util.ConfigTestCase): """Tests for certbot.main.run.""" def setUp(self): + super(RunTest, self).setUp() self.domain = 'example.org' self.patches = [ mock.patch('certbot.main._get_and_save_cert'), @@ -105,6 +108,15 @@ class RunTest(unittest.TestCase): self._call() self.mock_success_renewal.assert_called_once_with([self.domain]) + @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') + def test_run_enhancement_not_supported(self, mock_choose): + mock_choose.return_value = (null.Installer(self.config, "null"), None) + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.assertRaises(errors.NotSupportedError, + main.run, + self.config, plugins) + class CertonlyTest(unittest.TestCase): """Tests for certbot.main.certonly.""" @@ -1573,12 +1585,14 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): strict=self.config.strict_permissions) -class EnhanceTest(unittest.TestCase): +class EnhanceTest(test_util.ConfigTestCase): """Tests for certbot.main.enhance.""" def setUp(self): + super(EnhanceTest, self).setUp() self.get_utility_patch = test_util.patch_get_utility() self.mock_get_utility = self.get_utility_patch.start() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) def tearDown(self): self.get_utility_patch.stop() @@ -1670,7 +1684,7 @@ class EnhanceTest(unittest.TestCase): def test_no_enhancements_defined(self): self.assertRaises(errors.MisconfigurationError, - self._call, ['enhance']) + self._call, ['enhance', '-a', 'null']) @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') @mock.patch('certbot.main.display_ops.choose_values') @@ -1682,5 +1696,67 @@ class EnhanceTest(unittest.TestCase): mock_client = self._call(['enhance', '--hsts']) self.assertFalse(mock_client.enhance_config.called) + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @test_util.patch_get_utility() + def test_enhancement_enable(self, _, _rec, mock_inst, mock_choose, mock_lineage): + mock_inst.return_value = self.mockinstaller + mock_choose.return_value = ["example.com", "another.tld"] + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + self._call(['enhance', '--auto-hsts']) + self.assertTrue(self.mockinstaller.enable_autohsts.called) + self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0][1], + ["example.com", "another.tld"]) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @test_util.patch_get_utility() + def test_enhancement_enable_not_supported(self, _, _rec, mock_inst, mock_choose, mock_lineage): + mock_inst.return_value = null.Installer(self.config, "null") + mock_choose.return_value = ["example.com", "another.tld"] + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + self.assertRaises( + errors.NotSupportedError, + self._call, ['enhance', '--auto-hsts']) + + def test_enhancement_enable_conflict(self): + self.assertRaises( + errors.Error, + self._call, ['enhance', '--auto-hsts', '--hsts']) + + +class InstallTest(test_util.ConfigTestCase): + """Tests for certbot.main.install.""" + + def setUp(self): + super(InstallTest, self).setUp() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_install_enhancement_not_supported(self, mock_inst, _rec): + mock_inst.return_value = null.Installer(self.config, "null") + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.assertRaises(errors.NotSupportedError, + main.install, + self.config, plugins) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_install_enhancement_no_certname(self, mock_inst, _rec): + mock_inst.return_value = self.mockinstaller + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.config.certname = None + self.assertRaises(errors.ConfigurationError, + main.install, + self.config, plugins) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index bd1cd891e..c1b97843e 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -6,6 +6,8 @@ from certbot import interfaces from certbot import main from certbot import updater +from certbot.plugins import enhancements + import certbot.tests.util as test_util @@ -14,25 +16,10 @@ class RenewUpdaterTest(test_util.ConfigTestCase): def setUp(self): super(RenewUpdaterTest, self).setUp() - class MockInstallerGenericUpdater(interfaces.GenericUpdater): - """Mock class that implements GenericUpdater""" - def __init__(self, *args, **kwargs): - # pylint: disable=unused-argument - self.restart = mock.MagicMock() - self.callcounter = mock.MagicMock() - def generic_updates(self, lineage, *args, **kwargs): - self.callcounter(*args, **kwargs) - - class MockInstallerRenewDeployer(interfaces.RenewDeployer): - """Mock class that implements RenewDeployer""" - def __init__(self, *args, **kwargs): - # pylint: disable=unused-argument - self.callcounter = mock.MagicMock() - def renew_deploy(self, lineage, *args, **kwargs): - self.callcounter(*args, **kwargs) - - self.generic_updater = MockInstallerGenericUpdater() - self.renew_deployer = MockInstallerRenewDeployer() + self.generic_updater = mock.MagicMock(spec=interfaces.GenericUpdater) + self.generic_updater.restart = mock.MagicMock() + self.renew_deployer = mock.MagicMock(spec=interfaces.RenewDeployer) + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') @@ -48,16 +35,16 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertTrue(mock_generic_updater.restart.called) mock_generic_updater.restart.reset_mock() - mock_generic_updater.callcounter.reset_mock() + mock_generic_updater.generic_updates.reset_mock() updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertEqual(mock_generic_updater.callcounter.call_count, 1) + self.assertEqual(mock_generic_updater.generic_updates.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): lineage = mock.MagicMock() mock_deployer = self.renew_deployer updater.run_renewal_deployer(self.config, lineage, mock_deployer) - self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + self.assertTrue(mock_deployer.renew_deploy.called_with(lineage)) @mock.patch("certbot.updater.logger.debug") def test_updater_skip_dry_run(self, mock_log): @@ -75,6 +62,62 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertEquals(mock_log.call_args[0][0], "Skipping renewal deployer in dry-run mode.") + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_updates(self, mock_select): + mock_select.return_value = (self.mockinstaller, None) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertTrue(self.mockinstaller.update_autohsts.called) + self.assertEqual(self.mockinstaller.update_autohsts.call_count, 1) + + def test_enhancement_deployer(self): + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertTrue(self.mockinstaller.deploy_autohsts.called) + + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_updates_not_called(self, mock_select): + self.config.disable_renew_updates = True + mock_select.return_value = (self.mockinstaller, None) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertFalse(self.mockinstaller.update_autohsts.called) + + def test_enhancement_deployer_not_called(self): + self.config.disable_renew_updates = True + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertFalse(self.mockinstaller.deploy_autohsts.called) + + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_no_updater(self, mock_select): + FAKEINDEX = [ + { + "name": "Test", + "class": enhancements.AutoHSTSEnhancement, + "updater_function": None, + "deployer_function": "deploy_autohsts", + "enable_function": "enable_autohsts" + } + ] + mock_select.return_value = (self.mockinstaller, None) + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertFalse(self.mockinstaller.update_autohsts.called) + + def test_enhancement_no_deployer(self): + FAKEINDEX = [ + { + "name": "Test", + "class": enhancements.AutoHSTSEnhancement, + "updater_function": "deploy_autohsts", + "deployer_function": None, + "enable_function": "enable_autohsts" + } + ] + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertFalse(self.mockinstaller.deploy_autohsts.called) + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py index 112cf06ef..fb7c52f77 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -5,6 +5,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import selection as plug_sel +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) @@ -33,6 +34,7 @@ def run_generic_updaters(config, lineage, plugins): logger.warning("Could not choose appropriate plugin for updaters: %s", e) return _run_updaters(lineage, installer, config) + _run_enhancement_updaters(lineage, installer, config) def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used @@ -57,6 +59,7 @@ def run_renewal_deployer(config, lineage, installer): if not config.disable_renew_updates and isinstance(installer, interfaces.RenewDeployer): installer.renew_deploy(lineage) + _run_enhancement_deployers(lineage, installer, config) def _run_updaters(lineage, installer, config): """Helper function to run the updater interface methods if supported by the @@ -74,3 +77,46 @@ def _run_updaters(lineage, installer, config): if not config.disable_renew_updates: if isinstance(installer, interfaces.GenericUpdater): installer.generic_updates(lineage) + +def _run_enhancement_updaters(lineage, installer, config): + """Iterates through known enhancement interfaces. If the installer implements + an enhancement interface and the enhance interface has an updater method, the + updater method gets run. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration object + :type config: interfaces.IConfig + """ + + if config.disable_renew_updates: + return + for enh in enhancements._INDEX: # pylint: disable=protected-access + if isinstance(installer, enh["class"]) and enh["updater_function"]: + getattr(installer, enh["updater_function"])(lineage) + + +def _run_enhancement_deployers(lineage, installer, config): + """Iterates through known enhancement interfaces. If the installer implements + an enhancement interface and the enhance interface has an deployer method, the + deployer method gets run. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration object + :type config: interfaces.IConfig + """ + + if config.disable_renew_updates: + return + for enh in enhancements._INDEX: # pylint: disable=protected-access + if isinstance(installer, enh["class"]) and enh["deployer_function"]: + getattr(installer, enh["deployer_function"])(lineage) From 6771b8e05bfe438165df67d8cc671857510f3957 Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Thu, 21 Jun 2018 15:52:08 -0400 Subject: [PATCH 545/631] docs: move warning about distro provided renewal (#6133) Currently, you must read ten paragraphs about writing renewal hooks before you find that most distributions will automatically renew certs for you. This is burying the lede in a major way; moving it up to the header seems a better choice. --- docs/using.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 40d8f8452..46599a06e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -454,6 +454,12 @@ Renewing certificates days). Make sure you renew the certificates at least once in 3 months. +.. seealso:: Many of the certbot clients obtained through a + distribution come with automatic renewal out of the box, + such as Debian and Ubuntu versions installed through `apt`, + CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ + for more details. + As of version 0.10.0, Certbot supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew them. The simplest form is simply @@ -560,12 +566,6 @@ can run on a regular basis, like every week or every day). In that case, you are likely to want to use the ``-q`` or ``--quiet`` quiet flag to silence all output except errors. -.. seealso:: Many of the certbot clients obtained through a - distribution come with automatic renewal out of the box, - such as Debian and Ubuntu versions installed through `apt`, - CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ - for more details. - If you are manually renewing all of your certificates, the ``--force-renewal`` flag may be helpful; it causes the expiration time of the certificate(s) to be ignored when considering renewal, and attempts to From 2ac0b5520833b83b093b10f02e5a02686842e17a Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 21 Jun 2018 13:23:09 -0700 Subject: [PATCH 546/631] Reuse ACMEv1 accounts for ACMEv2 in production (#6134) * Reuse accounts made with ACMEv1 when using an ACMEv2 Let's Encrypt server. This commit turns the feature on for the production server; the bulk of the work was done in 8e4303a. * add upgrade test for production server --- certbot/constants.py | 1 + certbot/tests/account_test.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/certbot/constants.py b/certbot/constants.py index 7047424e5..d31faa71c 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -161,6 +161,7 @@ ACCOUNTS_DIR = "accounts" """Directory where all accounts are saved.""" LE_REUSE_SERVERS = { + 'acme-v02.api.letsencrypt.org/directory': 'acme-v01.api.letsencrypt.org/directory', 'acme-staging-v02.api.letsencrypt.org/directory': 'acme-staging.api.letsencrypt.org/directory' } diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index a8059fbcf..a4fe5edb7 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -218,12 +218,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) - def test_upgrade_version(self): + def test_upgrade_version_staging(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([self.acc], self.storage.find_all()) + def test_upgrade_version_production(self): + self._set_server('https://acme-v01.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-v02.api.letsencrypt.org/directory') + self.assertEqual([self.acc], self.storage.find_all()) + @mock.patch('os.rmdir') def test_corrupted_account(self, mock_rmdir): # pylint: disable=protected-access From 1e1e7d8e973be0b7376d96fb07c59816f694a83a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 21 Jun 2018 15:40:43 -0700 Subject: [PATCH 547/631] Improve UA default in docs (#6120) * Use less informative UA values in docs. * set CERTBOT_DOCS during release --- certbot/client.py | 12 ++++++++++-- certbot/tests/client_test.py | 33 +++++++++++++++++++++++++++++++++ tools/release.sh | 3 ++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d97de0571..4d4915a27 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -65,9 +65,17 @@ def determine_user_agent(config): if config.user_agent is None: ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") - ua = ua.format(certbot.__version__, cli.cli_command, util.get_os_info_ua(), + if os.environ.get("CERTBOT_DOCS") == "1": + cli_command = "certbot(-auto)" + os_info = "OS_NAME OS_VERSION" + python_version = "major.minor.patchlevel" + else: + cli_command = cli.cli_command + os_info = util.get_os_info_ua() + python_version = platform.python_version() + ua = ua.format(certbot.__version__, cli_command, os_info, config.authenticator, config.installer, config.verb, - ua_flags(config), platform.python_version(), + ua_flags(config), python_version, "; " + config.user_agent_comment if config.user_agent_comment else "") else: ua = config.user_agent diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a4425bca9..70ab4c798 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -1,5 +1,6 @@ """Tests for certbot.client.""" import os +import platform import shutil import tempfile import unittest @@ -16,6 +17,38 @@ KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") +class DetermineUserAgentTest(test_util.ConfigTestCase): + """Tests for certbot.client.determine_user_agent.""" + + def _call(self): + from certbot.client import determine_user_agent + return determine_user_agent(self.config) + + @mock.patch.dict(os.environ, {"CERTBOT_DOCS": "1"}) + def test_docs_value(self): + self._test(expect_doc_values=True) + + @mock.patch.dict(os.environ, {}) + def test_real_values(self): + self._test(expect_doc_values=False) + + def _test(self, expect_doc_values): + ua = self._call() + + if expect_doc_values: + doc_value_check = self.assertIn + real_value_check = self.assertNotIn + else: + doc_value_check = self.assertNotIn + real_value_check = self.assertIn + + doc_value_check("certbot(-auto)", ua) + doc_value_check("OS_NAME OS_VERSION", ua) + doc_value_check("major.minor.patchlevel", ua) + real_value_check(util.get_os_info_ua(), ua) + real_value_check(platform.python_version(), ua) + + class RegisterTest(test_util.ConfigTestCase): """Tests for certbot.client.register.""" diff --git a/tools/release.sh b/tools/release.sh index a8de208b5..069d311d1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -153,7 +153,8 @@ kill $! cd ~- # get a snapshot of the CLI help for the docs -certbot --help all > docs/cli-help.txt +# We set CERTBOT_DOCS to use dummy values in example user-agent string. +CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt jws --help > acme/docs/jws-help.txt cd .. From 7890de62ecfd0a3de60b81fabba4659cf80605d1 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 21 Jun 2018 16:11:02 -0700 Subject: [PATCH 548/631] doc(postfix): install instructions (#6136) fixes #6131 * doc(postfix): install instructions * address brad's comments --- certbot-postfix/README.rst | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index 78fd9a421..e6367e365 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -2,9 +2,22 @@ Postfix plugin for Certbot ========================== -To install your certs with this plugin, run: +Note: this MTA installer is in **developer beta**-- we appreciate any testing, feedback, or +feature requests for this plugin. -``certbot install --installer postfix --cert-path --key-path -d `` +To install this plugin, in the root of this repo, run:: -And there you go! If you'd like to obtain these certificates via certbot, there's more documentation on how to do this `here `_. + ./tools/venv.sh + source venv/bin/activate +You can use this installer with any `authenticator plugin +`_. +For instance, with the `standalone authenticator +`_, which requires no extra server +software, you might run:: + + sudo ./venv/bin/certbot run --standalone -i postfix -d + +To just install existing certs with this plugin, run:: + + sudo ./venv/bin/certbot install -i postfix --cert-path --key-path -d From 80cd134847edd1b860315796d3accf1a46171b16 Mon Sep 17 00:00:00 2001 From: r5d Date: Mon, 25 Jun 2018 21:02:07 -0400 Subject: [PATCH 549/631] certbot.cli: Remove debug-challenges option for `renew` subcommand. (#6141) Addresses issue #5005. --- certbot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 81e926e2f..5c4313ea4 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1082,7 +1082,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="Show tracebacks in case of errors, and allow certbot-auto " "execution on experimental platforms") helpful.add( - [None, "certonly", "renew", "run"], "--debug-challenges", action="store_true", + [None, "certonly", "run"], "--debug-challenges", action="store_true", default=flag_default("debug_challenges"), help="After setting up challenges, wait for user input before " "submitting to CA") From 87e1912bf91cf91e8ff1d3e4d5928353ae1a294c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 25 Jun 2018 18:09:30 -0700 Subject: [PATCH 550/631] Show both possible Nginx default server root values in docs (#6137) See https://github.com/certbot/website/pull/348#issuecomment-399257703. ``` $ certbot --help all | grep -C 3 nginx-server-root nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and ``` ``` $ CERTBOT_DOCS=1 certbot --help all | grep -C 3 nginx-server-root nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx or /usr/local/etc/nginx) --nginx-ctl NGINX_CTL ``` * Show both possible Nginx default server root values in docs * add test * check that exactly one server root is in the default * use default magic --- certbot-nginx/certbot_nginx/configurator.py | 11 +++++++- certbot-nginx/certbot_nginx/constants.py | 7 +++-- .../certbot_nginx/tests/configurator_test.py | 26 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 41b5124b8..b80d95613 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -69,8 +69,9 @@ class NginxConfigurator(common.Installer): @classmethod def add_parser_arguments(cls, add): + default_server_root = _determine_default_server_root() add("server-root", default=constants.CLI_DEFAULTS["server_root"], - help="Nginx server root directory.") + help="Nginx server root directory. (default: %s)" % default_server_root) add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the " "'nginx' binary, used for 'configtest' and retrieving nginx " "version number.") @@ -1129,3 +1130,11 @@ def install_ssl_options_conf(options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" return common.install_version_controlled_file(options_ssl, options_ssl_digest, constants.MOD_SSL_CONF_SRC, constants.ALL_SSL_OPTIONS_HASHES) + +def _determine_default_server_root(): + if os.environ.get("CERTBOT_DOCS") == "1": + default_server_root = "%s or %s" % (constants.LINUX_SERVER_ROOT, + constants.FREEBSD_DARWIN_SERVER_ROOT) + else: + default_server_root = constants.CLI_DEFAULTS["server_root"] + return default_server_root diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index dfc451202..d749b6989 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -2,10 +2,13 @@ import pkg_resources import platform +FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" +LINUX_SERVER_ROOT = "/etc/nginx" + if platform.system() in ('FreeBSD', 'Darwin'): - server_root_tmp = "/usr/local/etc/nginx" + server_root_tmp = FREEBSD_DARWIN_SERVER_ROOT else: - server_root_tmp = "/etc/nginx" + server_root_tmp = LINUX_SERVER_ROOT CLI_DEFAULTS = dict( server_root=server_root_tmp, diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 9386c3cd9..75dfca563 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -949,5 +949,31 @@ class InstallSslOptionsConfTest(util.NginxTest): " with the sha256 hash of self.config.mod_ssl_conf when it is updated.") +class DetermineDefaultServerRootTest(certbot_test_util.ConfigTestCase): + """Tests for certbot_nginx.configurator._determine_default_server_root.""" + + def _call(self): + from certbot_nginx.configurator import _determine_default_server_root + return _determine_default_server_root() + + @mock.patch.dict(os.environ, {"CERTBOT_DOCS": "1"}) + def test_docs_value(self): + self._test(expect_both_values=True) + + @mock.patch.dict(os.environ, {}) + def test_real_values(self): + self._test(expect_both_values=False) + + def _test(self, expect_both_values): + server_root = self._call() + + if expect_both_values: + self.assertIn("/usr/local/etc/nginx", server_root) + self.assertIn("/etc/nginx", server_root) + else: + self.assertTrue("/etc/nginx" in server_root or "/usr/local/etc/nginx" in server_root) + self.assertFalse("/etc/nginx" in server_root and "/usr/local/etc/nginx" in server_root) + + if __name__ == "__main__": unittest.main() # pragma: no cover From a4760cfe56ee88bbc776859fb95a141d128adcaf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 26 Jun 2018 15:33:41 -0700 Subject: [PATCH 551/631] Partially revert "Implement TLS-ALPN-01 challenge and standalone TLS-ALPN server (#5894)" (#6144) This partially reverts commit 15f1405fff7083bf5d4f599a58c54a43be499740. A basic tls-alpn-01 implementation is left so we can successfully parse the challenge so it can be used in boulder's tests. --- acme/acme/challenges.py | 158 +++------------------------- acme/acme/challenges_test.py | 93 +--------------- acme/acme/crypto_util.py | 61 +++-------- acme/acme/crypto_util_test.py | 16 +-- acme/acme/standalone.py | 48 +-------- acme/acme/standalone_test.py | 57 ---------- acme/acme/testdata/README | 6 +- acme/acme/testdata/rsa1024_cert.pem | 13 --- 8 files changed, 32 insertions(+), 420 deletions(-) delete mode 100644 acme/acme/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 30983e28f..2f0bf004d 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,6 +1,5 @@ """ACME Identifier Validation Challenges.""" import abc -import codecs import functools import hashlib import logging @@ -8,7 +7,7 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -from OpenSSL import crypto +import OpenSSL import requests import six @@ -412,8 +411,8 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ if key is None: - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, bits) + key = OpenSSL.crypto.PKey() + key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) return crypto_util.gen_ss_cert(key, [ # z_domain is too big to fit into CN, hence first dummy domain 'dummy', self.z_domain.decode()], force_san=True), key @@ -508,152 +507,19 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) -@ChallengeResponse.register -class TLSALPN01Response(KeyAuthorizationChallengeResponse): - """ACME tls-alpn-01 challenge response.""" - typ = "tls-alpn-01" - - PORT = 443 - """Verification port as defined by the protocol. - - You can override it (e.g. for testing) by passing ``port`` to - `simple_verify`. - - """ - - ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" - ACME_TLS_1_PROTOCOL = "acme-tls/1" - - @property - def h(self): - """Hash value stored in challenge certificate""" - return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() - - def gen_cert(self, domain, key=None, bits=2048): - """Generate tls-alpn-01 certificate. - - :param unicode domain: Domain verified by the challenge. - :param OpenSSL.crypto.PKey key: Optional private key used in - certificate generation. If not provided (``None``), then - fresh key will be generated. - :param int bits: Number of bits for newly generated key. - - :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` - - """ - if key is None: - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, bits) - - - # Instead of using a ASN.1 encoding library just append the OCTET STRING tag (0x04) - # and the length of the SHA256 hash (0x20) since both of these should never change - der_value = b"DER:0420" + codecs.encode(self.h, 'hex') - acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, - critical=True, value=der_value) - - return crypto_util.gen_ss_cert(key, [domain], force_san=True, - extensions=[acme_extension]), key - - def probe_cert(self, domain, host=None, port=None): - """Probe tls-alpn-01 challenge certificate. - - :param unicode domain: domain being validated, required. - :param string host: IP address used to probe the certificate. - :param int port: Port used to probe the certificate. - - """ - if host is None: - host = socket.gethostbyname(domain) - logger.debug('%s resolved to %s', domain, host) - if port is None: - port = self.PORT - - return crypto_util.probe_sni(host=host, port=port, name=domain, - alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) - - def verify_cert(self, domain, cert): - """Verify tls-alpn-01 challenge certificate. - - :param unicode domain: Domain name being validated. - :param OpensSSL.crypto.X509 cert: Challenge certificate. - - :returns: Whether the certificate was successfully verified. - :rtype: bool - - """ - # pylint: disable=protected-access - names = crypto_util._pyopenssl_cert_or_req_all_names(cert) - logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) - if len(names) != 1 or names[0].lower() != domain.lower(): - return False - - for i in range(cert.get_extension_count()): - ext = cert.get_extension(i) - # FIXME: assume this is the ACME extension. Currently there is no - # way to get full OID of an unknown extension from pyopenssl. - if ext.get_short_name() == b'UNDEF': - data = ext.get_data() - # Add the ASN.1 tag/length prefix to the hash before comparison - return data == b'\x04\x20' + self.h - - return False - - # pylint: disable=too-many-arguments - def simple_verify(self, chall, domain, account_public_key, - cert=None, host=None, port=None): - """Simple verify. - - Verify ``validation`` using ``account_public_key``, optionally - probe tls-alpn-01 certificate and check using `verify_cert`. - - :param .challenges.TLSALPN01 chall: Corresponding challenge. - :param str domain: Domain name being validated. - :param JWK account_public_key: - :param OpenSSL.crypto.X509 cert: Optional certificate. If not - provided (``None``) certificate will be retrieved using - `probe_cert`. - :param string host: IP address used to probe the certificate. - :param int port: Port used to probe the certificate. - - - :returns: ``True`` iff client's control of the domain has been - verified. - :rtype: bool - - """ - if not self.verify(chall, account_public_key): - logger.debug("Verification of key authorization in response failed") - return False - - if cert is None: - try: - cert = self.probe_cert(domain=domain, host=host, port=port) - except errors.Error as error: - logger.debug(str(error), exc_info=True) - return False - - return self.verify_cert(cert, domain) - - @Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): - """ACME tls-alpn-01 challenge.""" - response_cls = TLSALPN01Response - typ = response_cls.typ + """ACME tls-alpn-01 challenge. + + This class simply allows parsing the TLS-ALPN-01 challenge returned from + the CA. Full TLS-ALPN-01 support is not currently provided. + + """ + typ = "tls-alpn-01" def validation(self, account_key, **kwargs): - """Generate validation. - - :param JWK account_key: - :param OpenSSL.crypto.PKey cert_key: Optional private key used - in certificate generation. If not provided (``None``), then - fresh key will be generated. - - :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` - - """ - return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) + """Generate validation for the challenge.""" + raise NotImplementedError() @Challenge.register # pylint: disable=too-many-ancestors diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index b929d4939..661d25a35 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -393,91 +393,6 @@ class TLSSNI01Test(unittest.TestCase): mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) -class TLSALPN01ResponseTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes - - def setUp(self): - from acme.challenges import TLSALPN01 - self.chall = TLSALPN01( - token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) - self.domain = u'example.com' - self.domain2 = u'example2.com' - - self.response = self.chall.response(KEY) - self.jmsg = { - 'resource': 'challenge', - 'type': 'tls-alpn-01', - 'keyAuthorization': self.response.key_authorization, - } - - def test_to_partial_json(self): - self.assertEqual(self.jmsg, self.response.to_partial_json()) - - def test_from_json(self): - from acme.challenges import TLSALPN01Response - self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) - - def test_from_json_hashable(self): - from acme.challenges import TLSALPN01Response - hash(TLSALPN01Response.from_json(self.jmsg)) - - def test_gen_verify_cert(self): - key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') - cert, key2 = self.response.gen_cert(self.domain, key1) - self.assertEqual(key1, key2) - self.assertTrue(self.response.verify_cert(self.domain, cert)) - - def test_gen_verify_cert_gen_key(self): - cert, key = self.response.gen_cert(self.domain) - self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) - self.assertTrue(self.response.verify_cert(self.domain, cert)) - - def test_verify_bad_cert(self): - self.assertFalse(self.response.verify_cert(self.domain, - test_util.load_cert('cert.pem'))) - - def test_verify_bad_domain(self): - key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') - cert, key2 = self.response.gen_cert(self.domain, key1) - self.assertEqual(key1, key2) - self.assertFalse(self.response.verify_cert(self.domain2, cert)) - - def test_simple_verify_bad_key_authorization(self): - key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) - self.response.simple_verify(self.chall, "local", key2.public_key()) - - @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) - def test_simple_verify(self, mock_verify_cert): - mock_verify_cert.return_value = mock.sentinel.verification - self.assertEqual( - mock.sentinel.verification, self.response.simple_verify( - self.chall, self.domain, KEY.public_key(), - cert=mock.sentinel.cert)) - mock_verify_cert.assert_called_once_with( - self.response, mock.sentinel.cert, self.domain) - - @mock.patch('acme.challenges.socket.gethostbyname') - @mock.patch('acme.challenges.crypto_util.probe_sni') - def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): - mock_gethostbyname.return_value = '127.0.0.1' - self.response.probe_cert('foo.com') - mock_gethostbyname.assert_called_once_with('foo.com') - mock_probe_sni.assert_called_once_with( - host='127.0.0.1', port=self.response.PORT, name='foo.com', - alpn_protocols=['acme-tls/1']) - - self.response.probe_cert('foo.com', host='8.8.8.8') - mock_probe_sni.assert_called_with( - host='8.8.8.8', port=mock.ANY, name='foo.com', - alpn_protocols=['acme-tls/1']) - - @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') - def test_simple_verify_false_on_probe_error(self, mock_probe_cert): - mock_probe_cert.side_effect = errors.Error - self.assertFalse(self.response.simple_verify( - self.chall, self.domain, KEY.public_key())) - - class TLSALPN01Test(unittest.TestCase): def setUp(self): @@ -506,12 +421,8 @@ class TLSALPN01Test(unittest.TestCase): self.assertRaises( jose.DeserializationError, TLSALPN01.from_json, self.jmsg) - @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') - def test_validation(self, mock_gen_cert): - mock_gen_cert.return_value = ('cert', 'key') - self.assertEqual(('cert', 'key'), self.msg.validation( - KEY, cert_key=mock.sentinel.cert_key)) - mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + def test_validation(self): + self.assertRaises(NotImplementedError, self.msg.validation, KEY) class DNSTest(unittest.TestCase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d25c2340b..d0e203417 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -31,15 +31,6 @@ logger = logging.getLogger(__name__) _DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): - def __init__(self, certs): - self.certs = certs - - def __call__(self, connection): - server_name = connection.get_servername() - return self.certs.get(server_name, None) - - class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. @@ -47,25 +38,12 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. - :ivar alpn_selection: Hook to select negotiated ALPN protocol for - connection. - :ivar cert_selection: Hook to select certificate for connection. If given, - `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs=None, - method=_DEFAULT_TLSSNI01_SSL_METHOD, alpn_selection=None, - cert_selection=None): + def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): self.sock = sock - self.alpn_selection = alpn_selection + self.certs = certs self.method = method - if not cert_selection and not certs: - raise ValueError("Neither cert_selection or certs specified.") - if cert_selection and certs: - raise ValueError("Both cert_selection and certs specified.") - if cert_selection is None: - cert_selection = _DefaultCertSelection(certs) - self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -82,19 +60,18 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :type connection: :class:`OpenSSL.Connection` """ - pair = self.cert_selection(connection) - if pair is None: - logger.debug("Certificate selection for server name %s failed, dropping SSL", - connection.get_servername()) + server_name = connection.get_servername() + try: + key, cert = self.certs[server_name] + except KeyError: + logger.debug("Server name (%s) not recognized, dropping SSL", + server_name) return - key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) - if self.alpn_selection is not None: - new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -119,8 +96,6 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) - if self.alpn_selection is not None: - context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -136,9 +111,8 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0), - alpn_protocols=None): +def probe_sni(name, host, port=443, timeout=300, + method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -150,8 +124,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. - :param alpn_protocols: Protocols to request using ALPN. - :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -188,8 +160,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 - if alpn_protocols is not None: - client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -281,14 +251,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): + validity=(7 * 24 * 60 * 60), force_san=True): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: - :param extensions: List of additional extensions to include in the cert. - :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -301,13 +269,10 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - if extensions is None: - extensions = [] - - extensions.append( + extensions = [ crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ) + ] cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index b661e4e70..36d62b324 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -19,6 +19,7 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" + def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -33,8 +34,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), - certs) + self.socket = SSLSocket(socket.socket(), certs=certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -66,18 +66,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') -class SSLSocketTest(unittest.TestCase): - """Tests for acme.crypto_util.SSLSocket.""" - - def test_ssl_socket_invalid_arguments(self): - from acme.crypto_util import SSLSocket - with self.assertRaises(ValueError): - _ = SSLSocket(None, {'sni': ('key', 'cert')}, - cert_selection=lambda _: None) - with self.assertRaises(ValueError): - _ = SSLSocket(None) - - class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 3bcb0b230..ff9159933 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -43,14 +43,7 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, cert_selection=self._cert_selection, - alpn_selection=getattr(self, '_alpn_selection', None), - method=self.method) - - def _cert_selection(self, connection): - """Callback selecting certificate for connection.""" - server_name = connection.get_servername() - return self.certs.get(server_name, None) + self.socket, certs=self.certs, method=self.method) def server_bind(self): # pylint: disable=missing-docstring self._wrap_sock() @@ -154,45 +147,6 @@ class TLSSNI01DualNetworkedServers(BaseDualNetworkedServers): BaseDualNetworkedServers.__init__(self, TLSSNI01Server, *args, **kwargs) -class BadALPNProtos(Exception): - """Error raised when cannot negotiate ALPN protocol.""" - pass - - -class TLSALPN01Server(TLSServer, ACMEServerMixin): - """TLSALPN01 Server.""" - - ACME_TLS_1_PROTOCOL = b"acme-tls/1" - - def __init__(self, server_address, certs, challenge_certs, ipv6=False): - TLSServer.__init__( - self, server_address, BaseRequestHandlerWithLogging, certs=certs, - ipv6=ipv6) - self.challenge_certs = challenge_certs - - def _cert_selection(self, connection): - # TODO: We would like to serve challenge cert only if asked for it via - # ALPN. To do this, we need to retrieve the list of protos from client - # hello, but this is currently impossible with openssl [0], and ALPN - # negotiation is done after cert selection. - # Therefore, currently we always return challenge cert, and terminate - # handshake in alpn_selection() if ALPN protos are not what we expect. - # [0] https://github.com/openssl/openssl/issues/4952 - server_name = connection.get_servername() - logger.debug("Serving challenge cert for server name %s", server_name) - return self.challenge_certs.get(server_name, None) - - def _alpn_selection(self, _connection, alpn_protos): - """Callback to select alpn protocol.""" - if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: - logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) - return self.ACME_TLS_1_PROTOCOL - # Raising an exception causes openssl to terminate handshake and - # send fatal tls alert. - logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) - raise BadALPNProtos("Got: %s" % str(alpn_protos)) - - class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): """BaseRequestHandler with logging.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index aee592187..1591187e5 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -10,7 +10,6 @@ import unittest from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose import mock import requests @@ -120,62 +119,6 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) -@unittest.skipUnless( - hasattr(SSL.Connection, "set_alpn_protos") and - hasattr(SSL.Context, "set_alpn_select_callback"), - "pyOpenSSL too old") -class TLSALPN01ServerTest(unittest.TestCase): - """Test for acme.standalone.TLSALPN01Server.""" - - def setUp(self): - self.certs = {b'localhost': ( - test_util.load_pyopenssl_private_key('rsa2048_key.pem'), - test_util.load_cert('rsa2048_cert.pem'), - )} - # Use different certificate for challenge. - self.challenge_certs = {b'localhost': ( - test_util.load_pyopenssl_private_key('rsa1024_key.pem'), - test_util.load_cert('rsa1024_cert.pem'), - )} - from acme.standalone import TLSALPN01Server - self.server = TLSALPN01Server(("", 0), certs=self.certs, - challenge_certs=self.challenge_certs) - # pylint: disable=no-member - self.thread = threading.Thread(target=self.server.serve_forever) - self.thread.start() - - def tearDown(self): - self.server.shutdown() # pylint: disable=no-member - self.thread.join() - - #TODO: This is not implemented yet, see comments in standalone.py - #def test_certs(self): - # host, port = self.server.socket.getsockname()[:2] - # cert = crypto_util.probe_sni( - # b'localhost', host=host, port=port, timeout=1) - # # Expect normal cert when connecting without ALPN. - # self.assertEqual(jose.ComparableX509(cert), - # jose.ComparableX509(self.certs[b'localhost'][1])) - - def test_challenge_certs(self): - host, port = self.server.socket.getsockname()[:2] - cert = crypto_util.probe_sni( - b'localhost', host=host, port=port, timeout=1, - alpn_protocols=[b"acme-tls/1"]) - # Expect challenge cert when connecting with ALPN. - self.assertEqual( - jose.ComparableX509(cert), - jose.ComparableX509(self.challenge_certs[b'localhost'][1]) - ) - - def test_bad_alpn(self): - host, port = self.server.socket.getsockname()[:2] - with self.assertRaises(errors.Error): - crypto_util.probe_sni( - b'localhost', host=host, port=port, timeout=1, - alpn_protocols=[b"bad-alpn"]) - - class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" diff --git a/acme/acme/testdata/README b/acme/acme/testdata/README index d65cc3018..dfe3f5405 100644 --- a/acme/acme/testdata/README +++ b/acme/acme/testdata/README @@ -10,8 +10,6 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificates: +and for the certificate: - openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der - openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem - openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem + openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der diff --git a/acme/acme/testdata/rsa1024_cert.pem b/acme/acme/testdata/rsa1024_cert.pem deleted file mode 100644 index 1b7912181..000000000 --- a/acme/acme/testdata/rsa1024_cert.pem +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV -BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow -FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ -AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr -Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW -l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G -A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X -XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB -ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI -Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY -qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x ------END CERTIFICATE----- From 2304f7fcdadd6baad61a925e9a11541d2b534d6f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 27 Jun 2018 19:32:53 +0300 Subject: [PATCH 552/631] Remove unnecessary dotfiles (#6151) --- .../augeas_vhosts/apache2/mods-enabled/.gitignore | 0 .../multiple_vhosts/apache2/mods-enabled/.gitignore | 0 .../apache/apache2/modules.d/.keep_www-servers_apache-2 | 0 .../apache/apache2/vhosts.d/.keep_www-servers_apache-2 | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore delete mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore delete mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 delete mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 deleted file mode 100644 index e69de29bb..000000000 From f169e7374bedf11a77c860d522f7e9b7eaad0824 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 27 Jun 2018 21:35:01 +0300 Subject: [PATCH 553/631] Interactive certificate selection with install verb (#6097) If either --cert-name or both --key-path and --cert-path (in which case the user requests installation for a certificate not managed by Certbot) are not provided, prompt the user with managed certificates and let them choose. Fixes: #5824 --- certbot/main.py | 7 +++++++ certbot/tests/main_test.py | 26 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index a457919d4..6078f87a6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -789,6 +789,13 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) + custom_cert = (config.key_path and config.cert_path) + if not config.certname and not custom_cert: + certname_question = "Which certificate would you like to install?" + config.certname = cert_manager.get_certnames( + config, "install", allow_multiple=False, + custom_prompt=certname_question)[0] + if not enhancements.are_supported(config, installer): raise errors.NotSupportedError("One ore more of the requested enhancements " "are not supported by the selected installer") diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 24d059e53..a2115a486 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -761,20 +761,25 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_param_error(self, _inst, _rec): - self.assertRaises(errors.ConfigurationError, - self._call, - ['install', '--key-path', '/tmp/key_path']) - self.assertRaises(errors.ConfigurationError, - self._call, - ['install', '--cert-path', '/tmp/key_path']) - self.assertRaises(errors.ConfigurationError, - self._call, - ['install']) self.assertRaises(errors.ConfigurationError, self._call, ['install', '--cert-name', 'notfound', '--key-path', 'invalid']) + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.cert_manager.get_certnames') + @mock.patch('certbot.main._install_cert') + def test_installer_select_cert(self, mock_inst, mock_getcert, _inst, _rec): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install'], mockisfile=True) + self.assertTrue(mock_getcert.called) + self.assertTrue(mock_inst.called) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -1742,6 +1747,7 @@ class InstallTest(test_util.ConfigTestCase): mock_inst.return_value = null.Installer(self.config, "null") plugins = disco.PluginsRegistry.find_all() self.config.auto_hsts = True + self.config.certname = "nonexistent" self.assertRaises(errors.NotSupportedError, main.install, self.config, plugins) @@ -1753,6 +1759,8 @@ class InstallTest(test_util.ConfigTestCase): plugins = disco.PluginsRegistry.find_all() self.config.auto_hsts = True self.config.certname = None + self.config.key_path = "/tmp/nonexistent" + self.config.cert_path = "/tmp/nonexistent" self.assertRaises(errors.ConfigurationError, main.install, self.config, plugins) From ad3c547e1fcd88d7877f0abbed3cf999ba7fc94b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Jun 2018 14:29:21 -0700 Subject: [PATCH 554/631] Update cli-help.txt to use generic values (#6143) --- docs/cli-help.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 49821a7b5..594bcfd04 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,12 +108,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.1 (certbot; - darwin 10.13.5) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags - encoded in the user agent are: --duplicate, --force- - renew, --allow-subset-of-names, -n, and whether any - hooks are set. + "". (default: CertbotACMEClient/0.25.1 + (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX + Installer/YYY (SUBCOMMAND; flags: FLAGS) + Py/major.minor.patchlevel). The flags encoded in the + user agent are: --duplicate, --force-renew, --allow- + subset-of-names, -n, and whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -638,7 +638,7 @@ nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT - Nginx server root directory. (default: + Nginx server root directory. (default: /etc/nginx or /usr/local/etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and From 742a57722ba99812920baae47a8d6b094859a952 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Jun 2018 17:35:43 -0700 Subject: [PATCH 555/631] fix server_root default tests on macOS (#6149) --- certbot-nginx/certbot_nginx/tests/configurator_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 75dfca563..4d23f3518 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -971,8 +971,7 @@ class DetermineDefaultServerRootTest(certbot_test_util.ConfigTestCase): self.assertIn("/usr/local/etc/nginx", server_root) self.assertIn("/etc/nginx", server_root) else: - self.assertTrue("/etc/nginx" in server_root or "/usr/local/etc/nginx" in server_root) - self.assertFalse("/etc/nginx" in server_root and "/usr/local/etc/nginx" in server_root) + self.assertTrue(server_root == "/etc/nginx" or server_root == "/usr/local/etc/nginx") if __name__ == "__main__": From d00a31622dd4f797a75b140b95320e99bcc4c953 Mon Sep 17 00:00:00 2001 From: Bahram Aghaei Date: Thu, 28 Jun 2018 17:43:52 +0000 Subject: [PATCH 556/631] run Isort on imported packages (#6138) --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e32925583..8176883ad 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,7 @@ import codecs import os import re -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup # Workaround for http://bugs.python.org/issue8876, see # http://bugs.python.org/issue8876#msg208792 From 64e06d4201a8695580533c3cb07370fac226ebea Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 28 Jun 2018 10:55:21 -0700 Subject: [PATCH 557/631] Use greater than or equal to in requirements. (#6117) * Use greater than or equal to in requirements. This changes the existing requirements using strictly greater than to greater than or equal to so that they're more conventional. * Use >= for certbot-postfix. Despite it previously saying 'certbot>0.23.0', certbot-postfix/local-oldest-requirements.txt was pinned to 0.23.0 so let's just use certbot>=0.23.0. --- certbot-apache/setup.py | 4 ++-- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot-postfix/setup.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 8d15e7971..ffaa6a863 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -7,8 +7,8 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>0.24.0', - 'certbot>0.25.1', + 'acme>=0.25.0', + 'certbot>=0.26.0.dev0', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8e2821332..c8806e862 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -6,7 +6,7 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>0.24.0', + 'acme>=0.25.0', 'certbot>=0.21.1', 'boto3', 'mock', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index d9cb4a9c2..b486b2778 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -8,7 +8,7 @@ version = '0.26.0.dev0' # acme/certbot version. install_requires = [ 'acme>=0.25.0', - 'certbot>0.21.1', + 'certbot>=0.22.0', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index cde1ac7c2..4c53477d2 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -6,7 +6,7 @@ version = '0.26.0.dev0' install_requires = [ 'acme>=0.25.0', - 'certbot>0.23.0', + 'certbot>=0.23.0', 'setuptools', 'six', 'zope.component', diff --git a/setup.py b/setup.py index 8176883ad..9ef9ec0d2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>0.24.0', + 'acme>=0.25.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From a816cc89798823cc043058f016e4d89cb3fca4f6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 28 Jun 2018 13:34:16 -0700 Subject: [PATCH 558/631] Use account reuse symlink logic when loading an account (#6156) Fixes #6154. * add symlinking to load flow * test account reuse on load --- certbot/account.py | 36 ++++++++++++++++++++++------------- certbot/tests/account_test.py | 8 ++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index c4eeb1388..5e9455048 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -175,33 +175,43 @@ class AccountFileStorage(interfaces.AccountStorage): except errors.AccountStorageError: logger.debug("Account loading problem", exc_info=True) - if not accounts and server_path in constants.LE_REUSE_SERVERS: # find all for the next link down prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_accounts = self._find_all_for_server_path(prev_server_path) # if we found something, link to that if prev_accounts: - if os.path.islink(accounts_dir): - os.unlink(accounts_dir) - else: - try: - os.rmdir(accounts_dir) - except OSError: - return [] - prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) - os.symlink(prev_account_dir, accounts_dir) + try: + self._symlink_to_accounts_dir(prev_server_path, server_path) + except OSError: + return [] accounts = prev_accounts return accounts def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + if os.path.islink(accounts_dir): + os.unlink(accounts_dir) + else: + os.rmdir(accounts_dir) + prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) + os.symlink(prev_account_dir, accounts_dir) + def _load_for_server_path(self, account_id, server_path): account_dir_path = self._account_dir_path_for_server_path(account_id, server_path) - if not os.path.isdir(account_dir_path): - raise errors.AccountNotFound( - "Account at %s does not exist" % account_dir_path) + if not os.path.isdir(account_dir_path): # isdir is also true for symlinks + if server_path in constants.LE_REUSE_SERVERS: + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) + # we didn't error so we found something, so create a symlink to that + self._symlink_to_accounts_dir(prev_server_path, server_path) + return prev_loaded_account + else: + raise errors.AccountNotFound( + "Account at %s does not exist" % account_dir_path) try: with open(self._regr_path(account_dir_path)) as regr_file: diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index a4fe5edb7..e7f82a5b8 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -241,6 +241,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + def test_upgrade_load(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() From 6e13c2ccc71e7629f451f89339870013fad7dc30 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 28 Jun 2018 15:06:52 -0700 Subject: [PATCH 559/631] Add --disable=locally-enabled to .pylintrc. (#6159) --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 36d8c286f..1d3f0ac4f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -41,7 +41,7 @@ load-plugins=linter_plugin # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code +disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code # abstract-class-not-used cannot be disabled locally (at least in # pylint 1.4.1), same for abstract-class-little-used From 552e60a12691d434ea86a7c6bf5bc4a270651ba8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 29 Jun 2018 05:27:58 -0700 Subject: [PATCH 560/631] Don't use hardcoded port in tests (#6145) * Don't use port 1234 in standalone tests. * rename unused variable * add back failure case * Add back probe connection error test. * fix lint * remove unused import * fix test file coverage * prevent future heisenbug --- acme/acme/crypto_util_test.py | 26 ++++++++++++++++-------- acme/acme/standalone_test.py | 37 ++++++++++++++--------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 36d62b324..168489d86 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -42,28 +42,38 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.server_thread = threading.Thread( # pylint: disable=no-member target=self.server.handle_request) - self.server_thread.start() - time.sleep(1) # TODO: avoid race conditions in other way def tearDown(self): - self.server_thread.join() + if self.server_thread.is_alive(): + # The thread may have already terminated. + self.server_thread.join() # pragma: no cover def _probe(self, name): from acme.crypto_util import probe_sni return jose.ComparableX509(probe_sni( name, host='127.0.0.1', port=self.port)) + def _start_server(self): + self.server_thread.start() + time.sleep(1) # TODO: avoid race conditions in other way + def test_probe_ok(self): + self._start_server() self.assertEqual(self.cert, self._probe(b'foo')) def test_probe_not_recognized_name(self): + self._start_server() self.assertRaises(errors.Error, self._probe, b'bar') - # TODO: py33/py34 tox hangs forever on do_handshake in second probe - #def probe_connection_error(self): - # self._probe(b'foo') - # #time.sleep(1) # TODO: avoid race conditions in other way - # self.assertRaises(errors.Error, self._probe, b'bar') + def test_probe_connection_error(self): + # pylint has a hard time with six + self.server.server_close() # pylint: disable=no-member + original_timeout = socket.getdefaulttimeout() + try: + socket.setdefaulttimeout(1) + self.assertRaises(errors.Error, self._probe, b'bar') + finally: + socket.setdefaulttimeout(original_timeout) class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 1591187e5..6beea038e 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -4,10 +4,10 @@ import shutil import socket import threading import tempfile -import time import unittest from six.moves import http_client # pylint: disable=import-error +from six.moves import queue # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error import josepy as jose @@ -16,7 +16,6 @@ import requests from acme import challenges from acme import crypto_util -from acme import errors from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module @@ -261,10 +260,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): os.path.join(localhost_dir, 'key.pem')) from acme.standalone import simple_tls_sni_01_server - self.port = 1234 self.thread = threading.Thread( target=simple_tls_sni_01_server, kwargs={ - 'cli_args': ('xxx', '--port', str(self.port)), + 'cli_args': ('filename',), 'forever': False, }, ) @@ -276,25 +274,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): self.thread.join() shutil.rmtree(self.test_cwd) - def test_it(self): - max_attempts = 5 - for attempt in range(max_attempts): - try: - cert = crypto_util.probe_sni( - b'localhost', b'0.0.0.0', self.port) - except errors.Error: - self.assertTrue(attempt + 1 < max_attempts, "Timeout!") - time.sleep(1) # wait until thread starts - else: - self.assertEqual(jose.ComparableX509(cert), - test_util.load_comparable_cert( - 'rsa2048_cert.pem')) - break + @mock.patch('acme.standalone.logger') + def test_it(self, mock_logger): + # Use a Queue because mock objects aren't thread safe. + q = queue.Queue() # type: queue.Queue[int] + # Add port number to the queue. + mock_logger.info.side_effect = lambda *args: q.put(args[-1]) + self.thread.start() - if attempt == 0: - # the first attempt is always meant to fail, so we can test - # the socket failure code-path for probe_sni, as well - self.thread.start() + # After the timeout, an exception is raised if the queue is empty. + port = q.get(timeout=5) + cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port) + self.assertEqual(jose.ComparableX509(cert), + test_util.load_comparable_cert( + 'rsa2048_cert.pem')) if __name__ == "__main__": From ab9851d97e7f58290d0299da8fec49f072226750 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Jul 2018 06:57:58 -0700 Subject: [PATCH 561/631] Upgrade to the latest cryptography version (#6169) This allows certbot-auto and our development setup to work with Python 3.7. --- letsencrypt-auto-source/letsencrypt-auto | 51 ++++++++----------- .../pieces/dependency-requirements.txt | 51 ++++++++----------- 2 files changed, 40 insertions(+), 62 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index dc9190630..d571a5a8d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 376e19deb..54498cb3e 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -64,37 +64,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 From cad95466b05e6be51c1c29eaa91e6e3b7ea3cefd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Jul 2018 02:11:04 -0700 Subject: [PATCH 562/631] Allow py37 testing (#6170) * Reorganize packages in tox to allow for py37 tests certbot-dns-cloudflare doesn't currently work in Python 3.7 because it transitively depends on pyYAML which doesn't yet support Python 3.7. See https://github.com/yaml/pyyaml/issues/126 for more info. * add py37 tox environment --- tox.ini | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ef71b52be..b44d30449 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -dns_packages = - certbot-dns-cloudflare \ +python37_compatible_dns_packages = certbot-dns-cloudxns \ certbot-dns-digitalocean \ certbot-dns-dnsimple \ @@ -25,14 +24,22 @@ dns_packages = certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 -all_packages = +dns_packages = + certbot-dns-cloudflare \ + {[base]python37_compatible_dns_packages} +nondns_packages = acme[dev] \ .[dev] \ certbot-apache \ - {[base]dns_packages} \ certbot-nginx \ certbot-postfix \ letshelp-certbot +python37_compatible_packages = + {[base]nondns_packages} \ + {[base]python37_compatible_dns_packages} +all_packages = + {[base]nondns_packages} \ + {[base]dns_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -62,6 +69,13 @@ commands = setenv = PYTHONHASHSEED = 0 +[testenv:py37] +commands = + {[base]install_and_test} {[base]python37_compatible_packages} + python tests/lock_test.py +setenv = + {[testenv]setenv} + [testenv:py27-oldest] commands = {[testenv]commands} From cb076539ec99423b8e3f8ff7f0bc4fa2b7452766 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Jul 2018 08:26:42 -0700 Subject: [PATCH 563/631] Remove .dev0 from version numbers during releases. (#6116) This allows us to depend on packages like acme>=0.26.0.dev0 during development and automatically change it to acme>=0.26.0 during the release. We use `git add -p` to be safe, but if .dev0 is used at all in our released setup.py files, we're probably doing something wrong. --- tools/release.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/release.sh b/tools/release.sh index 069d311d1..02c7ccca5 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -87,6 +87,14 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" +for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +do + sed -i 's/\.dev0//' "$pkg_dir/setup.py" +done +# We only add Certbot's setup.py here because the other files are added in the +# call to SetVersion below. +git add -p setup.py + SetVersion() { ver="$1" # bumping Certbot's version number is done differently From 08378203df8978ec8b0c971abb88a0c51d094521 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 6 Jul 2018 09:08:40 -0700 Subject: [PATCH 564/631] Add Python 3.7 tests (#6179) * Remove apacheconftest packages. The apacheconftests handle installing Apache dependencies, so let's remove it from the general case. * We don't need to run dpkg -s in before_install. * Remove augeas sources. We only needed it for Ubuntu Precise which is dead and it doesn't work in Ubuntu Xenial. * Upgrade Python 3.6 tests to 3.7. Let's continue the approach of testing on the oldest and newest versions of Python 3. We will continue testing on Python 3.6 in the nightly tests. * Revert "We don't need to run dpkg -s in before_install." This reverts commit e5d35099a79985ee97a26931e08451620d711522. * let apacheconftest handle deps --- .travis.yml | 11 +++-------- .../tests/apache-conf-files/apache-conf-test | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3d964326..937d24610 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,8 +41,9 @@ matrix: env: TOXENV=py34 sudo: required services: docker - - python: "3.6" - env: TOXENV=py36 + - python: "3.7" + dist: xenial + env: TOXENV=py37 sudo: required services: docker - sudo: required @@ -77,8 +78,6 @@ sudo: false addons: apt: - sources: - - augeas packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder. - python-dev - python-virtualenv @@ -90,10 +89,6 @@ addons: # For certbot-nginx integration testing - nginx-light - openssl - # for apacheconftest - - apache2 - - libapache2-mod-wsgi - - libapache2-mod-macro install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" script: diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test index e0715a46b..dcbba9d3e 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test @@ -46,6 +46,7 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then + sudo apt-get install -y apache2 sudo apt-get install -y libapache2-mod-wsgi sudo apt-get install -y libapache2-mod-macro From 2564566e1c512619f71bb8bbdc77821907a46ebe Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 6 Jul 2018 23:19:29 +0300 Subject: [PATCH 565/631] Do not call IPlugin.prepare() for updaters when running renew (#6167) interfaces.GenericUpdater and new enhancement interface updater functions get run on every invocation of Certbot with "renew" verb for every lineage. This causes performance problems for users with large configurations, because of plugin plumbing and preparsing happening in prepare() method of installer plugins. This PR moves the responsibility to call prepare() to the plugin (possibly) implementing a new style enhancement interface. Fixes: #6153 * Do not call IPlugin.prepare() for updaters when running renew * Check prepare called in tests * Refine pydoc and make the function name more informative * Verify the plugin type --- certbot-apache/certbot_apache/configurator.py | 5 ++ .../certbot_apache/tests/autohsts_test.py | 5 +- certbot/interfaces.py | 3 ++ certbot/main.py | 2 +- certbot/plugins/enhancements.py | 7 ++- certbot/plugins/selection.py | 29 +++++++++++ certbot/plugins/selection_test.py | 48 ++++++++++++++++++- certbot/tests/main_test.py | 14 +++--- certbot/tests/renewupdater_test.py | 22 +++++---- certbot/updater.py | 10 ++-- 10 files changed, 119 insertions(+), 26 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index ab83a5332..bb77e2e41 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -165,6 +165,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] # These will be set in the prepare function + self._prepared = False self.parser = None self.version = version self.vhosts = None @@ -249,6 +250,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( "Unable to lock %s", self.conf("server-root")) + self._prepared = True def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. @@ -2394,6 +2396,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): continue nextstep = config["laststep"] + 1 if nextstep < len(constants.AUTOHSTS_STEPS): + # If installer hasn't been prepared yet, do it now + if not self._prepared: + self.prepare() # Have not reached the max value yet try: vhost = self.find_vhost_by_id(id_str) diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 86d985079..73da33f15 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -55,7 +55,9 @@ class AutoHSTSTest(util.ApacheTest): @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_autohsts_increase(self, _mock_restart): + @mock.patch("certbot_apache.configurator.ApacheConfigurator.prepare") + def test_autohsts_increase(self, mock_prepare, _mock_restart): + self.config._prepared = False maxage = "\"max-age={0}\"" initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) inc_val = maxage.format(constants.AUTOHSTS_STEPS[1]) @@ -69,6 +71,7 @@ class AutoHSTSTest(util.ApacheTest): # Verify increased value self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), inc_val) + self.assertTrue(mock_prepare.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase") diff --git a/certbot/interfaces.py b/certbot/interfaces.py index a5fb426e6..2e837d1d2 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -620,6 +620,9 @@ class GenericUpdater(object): methods, and interfaces.GenericUpdater.register(InstallerClass) should be called from the installer code. + The plugins implementing this enhancement are responsible of handling + the saving of configuration checkpoints as well as other calls to + interface methods of `interfaces.IInstaller` such as prepare() and restart() """ @abc.abstractmethod diff --git a/certbot/main.py b/certbot/main.py index 6078f87a6..556722104 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1199,11 +1199,11 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. + # Run deployer updater.run_renewal_deployer(config, renewed_lineage, installer) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) - # Run deployer def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. diff --git a/certbot/plugins/enhancements.py b/certbot/plugins/enhancements.py index 506abe433..7ca096610 100644 --- a/certbot/plugins/enhancements.py +++ b/certbot/plugins/enhancements.py @@ -88,7 +88,8 @@ class AutoHSTSEnhancement(object): The plugins implementing new style enhancements are responsible of handling the saving of configuration checkpoints as well as calling possible restarts - of managed software themselves. + of managed software themselves. For update_autohsts method, the installer may + have to call prepare() to finalize the plugin initialization. Methods: enable_autohsts is called when the header is initially installed using a @@ -112,6 +113,10 @@ class AutoHSTSEnhancement(object): :param lineage: Certificate lineage object :type lineage: certbot.storage.RenewableCert + + .. note:: prepare() method inherited from `interfaces.IPlugin` might need + to be called manually within implementation of this interface method + to finalize the plugin initialization. """ @abc.abstractmethod diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 030d5b6db..95f123a46 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -39,6 +39,35 @@ def pick_authenticator( return pick_plugin( config, default, plugins, question, (interfaces.IAuthenticator,)) +def get_unprepared_installer(config, plugins): + """ + Get an unprepared interfaces.IInstaller object. + + :param certbot.interfaces.IConfig config: Configuration + :param certbot.plugins.disco.PluginsRegistry plugins: + All plugins registered as entry points. + + :returns: Unprepared installer plugin or None + :rtype: IPlugin or None + """ + + _, req_inst = cli_plugin_requests(config) + if not req_inst: + return None + installers = plugins.filter(lambda p_ep: p_ep.name == req_inst) + installers.init(config) + installers = installers.verify((interfaces.IInstaller,)) + if len(installers) > 1: + raise errors.PluginSelectionError( + "Found multiple installers with the name %s, Certbot is unable to " + "determine which one to use. Skipping." % req_inst) + if installers: + inst = list(installers.values())[0] + logger.debug("Selecting plugin: %s", inst) + return inst.init(config) + else: + raise errors.PluginSelectionError( + "Could not select or initialize the requested installer %s." % req_inst) def pick_plugin(config, default, plugins, question, ifaces): """Pick plugin. diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index ab480544a..44d64ab8e 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -6,10 +6,13 @@ import unittest import mock import zope.component +from certbot import errors +from certbot import interfaces + from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.display import util as display_util +from certbot.plugins.disco import PluginsRegistry from certbot.tests import util as test_util -from certbot import interfaces class ConveniencePickPluginTest(unittest.TestCase): @@ -170,5 +173,48 @@ class ChoosePluginTest(unittest.TestCase): self.assertTrue("default" in mock_util().menu.call_args[1]) +class GetUnpreparedInstallerTest(test_util.ConfigTestCase): + """Tests for certbot.plugins.selection.get_unprepared_installer.""" + + def setUp(self): + super(GetUnpreparedInstallerTest, self).setUp() + self.mock_apache_fail_ep = mock.Mock( + description_with_name="afail") + self.mock_apache_fail_ep.name = "afail" + self.mock_apache_ep = mock.Mock( + description_with_name="apache") + self.mock_apache_ep.name = "apache" + self.mock_apache_plugin = mock.MagicMock() + self.mock_apache_ep.init.return_value = self.mock_apache_plugin + self.plugins = PluginsRegistry({ + "afail": self.mock_apache_fail_ep, + "apache": self.mock_apache_ep, + }) + + def _call(self): + from certbot.plugins.selection import get_unprepared_installer + return get_unprepared_installer(self.config, self.plugins) + + def test_no_installer_defined(self): + self.config.configurator = None + self.assertEquals(self._call(), None) + + def test_no_available_installers(self): + self.config.configurator = "apache" + self.plugins = PluginsRegistry({}) + self.assertRaises(errors.PluginSelectionError, self._call) + + def test_get_plugin(self): + self.config.configurator = "apache" + installer = self._call() + self.assertTrue(installer is self.mock_apache_plugin) + + def test_multiple_installers_returned(self): + self.config.configurator = "apache" + # Two plugins with the same name + self.mock_apache_fail_ep.name = "apache" + self.assertRaises(errors.PluginSelectionError, self._call) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index a2115a486..cc4e6c293 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1493,17 +1493,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(mock_handle.called) @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_plugin_selection_error(self, mock_choose): + @mock.patch('certbot.updater._run_updaters') + def test_plugin_selection_error(self, mock_run, mock_choose): mock_choose.side_effect = errors.PluginSelectionError self.assertRaises(errors.PluginSelectionError, main.renew_cert, None, None, None) - with mock.patch('certbot.updater.logger.warning') as mock_log: - self.config.dry_run = False - updater.run_generic_updaters(self.config, None, None) - self.assertTrue(mock_log.called) - self.assertTrue("Could not choose appropriate plugin for updaters" - in mock_log.call_args[0][0]) + self.config.dry_run = False + updater.run_generic_updaters(self.config, None, None) + # Make sure we're returning None, and hence not trying to run the + # without installer + self.assertFalse(mock_run.called) class UnregisterTest(unittest.TestCase): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c1b97843e..5a362072c 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -23,13 +23,15 @@ class RenewUpdaterTest(test_util.ConfigTestCase): @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + @mock.patch('certbot.plugins.selection.get_unprepared_installer') @test_util.patch_get_utility() - def test_server_updates(self, _, mock_select, mock_getsave): + def test_server_updates(self, _, mock_geti, mock_select, mock_getsave): mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater mock_select.return_value = (mock_generic_updater, None) + mock_geti.return_value = mock_generic_updater with mock.patch('certbot.main._init_le_client'): main.renew_cert(self.config, None, mock.MagicMock()) self.assertTrue(mock_generic_updater.restart.called) @@ -62,9 +64,9 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertEquals(mock_log.call_args[0][0], "Skipping renewal deployer in dry-run mode.") - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_updates(self, mock_select): - mock_select.return_value = (self.mockinstaller, None) + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_updates(self, mock_geti): + mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertTrue(self.mockinstaller.update_autohsts.called) self.assertEqual(self.mockinstaller.update_autohsts.call_count, 1) @@ -74,10 +76,10 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.mockinstaller) self.assertTrue(self.mockinstaller.deploy_autohsts.called) - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_updates_not_called(self, mock_select): + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_updates_not_called(self, mock_geti): self.config.disable_renew_updates = True - mock_select.return_value = (self.mockinstaller, None) + mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertFalse(self.mockinstaller.update_autohsts.called) @@ -87,8 +89,8 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.mockinstaller) self.assertFalse(self.mockinstaller.deploy_autohsts.called) - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_no_updater(self, mock_select): + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_no_updater(self, mock_geti): FAKEINDEX = [ { "name": "Test", @@ -98,7 +100,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): "enable_function": "enable_autohsts" } ] - mock_select.return_value = (self.mockinstaller, None) + mock_geti.return_value = self.mockinstaller with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertFalse(self.mockinstaller.update_autohsts.called) diff --git a/certbot/updater.py b/certbot/updater.py index fb7c52f77..58df6fcb4 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -28,13 +28,13 @@ def run_generic_updaters(config, lineage, plugins): logger.debug("Skipping updaters in dry-run mode.") return try: - # installers are used in auth mode to determine domain names - installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: + installer = plug_sel.get_unprepared_installer(config, plugins) + except errors.Error as e: logger.warning("Could not choose appropriate plugin for updaters: %s", e) return - _run_updaters(lineage, installer, config) - _run_enhancement_updaters(lineage, installer, config) + if installer: + _run_updaters(lineage, installer, config) + _run_enhancement_updaters(lineage, installer, config) def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used From dd600db436542f79363f1fbdc50e1a2f6cf0ba42 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:16:08 -0700 Subject: [PATCH 566/631] Upgrade pinned josepy version (#6184) We released josepy 1.1.0 a while ago to work around newer versions of cryptography deprecating some of the functionality we were using. We haven't yet upgraded our pinned josepy version though and since #6169 has landed, we're now seeing these deprecation warnings in our tests. This would be shown to certbot-auto users as well. This PR removes these warnings by upgrading our pinned version of josepy. * update pinned josepy version * build leauto * update pinned dev version of josepy --- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/dependency-requirements.txt | 6 +++--- tools/dev_constraints.txt | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d571a5a8d..9afa86849 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1092,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 54498cb3e..ae6079d96 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -96,9 +96,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 777222ffb..4a392e0b4 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -26,7 +26,7 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 -josepy==1.0.1 +josepy==1.1.0 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 From cdf93de33899a95a544bbf7b99ddc1c327cc2728 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:16:44 -0700 Subject: [PATCH 567/631] Full Python 3.7 support (#6182) Now that yaml/pyyaml#126 is resolved, #6170 can be reverted by bumping the pinned version of PyYAML. You can see this code passing with full macOS and integration tests at https://travis-ci.org/certbot/certbot/builds/400957729. * Revert "Allow py37 testing (#6170)" This reverts commit cad95466b05e6be51c1c29eaa91e6e3b7ea3cefd. * Bump pyyaml pinning to work on Python 3.7. --- tools/dev_constraints.txt | 2 +- tox.ini | 22 ++++------------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 4a392e0b4..5a16b8cba 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,7 +51,7 @@ pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 python-digitalocean==1.11 -PyYAML==3.12 +PyYAML==3.13 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 requests-toolbelt==0.8.0 diff --git a/tox.ini b/tox.ini index b44d30449..ef71b52be 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,8 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -python37_compatible_dns_packages = +dns_packages = + certbot-dns-cloudflare \ certbot-dns-cloudxns \ certbot-dns-digitalocean \ certbot-dns-dnsimple \ @@ -24,22 +25,14 @@ python37_compatible_dns_packages = certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 -dns_packages = - certbot-dns-cloudflare \ - {[base]python37_compatible_dns_packages} -nondns_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ + {[base]dns_packages} \ certbot-nginx \ certbot-postfix \ letshelp-certbot -python37_compatible_packages = - {[base]nondns_packages} \ - {[base]python37_compatible_dns_packages} -all_packages = - {[base]nondns_packages} \ - {[base]dns_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -69,13 +62,6 @@ commands = setenv = PYTHONHASHSEED = 0 -[testenv:py37] -commands = - {[base]install_and_test} {[base]python37_compatible_packages} - python tests/lock_test.py -setenv = - {[testenv]setenv} - [testenv:py27-oldest] commands = {[testenv]commands} From 43f2bfd6f18b9ab1375b27fc0fd9bf73be64d8ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:17:03 -0700 Subject: [PATCH 568/631] Advertise our packages work on Python 3.7. (#6183) --- acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot-postfix/setup.py | 1 + letshelp-certbot/setup.py | 1 + setup.py | 1 + 17 files changed, 17 insertions(+) diff --git a/acme/setup.py b/acme/setup.py index ecafac61a..8332febc1 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -68,6 +68,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index ffaa6a863..aca7f3bce 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bf86df5da..34e39ec5e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -46,6 +46,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 055a8cffc..9cf26aa52 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 2c0c074be..8077fe8f5 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index bd5f2ff26..5877d2d0d 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 581c478b8..8c00e6d58 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c5d310d71..34772c422 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 01d979c2b..ad56e58be 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -47,6 +47,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 44f67c0a7..796f7a489 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 87a7da4cc..d87470c3a 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c1fffc4a2..bbfbcbba1 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index c8806e862..8836dc6d8 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b486b2778..09192a956 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 4c53477d2..0ff2908df 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -40,6 +40,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Communications :: Email :: Mail Transport Agents', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 28ce0e962..3e9e31725 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -35,6 +35,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/setup.py b/setup.py index 9ef9ec0d2..5a68ff40e 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', From 83f7e72fefb8d9087a5ad488153a644e1b905572 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 10 Jul 2018 13:03:25 -0700 Subject: [PATCH 569/631] Update and delete registration with account reuse (#6174) * find the correct url when deactivating an acmev1 account on the acmev2 endpoint * set regr in ClientNetwork.account after deactivating on the server * update self.net.account * move logic into update_registration * return methods to their original order to please git * factor out common code * update test_fowarding to use a method that still gets forwarded * add acme module test coverage * pragma no cover on correct line * use previous regr uri * strip unnecessary items from regr before saving * add explanation to main.py * add extra check to client_test.py * use empty dict instead of empty string to indicate lack of body that we save to disk --- acme/acme/client.py | 26 +++++++++++++++++++++++++- acme/acme/client_test.py | 22 +++++++++++++++++++++- acme/acme/messages.py | 1 + certbot/account.py | 9 ++++++--- certbot/main.py | 5 +++++ 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index bdc07fb1c..a0bfe460d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -50,7 +50,6 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :ivar .ClientNetwork net: Client network. :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, net, acme_version): """Initialize. @@ -588,6 +587,30 @@ class ClientV2(ClientBase): self.net.account = regr return regr + def update_registration(self, regr, update=None): + """Update registration. + + :param messages.RegistrationResource regr: Registration Resource. + :param messages.Registration update: Updated body of the + resource. If not provided, body will be taken from `regr`. + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + # https://github.com/certbot/certbot/issues/6155 + new_regr = self._get_v2_account(regr) + return super(ClientV2, self).update_registration(new_regr, update) + + def _get_v2_account(self, regr): + self.net.account = None + only_existing_reg = regr.body.update(only_return_existing=True) + response = self._post(self.directory['newAccount'], only_existing_reg) + updated_uri = response.headers['Location'] + new_regr = regr.update(uri=updated_uri) + self.net.account = new_regr + return new_regr + def new_order(self, csr_pem): """Request a new Order object from the server. @@ -910,6 +933,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes if acme_version == 2: kwargs["url"] = url # newAccount and revokeCert work without the kid + # newAccount must not have kid if self.account is not None: kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index f3018ed81..cd31c4ac3 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -139,7 +139,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - self.assertEqual(client.update_registration, client.client.update_registration) + self.assertEqual(client.deactivate_registration, client.client.deactivate_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') self.assertRaises(AttributeError, client.__getattr__, 'new_account') @@ -270,6 +270,13 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.revoke(messages_test.CERT, self.rsn) mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + def test_update_registration(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.update_registration(mock.sentinel.regr, None) + mock_client().update_registration.assert_called_once_with(mock.sentinel.regr, None) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -789,6 +796,19 @@ class ClientV2Test(ClientTestBase): self.net.post.assert_called_once_with( self.directory["revokeCert"], mock.ANY, acme_version=2) + def test_update_registration(self): + # "Instance of 'Field' has no to_json/update member" bug: + # pylint: disable=no-member + self.response.headers['Location'] = self.regr.uri + self.response.json.return_value = self.regr.body.to_json() + self.assertEqual(self.regr, self.client.update_registration(self.regr)) + self.assertNotEqual(self.client.net.account, None) + self.assertEqual(self.client.net.post.call_count, 2) + self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0]) + + self.response.json.return_value = self.regr.body.update( + contact=()).to_json() + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 827a4dd11..5be458580 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -274,6 +274,7 @@ class Registration(ResourceBody): agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) + only_return_existing = jose.Field('onlyReturnExisting', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/certbot/account.py b/certbot/account.py index 5e9455048..2f261f759 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -270,9 +270,12 @@ class AccountFileStorage(interfaces.AccountStorage): if hasattr(acme.directory, "new-authz"): regr = RegistrationResourceWithNewAuthzrURI( new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) + body={}, + uri=regr.uri) + else: + regr = messages.RegistrationResource( + body={}, + uri=regr.uri) regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), diff --git a/certbot/main.py b/certbot/main.py index 556722104..2c7ded3e2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -735,8 +735,13 @@ def register(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. acc_contacts = ['mailto:' + email for email in config.email.split(',')] + prev_regr_uri = acc.regr.uri acc.regr = cb_client.acme.update_registration(acc.regr.update( body=acc.regr.body.update(contact=acc_contacts))) + # A v1 account being used as a v2 account will result in changing the uri to + # the v2 uri. Since it's the same object on disk, put it back to the v1 uri + # so that we can also continue to use the account object with acmev1. + acc.regr = acc.regr.update(uri=prev_regr_uri) account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) From 3855cfc08dacccce223eb6c0f3d127143225da2d Mon Sep 17 00:00:00 2001 From: Trinopoty Biswas Date: Wed, 11 Jul 2018 02:21:03 +0530 Subject: [PATCH 570/631] Linode DNS Authenticator (#5302) * Added DNS based authenticator plugin for Linode * Added linode plugin to docs * Added Dockerfile * Added .gitignore and readthedocs.org.requirements.txt * Updated default_propagation_seconds * Updated according to changes requested * Bump version to 0.26.0 * Advertise our packages work on Python 3.7. --- certbot-dns-linode/Dockerfile | 5 + certbot-dns-linode/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-linode/MANIFEST.in | 3 + certbot-dns-linode/README.rst | 1 + .../certbot_dns_linode/__init__.py | 86 ++++++++ .../certbot_dns_linode/dns_linode.py | 72 +++++++ .../certbot_dns_linode/dns_linode_test.py | 47 +++++ certbot-dns-linode/docs/.gitignore | 1 + certbot-dns-linode/docs/Makefile | 20 ++ certbot-dns-linode/docs/api.rst | 8 + certbot-dns-linode/docs/api/dns_linode.rst | 5 + certbot-dns-linode/docs/conf.py | 180 +++++++++++++++++ certbot-dns-linode/docs/index.rst | 28 +++ certbot-dns-linode/docs/make.bat | 36 ++++ .../local-oldest-requirements.txt | 2 + .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-linode/setup.cfg | 2 + certbot-dns-linode/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 4 +- docs/packaging.rst | 2 + docs/using.rst | 1 + tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 29 files changed, 784 insertions(+), 3 deletions(-) create mode 100644 certbot-dns-linode/Dockerfile create mode 100644 certbot-dns-linode/LICENSE.txt create mode 100644 certbot-dns-linode/MANIFEST.in create mode 100644 certbot-dns-linode/README.rst create mode 100644 certbot-dns-linode/certbot_dns_linode/__init__.py create mode 100644 certbot-dns-linode/certbot_dns_linode/dns_linode.py create mode 100644 certbot-dns-linode/certbot_dns_linode/dns_linode_test.py create mode 100644 certbot-dns-linode/docs/.gitignore create mode 100644 certbot-dns-linode/docs/Makefile create mode 100644 certbot-dns-linode/docs/api.rst create mode 100644 certbot-dns-linode/docs/api/dns_linode.rst create mode 100644 certbot-dns-linode/docs/conf.py create mode 100644 certbot-dns-linode/docs/index.rst create mode 100644 certbot-dns-linode/docs/make.bat create mode 100644 certbot-dns-linode/local-oldest-requirements.txt create mode 100644 certbot-dns-linode/readthedocs.org.requirements.txt create mode 100644 certbot-dns-linode/setup.cfg create mode 100644 certbot-dns-linode/setup.py diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile new file mode 100644 index 000000000..2e237b521 --- /dev/null +++ b/certbot-dns-linode/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-linode + +RUN pip install --no-cache-dir --editable src/certbot-dns-linode diff --git a/certbot-dns-linode/LICENSE.txt b/certbot-dns-linode/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-linode/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-linode/MANIFEST.in b/certbot-dns-linode/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-linode/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-linode/README.rst b/certbot-dns-linode/README.rst new file mode 100644 index 000000000..69e1fa056 --- /dev/null +++ b/certbot-dns-linode/README.rst @@ -0,0 +1 @@ +Linode DNS Authenticator plugin for Certbot diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py new file mode 100644 index 000000000..aaed61450 --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -0,0 +1,86 @@ +""" +The `~certbot_dns_linode.dns_linode` plugin automates the process of +completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and +subsequently removing, TXT records using the Linode API. + + +Named Arguments +--------------- + +========================================== =================================== +``--dns-linode-credentials`` Linode credentials_ INI file. + (Required) +``--dns-linode-propagation-seconds`` The number of seconds to wait for + DNS to propagate before asking the + ACME server to verify the DNS + record. + (Default: 960) +========================================== =================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing Linode API +credentials, obtained from your Linode account's `Applications & API +Tokens page `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Linode API credentials used by Certbot + dns_linode_key = 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ64 + +The path to this file can be provided interactively or using the +``--dns-linode-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Linode account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire + new certificates or revoke existing certificates for associated domains, + even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + --dns-linode-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py new file mode 100644 index 000000000..323c0810a --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -0,0 +1,72 @@ +"""DNS Authenticator for Linode.""" +import logging + +import zope.interface +from lexicon.providers import linode + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +API_KEY_URL = 'https://manager.linode.com/profile/api' + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Linode + + This Authenticator uses the Linode API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using Linode for DNS).' + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960) + add('credentials', help='Linode credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Linode API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Linode credentials INI file', + { + 'key': 'API key for Linode account, obtained from {0}'.format(API_KEY_URL) + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_linode_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_linode_client().del_txt_record(domain, validation_name, validation) + + def _get_linode_client(self): + return _LinodeLexiconClient(self.credentials.conf('key')) + +class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Linode API. + """ + + def __init__(self, api_key): + super(_LinodeLexiconClient, self).__init__() + self.provider = linode.Provider({ + 'auth_token': api_key + }) + + def _handle_general_error(self, e, domain_name): + if not str(e).startswith('Domain not found'): + return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' + .format(domain_name, e)) + diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py b/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py new file mode 100644 index 000000000..2a0ee49f7 --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py @@ -0,0 +1,47 @@ +"""Tests for certbot_dns_linode.dns_linode.""" + +import os +import unittest + +import mock + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +TOKEN = 'a-token' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_linode.dns_linode import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"linode_key": TOKEN}, path) + + self.config = mock.MagicMock(linode_credentials=path, + linode_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "linode") + + self.mock_client = mock.MagicMock() + # _get_linode_client | pylint: disable=protected-access + self.auth._get_linode_client = mock.MagicMock(return_value=self.mock_client) + +class LinodeLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + + DOMAIN_NOT_FOUND = Exception('Domain not found') + + def setUp(self): + from certbot_dns_linode.dns_linode import _LinodeLexiconClient + + self.client = _LinodeLexiconClient(TOKEN) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-linode/docs/.gitignore b/certbot-dns-linode/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-linode/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-linode/docs/Makefile b/certbot-dns-linode/docs/Makefile new file mode 100644 index 000000000..bcfbfd5b1 --- /dev/null +++ b/certbot-dns-linode/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-linode +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/certbot-dns-linode/docs/api.rst b/certbot-dns-linode/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-linode/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-linode/docs/api/dns_linode.rst b/certbot-dns-linode/docs/api/dns_linode.rst new file mode 100644 index 000000000..6380b3eba --- /dev/null +++ b/certbot-dns-linode/docs/api/dns_linode.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_linode.dns_linode` +------------------------------------------------ + +.. automodule:: certbot_dns_linode.dns_linode + :members: diff --git a/certbot-dns-linode/docs/conf.py b/certbot-dns-linode/docs/conf.py new file mode 100644 index 000000000..1fb721400 --- /dev/null +++ b/certbot-dns-linode/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-linode documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 10:52:06 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-linode' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-linodedoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-linode.tex', u'certbot-dns-linode Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-linode', u'certbot-dns-linode Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-linode', u'certbot-dns-linode Documentation', + author, 'certbot-dns-linode', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-linode/docs/index.rst b/certbot-dns-linode/docs/index.rst new file mode 100644 index 000000000..dd430554b --- /dev/null +++ b/certbot-dns-linode/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-linode documentation master file, created by + sphinx-quickstart on Wed May 10 10:52:06 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-linode's documentation! +==================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_linode + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-linode/docs/make.bat b/certbot-dns-linode/docs/make.bat new file mode 100644 index 000000000..1f2a6867f --- /dev/null +++ b/certbot-dns-linode/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-linode + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-linode/readthedocs.org.requirements.txt b/certbot-dns-linode/readthedocs.org.requirements.txt new file mode 100644 index 000000000..47449454f --- /dev/null +++ b/certbot-dns-linode/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-linode[docs] diff --git a/certbot-dns-linode/setup.cfg b/certbot-dns-linode/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-linode/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py new file mode 100644 index 000000000..2abd19b68 --- /dev/null +++ b/certbot-dns-linode/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + +version = '0.26.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.2.1', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-linode', + version=version, + description="Linode DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-linode = certbot_dns_linode.dns_linode:Authenticator', + ], + }, + test_suite='certbot_dns_linode', +) diff --git a/certbot/cli.py b/certbot/cli.py index 5c4313ea4..244daf546 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1419,6 +1419,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_google"), help=("Obtain certificates using a DNS TXT record (if you are " "using Google Cloud DNS).")) + helpful.add(["plugins", "certonly"], "--dns-linode", action="store_true", + default=flag_default("dns_linode"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using Linode for DNS).")) helpful.add(["plugins", "certonly"], "--dns-luadns", action="store_true", default=flag_default("dns_luadns"), help=("Obtain certificates using a DNS TXT record (if you are " diff --git a/certbot/constants.py b/certbot/constants.py index d31faa71c..750db83b7 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -105,6 +105,7 @@ CLI_DEFAULTS = dict( dns_dnsimple=False, dns_dnsmadeeasy=False, dns_google=False, + dns_linode=False, dns_luadns=False, dns_nsone=False, dns_rfc2136=False, diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index bf9f60e3b..8672ba0ab 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -31,6 +31,7 @@ class PluginEntryPoint(object): "certbot-dns-dnsimple", "certbot-dns-dnsmadeeasy", "certbot-dns-google", + "certbot-dns-linode", "certbot-dns-luadns", "certbot-dns-nsone", "certbot-dns-rfc2136", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 95f123a46..ef98e9cd8 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,7 +165,7 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] + "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -290,6 +290,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-dnsmadeeasy") if config.dns_google: req_auth = set_configurator(req_auth, "dns-google") + if config.dns_linode: + req_auth = set_configurator(req_auth, "dns-linode") if config.dns_luadns: req_auth = set_configurator(req_auth, "dns-luadns") if config.dns_nsone: diff --git a/docs/packaging.rst b/docs/packaging.rst index 3d58ea92e..dc75e34b0 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -17,6 +17,7 @@ We release packages and upload them to PyPI (wheels and source tarballs). - https://pypi.python.org/pypi/certbot-dns-dnsimple - https://pypi.python.org/pypi/certbot-dns-dnsmadeeasy - https://pypi.python.org/pypi/certbot-dns-google +- https://pypi.python.org/pypi/certbot-dns-linode - https://pypi.python.org/pypi/certbot-dns-luadns - https://pypi.python.org/pypi/certbot-dns-nsone - https://pypi.python.org/pypi/certbot-dns-rfc2136 @@ -63,6 +64,7 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-dnsimple - https://www.archlinux.org/packages/community/any/certbot-dns-dnsmadeeasy - https://www.archlinux.org/packages/community/any/certbot-dns-google +- https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 diff --git a/docs/using.rst b/docs/using.rst index 46599a06e..50c27d45e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -203,6 +203,7 @@ Once installed, you can find documentation on how to use each plugin at: * `certbot-dns-dnsimple `_ * `certbot-dns-dnsmadeeasy `_ * `certbot-dns-google `_ +* `certbot-dns-linode `_ * `certbot-dns-luadns `_ * `certbot-dns-nsone `_ * `certbot-dns-rfc2136 `_ diff --git a/tools/release.sh b/tools/release.sh index 02c7ccca5..1286a7b45 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index a623ec529..177bb2714 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -20,6 +20,7 @@ fi -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ -e certbot-dns-google \ + -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-rfc2136 \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 602118004..f18f05374 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -19,6 +19,7 @@ fi -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ -e certbot-dns-google \ + -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-route53 \ diff --git a/tox.cover.sh b/tox.cover.sh index 4167699d6..421771d08 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -33,6 +33,8 @@ cover () { min=99 elif [ "$1" = "certbot_dns_google" ]; then min=99 + elif [ "$1" = "certbot_dns_linode" ]; then + min=98 elif [ "$1" = "certbot_dns_luadns" ]; then min=98 elif [ "$1" = "certbot_dns_nsone" ]; then diff --git a/tox.ini b/tox.ini index ef71b52be..c05627fc1 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ dns_packages = certbot-dns-dnsimple \ certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-linode \ certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ @@ -46,6 +47,7 @@ source_paths = certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-google/certbot_dns_google + certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-rfc2136/certbot_dns_rfc2136 From 0672e63176f51b30b5faaa089e425bbe4aa28dd9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 10 Jul 2018 13:52:58 -0700 Subject: [PATCH 571/631] Remove main components from Alpha. (#6187) acme, certbot, and the Nginx and Apache plugins should no longer be considered alpha-quality. --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot-nginx/setup.py | 2 +- docs/cli-help.txt | 2 +- setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 8332febc1..e6cfcafc6 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -58,7 +58,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aca7f3bce..cbd434f01 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -31,7 +31,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index b80d95613..d31a54c17 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -60,7 +60,7 @@ class NginxConfigurator(common.Installer): """ - description = "Nginx Web Server plugin - Alpha" + description = "Nginx Web Server plugin" DEFAULT_LISTEN_PORT = '80' diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 09192a956..b43684feb 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -31,7 +31,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 594bcfd04..7b8b522c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -635,7 +635,7 @@ manual: Automatically allows public IP logging (default: Ask) nginx: - Nginx Web Server plugin - Alpha + Nginx Web Server plugin --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx or diff --git a/setup.py b/setup.py index 5a68ff40e..fc87917fb 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Environment :: Console :: Curses', 'Intended Audience :: System Administrators', From 8a5abb620338cd804bd7b09f52865c0e2e7658d3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 14:06:45 -0700 Subject: [PATCH 572/631] Always save server value in renewal conf file (#6189) --- certbot/storage.py | 7 ++++++- certbot/tests/storage_test.py | 22 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 5b2293bd1..32d6771c2 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -239,10 +239,15 @@ def relevant_values(all_values): :rtype dict: """ - return dict( + rv = dict( (option, value) for option, value in six.iteritems(all_values) if _relevant(option) and cli.option_was_set(option, value)) + # We always save the server value to help with forward compatibility + # and behavioral consistency when versions of Certbot with different + # server defaults are used. + rv["server"] = all_values["server"] + return rv def lineagename_for_filename(config_filename): """Returns the lineagename for a configuration filename. diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index aa6c52ad4..db6aec1de 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -544,14 +544,22 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(os.path.exists(temp_config_file)) def _test_relevant_values_common(self, values): - option = "rsa_key_size" + defaults = dict((option, cli.flag_default(option)) + for option in ("rsa_key_size", "server",)) mock_parser = mock.Mock(args=["--standalone"], verb="certonly", - defaults={option: cli.flag_default(option)}) + defaults=defaults) + + # make a copy to ensure values isn't modified + values = values.copy() + values.setdefault("server", defaults["server"]) + expected_server = values["server"] from certbot.storage import relevant_values with mock.patch("certbot.cli.helpful_parser", mock_parser): - # make a copy to ensure values isn't modified - return relevant_values(values.copy()) + rv = relevant_values(values) + self.assertIn("server", rv) + self.assertEqual(rv.pop("server"), expected_server) + return rv def test_relevant_values(self): """Test that relevant_values() can reject an irrelevant value.""" @@ -589,6 +597,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual( self._test_relevant_values_common(values), values) + def test_relevant_values_server(self): + self.assertEqual( + # _test_relevant_values_common handles testing the server + # value and removes it + self._test_relevant_values_common({"server": "example.org"}), {}) + @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" From 9314911135ad2ef36dd561eafe4a5e23c4c422ad Mon Sep 17 00:00:00 2001 From: chibiegg Date: Wed, 11 Jul 2018 06:30:37 +0900 Subject: [PATCH 573/631] Sakura Cloud DNS Authenticator (#5701) Implement an Authenticator which can fulfill a dns-01 challenge using the Sakura Cloud DNS API. Applicable only for domains using Sakura Cloud for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-sakuracloud -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to Sakura Cloud account. --- certbot-dns-sakuracloud/Dockerfile | 5 + certbot-dns-sakuracloud/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-sakuracloud/MANIFEST.in | 3 + certbot-dns-sakuracloud/README.rst | 1 + .../certbot_dns_sakuracloud/__init__.py | 86 ++++++++ .../dns_sakuracloud.py | 87 ++++++++ .../dns_sakuracloud_test.py | 55 +++++ certbot-dns-sakuracloud/docs/.gitignore | 1 + certbot-dns-sakuracloud/docs/Makefile | 20 ++ certbot-dns-sakuracloud/docs/api.rst | 8 + .../docs/api/dns_sakuracloud.rst | 5 + certbot-dns-sakuracloud/docs/conf.py | 180 +++++++++++++++++ certbot-dns-sakuracloud/docs/index.rst | 28 +++ certbot-dns-sakuracloud/docs/make.bat | 36 ++++ .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-sakuracloud/setup.cfg | 2 + certbot-dns-sakuracloud/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 3 +- certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 5 +- tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 4 +- 26 files changed, 805 insertions(+), 5 deletions(-) create mode 100644 certbot-dns-sakuracloud/Dockerfile create mode 100644 certbot-dns-sakuracloud/LICENSE.txt create mode 100644 certbot-dns-sakuracloud/MANIFEST.in create mode 100644 certbot-dns-sakuracloud/README.rst create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py create mode 100644 certbot-dns-sakuracloud/docs/.gitignore create mode 100644 certbot-dns-sakuracloud/docs/Makefile create mode 100644 certbot-dns-sakuracloud/docs/api.rst create mode 100644 certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst create mode 100644 certbot-dns-sakuracloud/docs/conf.py create mode 100644 certbot-dns-sakuracloud/docs/index.rst create mode 100644 certbot-dns-sakuracloud/docs/make.bat create mode 100644 certbot-dns-sakuracloud/readthedocs.org.requirements.txt create mode 100644 certbot-dns-sakuracloud/setup.cfg create mode 100644 certbot-dns-sakuracloud/setup.py diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile new file mode 100644 index 000000000..694773f61 --- /dev/null +++ b/certbot-dns-sakuracloud/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-sakuracloud + +RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud diff --git a/certbot-dns-sakuracloud/LICENSE.txt b/certbot-dns-sakuracloud/LICENSE.txt new file mode 100644 index 000000000..8316b6a0e --- /dev/null +++ b/certbot-dns-sakuracloud/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2018 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-sakuracloud/MANIFEST.in b/certbot-dns-sakuracloud/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-sakuracloud/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-sakuracloud/README.rst b/certbot-dns-sakuracloud/README.rst new file mode 100644 index 000000000..46a082b9c --- /dev/null +++ b/certbot-dns-sakuracloud/README.rst @@ -0,0 +1 @@ +Sakura Cloud DNS Authenticator plugin for Certbot diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py new file mode 100644 index 000000000..f18780c18 --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py @@ -0,0 +1,86 @@ +""" +The `~certbot_dns_sakuracloud.dns_sakuracloud` plugin automates the process of completing +a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently +removing, TXT records using the Sakura Cloud DNS API. + + +Named Arguments +--------------- + +========================================== ====================================== +``--dns-sakuracloud-credentials`` Sakura Cloud credentials_ INI file. + (Required) +``--dns-sakuracloud-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 90) +========================================== ====================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing +Sakura Cloud DNS API credentials, obtained from your Sakura Cloud DNS +`apikey page `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Sakura Cloud API credentials used by Certbot + dns_sakuracloud_api_token = 00000000-0000-0000-0000-000000000000 + dns_sakuracloud_api_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-sakuracloud-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Sakura Cloud account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire new + certificates or revoke existing certificates for associated domains, even if + those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + --dns-sakuracloud-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py new file mode 100644 index 000000000..6f1c74b68 --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -0,0 +1,87 @@ +"""DNS Authenticator for Sakura Cloud DNS.""" +import logging + +import zope.interface +from lexicon.providers import sakuracloud + +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +APIKEY_URL = "https://secure.sakura.ad.jp/cloud/#!/apikey/top/" + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Sakura Cloud DNS + + This Authenticator uses the Sakura Cloud API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record ' + \ + '(if you are using Sakura Cloud for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments( + add, default_propagation_seconds=90) + add('credentials', help='Sakura Cloud credentials file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Sakura Cloud API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Sakura Cloud credentials file', + { + 'api-token': \ + 'API token for Sakura Cloud API obtained from {0}'.format(APIKEY_URL), + 'api-secret': \ + 'API secret for Sakura Cloud API obtained from {0}'.format(APIKEY_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_sakuracloud_client().add_txt_record( + domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_sakuracloud_client().del_txt_record( + domain, validation_name, validation) + + def _get_sakuracloud_client(self): + return _SakuraCloudLexiconClient( + self.credentials.conf('api-token'), + self.credentials.conf('api-secret'), + self.ttl + ) + + +class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Sakura Cloud via Lexicon. + """ + + def __init__(self, api_token, api_secret, ttl): + super(_SakuraCloudLexiconClient, self).__init__() + + self.provider = sakuracloud.Provider({ + 'auth_token': api_token, + 'auth_secret': api_secret, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): + return # Expected errors when zone name guess is wrong + return super(_SakuraCloudLexiconClient, self)._handle_http_error(e, domain_name) diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py new file mode 100644 index 000000000..84605d06f --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py @@ -0,0 +1,55 @@ +"""Tests for certbot_dns_sakuracloud.dns_sakuracloud.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_TOKEN = '00000000-0000-0000-0000-000000000000' +API_SECRET = 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_sakuracloud.dns_sakuracloud import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write( + {"sakuracloud_api_token": API_TOKEN, "sakuracloud_api_secret": API_SECRET}, + path + ) + + self.config = mock.MagicMock(sakuracloud_credentials=path, + sakuracloud_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "sakuracloud") + + self.mock_client = mock.MagicMock() + # _get_sakuracloud_client | pylint: disable=protected-access + self.auth._get_sakuracloud_client = mock.MagicMock(return_value=self.mock_client) + + +class NS1LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) + LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) + + def setUp(self): + from certbot_dns_sakuracloud.dns_sakuracloud import _SakuraCloudLexiconClient + + self.client = _SakuraCloudLexiconClient(API_TOKEN, API_SECRET, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-sakuracloud/docs/.gitignore b/certbot-dns-sakuracloud/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-sakuracloud/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-sakuracloud/docs/Makefile b/certbot-dns-sakuracloud/docs/Makefile new file mode 100644 index 000000000..c2969dd98 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-sakuracloud +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-sakuracloud/docs/api.rst b/certbot-dns-sakuracloud/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst b/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst new file mode 100644 index 000000000..74692e15b --- /dev/null +++ b/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_sakuracloud.dns_sakuracloud` +---------------------------------------------- + +.. automodule:: certbot_dns_sakuracloud.dns_sakuracloud + :members: diff --git a/certbot-dns-sakuracloud/docs/conf.py b/certbot-dns-sakuracloud/docs/conf.py new file mode 100644 index 000000000..e14fe1d4c --- /dev/null +++ b/certbot-dns-sakuracloud/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-sakuracloud documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 18:30:40 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-sakuracloud' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-sakuraclouddoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-sakuracloud.tex', u'certbot-dns-sakuracloud Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-sakuracloud', u'certbot-dns-sakuracloud Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-sakuracloud', u'certbot-dns-sakuracloud Documentation', + author, 'certbot-dns-sakuracloud', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-sakuracloud/docs/index.rst b/certbot-dns-sakuracloud/docs/index.rst new file mode 100644 index 000000000..715028591 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-sakuracloud documentation master file, created by + sphinx-quickstart on Wed May 10 18:30:40 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-sakuracloud's documentation! +=================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_sakuracloud + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-sakuracloud/docs/make.bat b/certbot-dns-sakuracloud/docs/make.bat new file mode 100644 index 000000000..0d7706bc7 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-sakuracloud + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-sakuracloud/readthedocs.org.requirements.txt b/certbot-dns-sakuracloud/readthedocs.org.requirements.txt new file mode 100644 index 000000000..3f46d95ef --- /dev/null +++ b/certbot-dns-sakuracloud/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-sakuracloud[docs] diff --git a/certbot-dns-sakuracloud/setup.cfg b/certbot-dns-sakuracloud/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-sakuracloud/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py new file mode 100644 index 000000000..dd8903fdd --- /dev/null +++ b/certbot-dns-sakuracloud/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.1.23', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-sakuracloud', + version=version, + description="Sakura Cloud DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-sakuracloud = certbot_dns_sakuracloud.dns_sakuracloud:Authenticator', + ], + }, + test_suite='certbot_dns_sakuracloud', +) diff --git a/certbot/cli.py b/certbot/cli.py index 244daf546..c5199dc25 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1438,6 +1438,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_route53"), help=("Obtain certificates using a DNS TXT record (if you are using Route53 for " "DNS).")) + helpful.add(["plugins", "certonly"], "--dns-sakuracloud", action="store_true", + default=flag_default("dns_sakuracloud"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Sakura Cloud for DNS).")) # things should not be reorder past/pre this comment: # plugins_group should be displayed in --help before plugin diff --git a/certbot/constants.py b/certbot/constants.py index 750db83b7..2a752beba 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -109,7 +109,8 @@ CLI_DEFAULTS = dict( dns_luadns=False, dns_nsone=False, dns_rfc2136=False, - dns_route53=False + dns_route53=False, + dns_sakuracloud=False ) STAGING_URI = "https://acme-staging-v02.api.letsencrypt.org/directory" diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 8672ba0ab..d0a20c3ac 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -36,6 +36,7 @@ class PluginEntryPoint(object): "certbot-dns-nsone", "certbot-dns-rfc2136", "certbot-dns-route53", + "certbot-dns-sakuracloud", "certbot-nginx", "certbot-postfix", ] diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index ef98e9cd8..ec0dfbb19 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,7 +165,8 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] + "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53", + "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -300,6 +301,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-rfc2136") if config.dns_route53: req_auth = set_configurator(req_auth, "dns-route53") + if config.dns_sakuracloud: + req_auth = set_configurator(req_auth, "dns-sakuracloud") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) return req_auth, req_inst diff --git a/tools/release.sh b/tools/release.sh index 1286a7b45..fdd810497 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 177bb2714..1610d37d3 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -25,6 +25,7 @@ fi -e certbot-dns-nsone \ -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ + -e certbot-dns-sakuracloud \ -e certbot-nginx \ -e certbot-postfix \ -e letshelp-certbot \ diff --git a/tools/venv3.sh b/tools/venv3.sh index f18f05374..1cf5554b1 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,6 +23,7 @@ fi -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-route53 \ + -e certbot-dns-sakuracloud \ -e certbot-nginx \ -e certbot-postfix \ -e letshelp-certbot \ diff --git a/tox.cover.sh b/tox.cover.sh index 421771d08..8c9766d75 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -43,6 +43,8 @@ cover () { min=99 elif [ "$1" = "certbot_dns_route53" ]; then min=92 + elif [ "$1" = "certbot_dns_sakuracloud" ]; then + min=97 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "certbot_postfix" ]; then diff --git a/tox.ini b/tox.ini index c05627fc1..9373b3aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,8 @@ dns_packages = certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ - certbot-dns-route53 + certbot-dns-route53 \ + certbot-dns-sakuracloud all_packages = acme[dev] \ .[dev] \ @@ -52,6 +53,7 @@ source_paths = certbot-dns-nsone/certbot_dns_nsone certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 + certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx certbot-postfix/certbot_postfix letshelp-certbot/letshelp_certbot From 3b39266813e560448c5ea42c9799ee02de0072b0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 14:42:27 -0700 Subject: [PATCH 574/631] Don't use quoted None for plugins in the config (#6195) This stops us from printing messages like: "Could not choose appropriate plugin for updaters: Could not select or initialize the requested installer None." when certbot renew --force-renewal is run with a lineage that doesn't have an installer. * unquote None * Test None values aren't saved in config file. --- certbot/main.py | 2 +- certbot/plugins/selection.py | 4 ++-- certbot/tests/storage_test.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 2c7ded3e2..2cba8745b 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config """ # For user-agent construction - config.installer = config.authenticator = "None" + config.installer = config.authenticator = None if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index ec0dfbb19..2fd84986e 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -170,8 +170,8 @@ noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dn def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." - config.authenticator = plugins.find_init(auth).name if auth else "None" - config.installer = plugins.find_init(inst).name if inst else "None" + config.authenticator = plugins.find_init(auth).name if auth else None + config.installer = plugins.find_init(inst).name if inst else None logger.info("Plugins selected: Authenticator %s, Installer %s", config.authenticator, config.installer) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index db6aec1de..53a976f8d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -545,8 +545,9 @@ class RenewableCertTests(BaseRenewableCertTest): def _test_relevant_values_common(self, values): defaults = dict((option, cli.flag_default(option)) - for option in ("rsa_key_size", "server",)) - mock_parser = mock.Mock(args=["--standalone"], verb="certonly", + for option in ("authenticator", "installer", + "rsa_key_size", "server",)) + mock_parser = mock.Mock(args=[], verb="plugins", defaults=defaults) # make a copy to ensure values isn't modified @@ -588,6 +589,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual( self._test_relevant_values_common(values), values) + def test_relevant_values_plugins_none(self): + self.assertEqual( + self._test_relevant_values_common( + {"authenticator": None, "installer": None}), {}) + @mock.patch("certbot.cli.set_by_cli") @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): From 3f6a90882113fc5954106dd29936f6699487ba0a Mon Sep 17 00:00:00 2001 From: chibiegg Date: Wed, 11 Jul 2018 09:36:20 +0900 Subject: [PATCH 575/631] Gehirn Infrastracture Service DNS Authenticator (#5702) Implement an Authenticator which can fulfill a dns-01 challenge using the Gehirn DNS (Gehirn Infrastructure Service) API. Applicable only for domains using Gehirn DNS for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-gehirn -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to Gehirn DNS account. --- certbot-dns-gehirn/Dockerfile | 5 + certbot-dns-gehirn/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-gehirn/MANIFEST.in | 3 + certbot-dns-gehirn/README.rst | 1 + .../certbot_dns_gehirn/__init__.py | 88 ++++++++ .../certbot_dns_gehirn/dns_gehirn.py | 84 ++++++++ .../certbot_dns_gehirn/dns_gehirn_test.py | 55 +++++ certbot-dns-gehirn/docs/.gitignore | 1 + certbot-dns-gehirn/docs/Makefile | 20 ++ certbot-dns-gehirn/docs/api.rst | 8 + certbot-dns-gehirn/docs/api/dns_gehirn.rst | 5 + certbot-dns-gehirn/docs/conf.py | 180 +++++++++++++++++ certbot-dns-gehirn/docs/index.rst | 28 +++ certbot-dns-gehirn/docs/make.bat | 36 ++++ .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-gehirn/setup.cfg | 2 + certbot-dns-gehirn/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 8 +- tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 26 files changed, 803 insertions(+), 5 deletions(-) create mode 100644 certbot-dns-gehirn/Dockerfile create mode 100644 certbot-dns-gehirn/LICENSE.txt create mode 100644 certbot-dns-gehirn/MANIFEST.in create mode 100644 certbot-dns-gehirn/README.rst create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/__init__.py create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py create mode 100644 certbot-dns-gehirn/docs/.gitignore create mode 100644 certbot-dns-gehirn/docs/Makefile create mode 100644 certbot-dns-gehirn/docs/api.rst create mode 100644 certbot-dns-gehirn/docs/api/dns_gehirn.rst create mode 100644 certbot-dns-gehirn/docs/conf.py create mode 100644 certbot-dns-gehirn/docs/index.rst create mode 100644 certbot-dns-gehirn/docs/make.bat create mode 100644 certbot-dns-gehirn/readthedocs.org.requirements.txt create mode 100644 certbot-dns-gehirn/setup.cfg create mode 100644 certbot-dns-gehirn/setup.py diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile new file mode 100644 index 000000000..48ad902b5 --- /dev/null +++ b/certbot-dns-gehirn/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-gehirn + +RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn diff --git a/certbot-dns-gehirn/LICENSE.txt b/certbot-dns-gehirn/LICENSE.txt new file mode 100644 index 000000000..8316b6a0e --- /dev/null +++ b/certbot-dns-gehirn/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2018 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-gehirn/MANIFEST.in b/certbot-dns-gehirn/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-gehirn/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-gehirn/README.rst b/certbot-dns-gehirn/README.rst new file mode 100644 index 000000000..16058eff8 --- /dev/null +++ b/certbot-dns-gehirn/README.rst @@ -0,0 +1 @@ +Gehirn Infrastracture Service DNS Authenticator plugin for Certbot diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py new file mode 100644 index 000000000..db54154ac --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py @@ -0,0 +1,88 @@ +""" +The `~certbot_dns_gehirn.dns_gehirn` plugin automates the process of completing +a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently +removing, TXT records using the Gehirn Infrastracture Service DNS API. + + +Named Arguments +--------------- + +======================================== ===================================== +``--dns-gehirn-credentials`` Gehirn Infrastracture Service + credentials_ INI file. + (Required) +``--dns-gehirn-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 30) +======================================== ===================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing +Gehirn Infrastracture Service DNS API credentials, +obtained from your Gehirn Infrastracture Service +`dashboard `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Gehirn Infrastracture Service API credentials used by Certbot + dns_gehirn_api_token = 00000000-0000-0000-0000-000000000000 + dns_gehirn_api_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-gehirn-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Gehirn Infrastracture Service account. Users who can read this file can use + these credentials to issue arbitrary API calls on your behalf. Users who can + cause Certbot to run using these credentials can complete a ``dns-01`` + challenge to acquire new certificates or revoke existing certificates for + associated domains, even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + --dns-gehirn-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py new file mode 100644 index 000000000..50bfce1ae --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -0,0 +1,84 @@ +"""DNS Authenticator for Gehirn Infrastracture Service DNS.""" +import logging + +import zope.interface +from lexicon.providers import gehirn + +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +DASHBOARD_URL = "https://gis.gehirn.jp/" + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Gehirn Infrastracture Service DNS + + This Authenticator uses the Gehirn Infrastracture Service API to fulfill + a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record ' + \ + '(if you are using Gehirn Infrastracture Service for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='Gehirn Infrastracture Service credentials file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Gehirn Infrastracture Service API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Gehirn Infrastracture Service credentials file', + { + 'api-token': 'API token for Gehirn Infrastracture Service ' + \ + 'API obtained from {0}'.format(DASHBOARD_URL), + 'api-secret': 'API secret for Gehirn Infrastracture Service ' + \ + 'API obtained from {0}'.format(DASHBOARD_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_gehirn_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_gehirn_client().del_txt_record(domain, validation_name, validation) + + def _get_gehirn_client(self): + return _GehirnLexiconClient( + self.credentials.conf('api-token'), + self.credentials.conf('api-secret'), + self.ttl + ) + + +class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Gehirn Infrastracture Service via Lexicon. + """ + + def __init__(self, api_token, api_secret, ttl): + super(_GehirnLexiconClient, self).__init__() + + self.provider = gehirn.Provider({ + 'auth_token': api_token, + 'auth_secret': api_secret, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): + return # Expected errors when zone name guess is wrong + return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name) diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py new file mode 100644 index 000000000..b771c103e --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py @@ -0,0 +1,55 @@ +"""Tests for certbot_dns_gehirn.dns_gehirn.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_TOKEN = '00000000-0000-0000-0000-000000000000' +API_SECRET = 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_gehirn.dns_gehirn import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write( + {"gehirn_api_token": API_TOKEN, "gehirn_api_secret": API_SECRET}, + path + ) + + self.config = mock.MagicMock(gehirn_credentials=path, + gehirn_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "gehirn") + + self.mock_client = mock.MagicMock() + # _get_gehirn_client | pylint: disable=protected-access + self.auth._get_gehirn_client = mock.MagicMock(return_value=self.mock_client) + + +class GehirnLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) + LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) + + def setUp(self): + from certbot_dns_gehirn.dns_gehirn import _GehirnLexiconClient + + self.client = _GehirnLexiconClient(API_TOKEN, API_SECRET, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-gehirn/docs/.gitignore b/certbot-dns-gehirn/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-gehirn/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-gehirn/docs/Makefile b/certbot-dns-gehirn/docs/Makefile new file mode 100644 index 000000000..a363d1b47 --- /dev/null +++ b/certbot-dns-gehirn/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-gehirn +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-gehirn/docs/api.rst b/certbot-dns-gehirn/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-gehirn/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-gehirn/docs/api/dns_gehirn.rst b/certbot-dns-gehirn/docs/api/dns_gehirn.rst new file mode 100644 index 000000000..35a13e9c1 --- /dev/null +++ b/certbot-dns-gehirn/docs/api/dns_gehirn.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_gehirn.dns_gehirn` +------------------------------------ + +.. automodule:: certbot_dns_gehirn.dns_gehirn + :members: diff --git a/certbot-dns-gehirn/docs/conf.py b/certbot-dns-gehirn/docs/conf.py new file mode 100644 index 000000000..a1b2799fb --- /dev/null +++ b/certbot-dns-gehirn/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-gehirn documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 18:30:40 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-gehirn' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-gehirndoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-gehirn.tex', u'certbot-dns-gehirn Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation', + author, 'certbot-dns-gehirn', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-gehirn/docs/index.rst b/certbot-dns-gehirn/docs/index.rst new file mode 100644 index 000000000..77546fa89 --- /dev/null +++ b/certbot-dns-gehirn/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-gehirn documentation master file, created by + sphinx-quickstart on Wed May 10 18:30:40 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-gehirn's documentation! +============================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_gehirn + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-gehirn/docs/make.bat b/certbot-dns-gehirn/docs/make.bat new file mode 100644 index 000000000..905d4ee90 --- /dev/null +++ b/certbot-dns-gehirn/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-gehirn + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-gehirn/readthedocs.org.requirements.txt b/certbot-dns-gehirn/readthedocs.org.requirements.txt new file mode 100644 index 000000000..d9f4f9823 --- /dev/null +++ b/certbot-dns-gehirn/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-gehirn[docs] diff --git a/certbot-dns-gehirn/setup.cfg b/certbot-dns-gehirn/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-gehirn/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py new file mode 100644 index 000000000..cc47da327 --- /dev/null +++ b/certbot-dns-gehirn/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.1.22', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-gehirn', + version=version, + description="Gehirn Infrastracture Service DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-gehirn = certbot_dns_gehirn.dns_gehirn:Authenticator', + ], + }, + test_suite='certbot_dns_gehirn', +) diff --git a/certbot/cli.py b/certbot/cli.py index c5199dc25..6d262ed72 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1415,6 +1415,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_dnsmadeeasy"), help=("Obtain certificates using a DNS TXT record (if you are" "using DNS Made Easy for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-gehirn", action="store_true", + default=flag_default("dns_gehirn"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Gehirn Infrastracture Service for DNS).")) helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", default=flag_default("dns_google"), help=("Obtain certificates using a DNS TXT record (if you are " diff --git a/certbot/constants.py b/certbot/constants.py index 2a752beba..70249b89b 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -104,6 +104,7 @@ CLI_DEFAULTS = dict( dns_digitalocean=False, dns_dnsimple=False, dns_dnsmadeeasy=False, + dns_gehirn=False, dns_google=False, dns_linode=False, dns_luadns=False, diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index d0a20c3ac..6ed0cf7b7 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -30,6 +30,7 @@ class PluginEntryPoint(object): "certbot-dns-digitalocean", "certbot-dns-dnsimple", "certbot-dns-dnsmadeeasy", + "certbot-dns-gehirn", "certbot-dns-google", "certbot-dns-linode", "certbot-dns-luadns", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 2fd84986e..0073c99fe 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -164,9 +164,9 @@ def choose_plugin(prepared, question): return None noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", - "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53", - "dns-sakuracloud"] + "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-gehirn", + "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", + "dns-route53", "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -289,6 +289,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-dnsimple") if config.dns_dnsmadeeasy: req_auth = set_configurator(req_auth, "dns-dnsmadeeasy") + if config.dns_gehirn: + req_auth = set_configurator(req_auth, "dns-gehirn") if config.dns_google: req_auth = set_configurator(req_auth, "dns-google") if config.dns_linode: diff --git a/tools/release.sh b/tools/release.sh index fdd810497..0d42bc22a 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 1610d37d3..159fc16fb 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -19,6 +19,7 @@ fi -e certbot-dns-digitalocean \ -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ + -e certbot-dns-gehirn \ -e certbot-dns-google \ -e certbot-dns-linode \ -e certbot-dns-luadns \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 1cf5554b1..a1489df22 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -18,6 +18,7 @@ fi -e certbot-dns-digitalocean \ -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ + -e certbot-dns-gehirn \ -e certbot-dns-google \ -e certbot-dns-linode \ -e certbot-dns-luadns \ diff --git a/tox.cover.sh b/tox.cover.sh index 8c9766d75..cccaf6103 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -31,6 +31,8 @@ cover () { min=98 elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then min=99 + elif [ "$1" = "certbot_dns_gehirn" ]; then + min=97 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_dns_linode" ]; then diff --git a/tox.ini b/tox.ini index 9373b3aa7..32020cf14 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ dns_packages = certbot-dns-digitalocean \ certbot-dns-dnsimple \ certbot-dns-dnsmadeeasy \ + certbot-dns-gehirn \ certbot-dns-google \ certbot-dns-linode \ certbot-dns-luadns \ @@ -47,6 +48,7 @@ source_paths = certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy + certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns From b7113a35eb4bf93dfa12b8f16f44043739ace11d Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 10 Jul 2018 18:48:09 -0700 Subject: [PATCH 576/631] Delete empty directories after deleting an account, including symlinks up and down the chain, as appropriate (#6176) --- certbot/account.py | 38 +++++++++++++++++++++++++++++++++++ certbot/tests/account_test.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/certbot/account.py b/certbot/account.py index 2f261f759..f2ed5cfd5 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -250,12 +250,50 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ + # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) shutil.rmtree(account_dir_path) + # Step 2: remove the directory if it's empty, and linked directories + if not os.listdir(self.config.accounts_dir): + self._delete_accounts_dir_for_server_path(self.config.server_path) + + def _delete_accounts_dir_for_server_path(self, server_path): + accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + + # does an appropriate directory link to me? if so, make sure that's gone + reused_servers = {} + for k in constants.LE_REUSE_SERVERS: + reused_servers[constants.LE_REUSE_SERVERS[k]] = k + + # is there a next one up? call that and be done + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) + if os.path.islink(next_accounts_dir_path) \ + and os.readlink(next_accounts_dir_path) == accounts_dir_path: + self._delete_accounts_dir_for_server_path(next_server_path) + return + + # if there's not a next one up to delete, then delete me + # and whatever I link to if applicable + if os.path.islink(accounts_dir_path): + # save my info then delete me + target = os.readlink(accounts_dir_path) + os.unlink(accounts_dir_path) + # then delete whatever I linked to, if appropriate + if server_path in constants.LE_REUSE_SERVERS: + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) + if target == prev_accounts_dir_path: + self._delete_accounts_dir_for_server_path(prev_server_path) + else: + # just delete me + os.rmdir(accounts_dir_path) + def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e7f82a5b8..e0ec3d5f8 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -273,6 +273,40 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_delete_no_account(self): self.assertRaises(errors.AccountNotFound, self.storage.delete, self.acc.id) + def _assert_symlinked_account_removed(self): + # create v1 account + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + # ensure v2 isn't already linked to it + with mock.patch('certbot.constants.LE_REUSE_SERVERS', {}): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + + def _test_delete_folders(self, server_url): + # create symlinked servers + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.storage.find_all() + + # delete starting at given server_url + self._set_server(server_url) + self.storage.delete(self.acc.id) + + # make sure we're gone from both urls + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + + def test_delete_folders_up(self): + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + self._assert_symlinked_account_removed() + + def test_delete_folders_down(self): + self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') + self._assert_symlinked_account_removed() + if __name__ == "__main__": unittest.main() # pragma: no cover From 148d68b99a86b3ef0a1257af8becf481a075ebe0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 18:48:49 -0700 Subject: [PATCH 577/631] Fix broken fedora links (#6196) * fix broken fedora links * Add packaged plugin links --- docs/packaging.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index dc75e34b0..48ea02ae7 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -84,8 +84,21 @@ Fedora In Fedora 23+. -- https://admin.fedoraproject.org/pkgdb/package/certbot/ -- https://admin.fedoraproject.org/pkgdb/package/python-acme/ +- https://apps.fedoraproject.org/packages/python-acme +- https://apps.fedoraproject.org/packages/certbot +- https://apps.fedoraproject.org/packages/python-certbot-apache +- https://apps.fedoraproject.org/packages/python-certbot-dns-cloudflare +- https://apps.fedoraproject.org/packages/python-certbot-dns-cloudxns +- https://apps.fedoraproject.org/packages/python-certbot-dns-digitalocean +- https://apps.fedoraproject.org/packages/python-certbot-dns-dnsimple +- https://apps.fedoraproject.org/packages/python-certbot-dns-dnsmadeeasy +- https://apps.fedoraproject.org/packages/python-certbot-dns-google +- https://apps.fedoraproject.org/packages/python-certbot-dns-linode +- https://apps.fedoraproject.org/packages/python-certbot-dns-luadns +- https://apps.fedoraproject.org/packages/python-certbot-dns-nsone +- https://apps.fedoraproject.org/packages/python-certbot-dns-rfc2136 +- https://apps.fedoraproject.org/packages/python-certbot-dns-route53 +- https://apps.fedoraproject.org/packages/python-certbot-nginx FreeBSD ------- From a2222d5bdff304f8c55b63986d09d3f155ef6893 Mon Sep 17 00:00:00 2001 From: Nicolas Bachschmidt Date: Wed, 11 Jul 2018 05:52:32 +0200 Subject: [PATCH 578/631] OVH DNS Authenticator (#5423) Implement an Authenticator which can fulfill a dns-01 challenge using the OVH DNS API. Applicable only for domains using OVH DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-ovh -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-ovh -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting dnsimple interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Path to credentials file with an invalid application key. * Path to credentials file with an invalid application secret. * Path to credentials file with an invalid consumer key. * Path to credentials file with missing properties. * Domain name not registered to OVH account. --- certbot-dns-ovh/Dockerfile | 5 + certbot-dns-ovh/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-ovh/MANIFEST.in | 3 + certbot-dns-ovh/README.rst | 1 + certbot-dns-ovh/certbot_dns_ovh/__init__.py | 98 +++++++++ certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py | 102 ++++++++++ .../certbot_dns_ovh/dns_ovh_test.py | 62 ++++++ certbot-dns-ovh/docs/.gitignore | 1 + certbot-dns-ovh/docs/Makefile | 20 ++ certbot-dns-ovh/docs/api.rst | 8 + certbot-dns-ovh/docs/api/dns_ovh.rst | 5 + certbot-dns-ovh/docs/conf.py | 180 +++++++++++++++++ certbot-dns-ovh/docs/index.rst | 28 +++ certbot-dns-ovh/docs/make.bat | 36 ++++ certbot-dns-ovh/local-oldest-requirements.txt | 2 + .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-ovh/setup.cfg | 2 + certbot-dns-ovh/setup.py | 69 +++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 6 +- docs/cli-help.txt | 12 ++ docs/packaging.rst | 2 + docs/using.rst | 1 + tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 30 files changed, 857 insertions(+), 4 deletions(-) create mode 100644 certbot-dns-ovh/Dockerfile create mode 100644 certbot-dns-ovh/LICENSE.txt create mode 100644 certbot-dns-ovh/MANIFEST.in create mode 100644 certbot-dns-ovh/README.rst create mode 100644 certbot-dns-ovh/certbot_dns_ovh/__init__.py create mode 100644 certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py create mode 100644 certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py create mode 100644 certbot-dns-ovh/docs/.gitignore create mode 100644 certbot-dns-ovh/docs/Makefile create mode 100644 certbot-dns-ovh/docs/api.rst create mode 100644 certbot-dns-ovh/docs/api/dns_ovh.rst create mode 100644 certbot-dns-ovh/docs/conf.py create mode 100644 certbot-dns-ovh/docs/index.rst create mode 100644 certbot-dns-ovh/docs/make.bat create mode 100644 certbot-dns-ovh/local-oldest-requirements.txt create mode 100644 certbot-dns-ovh/readthedocs.org.requirements.txt create mode 100644 certbot-dns-ovh/setup.cfg create mode 100644 certbot-dns-ovh/setup.py diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile new file mode 100644 index 000000000..e8da96d95 --- /dev/null +++ b/certbot-dns-ovh/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-ovh + +RUN pip install --no-cache-dir --editable src/certbot-dns-ovh diff --git a/certbot-dns-ovh/LICENSE.txt b/certbot-dns-ovh/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-ovh/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-ovh/MANIFEST.in b/certbot-dns-ovh/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-ovh/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-ovh/README.rst b/certbot-dns-ovh/README.rst new file mode 100644 index 000000000..05ffe2a16 --- /dev/null +++ b/certbot-dns-ovh/README.rst @@ -0,0 +1 @@ +OVH DNS Authenticator plugin for Certbot diff --git a/certbot-dns-ovh/certbot_dns_ovh/__init__.py b/certbot-dns-ovh/certbot_dns_ovh/__init__.py new file mode 100644 index 000000000..47f8bda9f --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/__init__.py @@ -0,0 +1,98 @@ +""" +The `~certbot_dns_ovh.dns_ovh` plugin automates the process of +completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and +subsequently removing, TXT records using the OVH API. + + +Named Arguments +--------------- + +=================================== ========================================== +``--dns-ovh-credentials`` OVH credentials_ INI file. + (Required) +``--dns-ovh-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 30) +=================================== ========================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing OVH API +credentials for an account with the following access rules: + +* ``GET /domain/zone/*`` +* ``PUT /domain/zone/*`` +* ``POST /domain/zone/*`` +* ``DELETE /domain/zone/*`` + +These credentials can be obtained there: + +* `OVH Europe `_ (endpoint: ``ovh-eu``) +* `OVH North America `_ (endpoint: + ``ovh-ca``) + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # OVH API credentials used by Certbot + dns_ovh_endpoint = ovh-eu + dns_ovh_application_key = MDAwMDAwMDAwMDAw + dns_ovh_application_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + dns_ovh_consumer_key = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-ovh-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + OVH account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire + new certificates or revoke existing certificates for associated domains, + even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ohv.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ovh.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ovh.ini \\ + --dns-ovh-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py new file mode 100644 index 000000000..c4ded7748 --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -0,0 +1,102 @@ +"""DNS Authenticator for OVH DNS.""" +import logging + +import zope.interface +from lexicon.providers import ovh + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +TOKEN_URL = 'https://eu.api.ovh.com/createToken/ or https://ca.api.ovh.com/createToken/' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for OVH + + This Authenticator uses the OVH API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record (if you are using OVH for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='OVH credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the OVH API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'OVH credentials INI file', + { + 'endpoint': 'OVH API endpoint (ovh-eu or ovh-ca)', + 'application-key': 'Application key for OVH API, obtained from {0}' + .format(TOKEN_URL), + 'application-secret': 'Application secret for OVH API, obtained from {0}' + .format(TOKEN_URL), + 'consumer-key': 'Consumer key for OVH API, obtained from {0}' + .format(TOKEN_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_ovh_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_ovh_client().del_txt_record(domain, validation_name, validation) + + def _get_ovh_client(self): + return _OVHLexiconClient( + self.credentials.conf('endpoint'), + self.credentials.conf('application-key'), + self.credentials.conf('application-secret'), + self.credentials.conf('consumer-key'), + self.ttl + ) + + +class _OVHLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the OVH API via Lexicon. + """ + + def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): + super(_OVHLexiconClient, self).__init__() + + self.provider = ovh.Provider({ + 'auth_entrypoint': endpoint, + 'auth_application_key': application_key, + 'auth_application_secret': application_secret, + 'auth_consumer_key': consumer_key, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + hint = None + if str(e).startswith('400 Client Error:'): + hint = 'Is your Application Secret value correct?' + if str(e).startswith('403 Client Error:'): + hint = 'Are your Application Key and Consumer Key values correct?' + + return errors.PluginError('Error determining zone identifier for {0}: {1}.{2}' + .format(domain_name, e, ' ({0})'.format(hint) if hint else '')) + + def _handle_general_error(self, e, domain_name): + if domain_name in str(e) and str(e).endswith('not found'): + return + + super(_OVHLexiconClient, self)._handle_general_error(e, domain_name) diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py new file mode 100644 index 000000000..f2a10485d --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py @@ -0,0 +1,62 @@ +"""Tests for certbot_dns_ovh.dns_ovh.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +ENDPOINT = 'ovh-eu' +APPLICATION_KEY = 'foo' +APPLICATION_SECRET = 'bar' +CONSUMER_KEY = 'spam' + + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_ovh.dns_ovh import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + credentials = { + "ovh_endpoint": ENDPOINT, + "ovh_application_key": APPLICATION_KEY, + "ovh_application_secret": APPLICATION_SECRET, + "ovh_consumer_key": CONSUMER_KEY, + } + dns_test_common.write(credentials, path) + + self.config = mock.MagicMock(ovh_credentials=path, + ovh_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "ovh") + + self.mock_client = mock.MagicMock() + # _get_ovh_client | pylint: disable=protected-access + self.auth._get_ovh_client = mock.MagicMock(return_value=self.mock_client) + + +class OVHLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = Exception('Domain example.com not found') + LOGIN_ERROR = HTTPError('403 Client Error: Forbidden for url: https://eu.api.ovh.com/1.0/...') + + def setUp(self): + from certbot_dns_ovh.dns_ovh import _OVHLexiconClient + + self.client = _OVHLexiconClient( + ENDPOINT, APPLICATION_KEY, APPLICATION_SECRET, CONSUMER_KEY, 0 + ) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-ovh/docs/.gitignore b/certbot-dns-ovh/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-ovh/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-ovh/docs/Makefile b/certbot-dns-ovh/docs/Makefile new file mode 100644 index 000000000..38f6a9159 --- /dev/null +++ b/certbot-dns-ovh/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-ovh +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-ovh/docs/api.rst b/certbot-dns-ovh/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-ovh/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-ovh/docs/api/dns_ovh.rst b/certbot-dns-ovh/docs/api/dns_ovh.rst new file mode 100644 index 000000000..79863d05f --- /dev/null +++ b/certbot-dns-ovh/docs/api/dns_ovh.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_ovh.dns_ovh` +------------------------------ + +.. automodule:: certbot_dns_ovh.dns_ovh + :members: diff --git a/certbot-dns-ovh/docs/conf.py b/certbot-dns-ovh/docs/conf.py new file mode 100644 index 000000000..57194666e --- /dev/null +++ b/certbot-dns-ovh/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-ovh documentation build configuration file, created by +# sphinx-quickstart on Fri Jan 12 10:14:31 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-ovh' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-ovhdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-ovh.tex', u'certbot-dns-ovh Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-ovh', u'certbot-dns-ovh Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-ovh', u'certbot-dns-ovh Documentation', + author, 'certbot-dns-ovh', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-ovh/docs/index.rst b/certbot-dns-ovh/docs/index.rst new file mode 100644 index 000000000..ad5860289 --- /dev/null +++ b/certbot-dns-ovh/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-ovh documentation master file, created by + sphinx-quickstart on Fri Jan 12 10:14:31 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-ovh's documentation! +=========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: certbot_dns_ovh + :members: + +.. toctree:: + :maxdepth: 1 + + api + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-ovh/docs/make.bat b/certbot-dns-ovh/docs/make.bat new file mode 100644 index 000000000..78f7dd669 --- /dev/null +++ b/certbot-dns-ovh/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-ovh + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-ovh/readthedocs.org.requirements.txt b/certbot-dns-ovh/readthedocs.org.requirements.txt new file mode 100644 index 000000000..0780e12a1 --- /dev/null +++ b/certbot-dns-ovh/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-ovh[docs] diff --git a/certbot-dns-ovh/setup.cfg b/certbot-dns-ovh/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-ovh/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py new file mode 100644 index 000000000..4e2e664a4 --- /dev/null +++ b/certbot-dns-ovh/setup.py @@ -0,0 +1,69 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'mock', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-ovh', + version=version, + description="OVH DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-ovh = certbot_dns_ovh.dns_ovh:Authenticator', + ], + }, + test_suite='certbot_dns_ovh', +) diff --git a/certbot/cli.py b/certbot/cli.py index 6d262ed72..2c4aa6530 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1435,6 +1435,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_nsone"), help=("Obtain certificates using a DNS TXT record (if you are " "using NS1 for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-ovh", action="store_true", + default=flag_default("dns_ovh"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using OVH for DNS).")) helpful.add(["plugins", "certonly"], "--dns-rfc2136", action="store_true", default=flag_default("dns_rfc2136"), help="Obtain certificates using a DNS TXT record (if you are using BIND for DNS).") diff --git a/certbot/constants.py b/certbot/constants.py index 70249b89b..46523ce4d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -109,6 +109,7 @@ CLI_DEFAULTS = dict( dns_linode=False, dns_luadns=False, dns_nsone=False, + dns_ovh=False, dns_rfc2136=False, dns_route53=False, dns_sakuracloud=False diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 6ed0cf7b7..7be320efc 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -35,6 +35,7 @@ class PluginEntryPoint(object): "certbot-dns-linode", "certbot-dns-luadns", "certbot-dns-nsone", + "certbot-dns-ovh", "certbot-dns-rfc2136", "certbot-dns-route53", "certbot-dns-sakuracloud", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 0073c99fe..9c2138247 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,8 +165,8 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-gehirn", - "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", - "dns-route53", "dns-sakuracloud"] + "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-ovh", + "dns-rfc2136", "dns-route53", "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -299,6 +299,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-luadns") if config.dns_nsone: req_auth = set_configurator(req_auth, "dns-nsone") + if config.dns_ovh: + req_auth = set_configurator(req_auth, "dns-ovh") if config.dns_rfc2136: req_auth = set_configurator(req_auth, "dns-rfc2136") if config.dns_route53: diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 7b8b522c9..8bba718d5 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -454,6 +454,8 @@ plugins: using LuaDNS for DNS). (default: False) --dns-nsone Obtain certificates using a DNS TXT record (if you are using NS1 for DNS). (default: False) + --dns-ovh Obtain certificates using a DNS TXT record (if you are + using OVH for DNS). (default: False) --dns-rfc2136 Obtain certificates using a DNS TXT record (if you are using BIND for DNS). (default: False) --dns-route53 Obtain certificates using a DNS TXT record (if you are @@ -589,6 +591,16 @@ dns-nsone: --dns-nsone-credentials DNS_NSONE_CREDENTIALS NS1 credentials file. (default: None) +dns-ovh: + Obtain certificates using a DNS TXT record (if you are using OVH for DNS). + + --dns-ovh-propagation-seconds DNS_OVH_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 30) + --dns-ovh-credentials DNS_OVH_CREDENTIALS + OVH credentials file. (default: None) + dns-rfc2136: Obtain certificates using a DNS TXT record (if you are using BIND for DNS). diff --git a/docs/packaging.rst b/docs/packaging.rst index 48ea02ae7..a86b770c5 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -20,6 +20,7 @@ We release packages and upload them to PyPI (wheels and source tarballs). - https://pypi.python.org/pypi/certbot-dns-linode - https://pypi.python.org/pypi/certbot-dns-luadns - https://pypi.python.org/pypi/certbot-dns-nsone +- https://pypi.python.org/pypi/certbot-dns-ovh - https://pypi.python.org/pypi/certbot-dns-rfc2136 - https://pypi.python.org/pypi/certbot-dns-route53 @@ -67,6 +68,7 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone +- https://www.archlinux.org/packages/community/any/certbot-dns-ovh - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 - https://www.archlinux.org/packages/community/any/certbot-dns-route53 diff --git a/docs/using.rst b/docs/using.rst index 50c27d45e..946c12bc6 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -206,6 +206,7 @@ Once installed, you can find documentation on how to use each plugin at: * `certbot-dns-linode `_ * `certbot-dns-luadns `_ * `certbot-dns-nsone `_ +* `certbot-dns-ovh `_ * `certbot-dns-rfc2136 `_ * `certbot-dns-route53 `_ diff --git a/tools/release.sh b/tools/release.sh index 0d42bc22a..880563b4b 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 159fc16fb..5692f9ebf 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -24,6 +24,7 @@ fi -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ + -e certbot-dns-ovh \ -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ diff --git a/tools/venv3.sh b/tools/venv3.sh index a1489df22..784fc42e8 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,6 +23,7 @@ fi -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ + -e certbot-dns-ovh \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ -e certbot-nginx \ diff --git a/tox.cover.sh b/tox.cover.sh index cccaf6103..c713327c5 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -41,6 +41,8 @@ cover () { min=98 elif [ "$1" = "certbot_dns_nsone" ]; then min=99 + elif [ "$1" = "certbot_dns_ovh" ]; then + min=97 elif [ "$1" = "certbot_dns_rfc2136" ]; then min=99 elif [ "$1" = "certbot_dns_route53" ]; then diff --git a/tox.ini b/tox.ini index 32020cf14..482c65c36 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,7 @@ dns_packages = certbot-dns-linode \ certbot-dns-luadns \ certbot-dns-nsone \ + certbot-dns-ovh \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-dns-sakuracloud @@ -53,6 +54,7 @@ source_paths = certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone + certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud From c326c021082dede7c3b2bd411cec3aec6dff0ac5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 11 Jul 2018 11:20:36 -0700 Subject: [PATCH 579/631] Update default to ACMEv2 server (#6152) --- certbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/constants.py b/certbot/constants.py index 46523ce4d..a2de2d27a 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -88,7 +88,7 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", - server="https://acme-v01.api.letsencrypt.org/directory", + server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers configurator=None, From 95e271bfcd9efcb6f42d2a712d7dcfc3a21408d6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 14:18:26 -0700 Subject: [PATCH 580/631] Release 0.26.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 4 +- certbot-auto | 83 ++++++++---------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 49 ++++++++++- letsencrypt-auto | 83 ++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 ++-- letsencrypt-auto-source/letsencrypt-auto | 26 +++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++--- 26 files changed, 171 insertions(+), 150 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e6cfcafc6..88967c716 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index cbd434f01..f0a40a2f3 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.25.0', - 'certbot>=0.26.0.dev0', + 'certbot>=0.26.0', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-auto b/certbot-auto index d2cfa672d..17e00929b 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.1" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1103,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c @@ -1208,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 34e39ec5e..c5381278e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9cf26aa52..c7a3418c6 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 8077fe8f5..e5c32af9d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 5877d2d0d..039c49767 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 8c00e6d58..2f3f95a9c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 34772c422..3d1ce16c9 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index cc47da327..be7515274 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ad56e58be..e020412b5 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 2abd19b68..3043aafed 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 796f7a489..3ea232bc3 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d87470c3a..2e63de4f0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 4e2e664a4..fc76b6e0c 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bbfbcbba1..813dc4981 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8836dc6d8..97039dade 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index dd8903fdd..53c65af0f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b43684feb..dd30dae7f 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 3ae0e315b..766be39a3 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0.dev0' +__version__ = '0.26.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 8bba718d5..c044c206a 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.1 + "". (default: CertbotACMEClient/0.26.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -196,6 +196,8 @@ security: --strict-permissions Require that all configuration files are owned by the current user; only needed if your config is somewhere unsafe like /tmp/ (default: False) + --auto-hsts Gradually increasing max-age value for HTTP Strict + Transport Security security header (default: False) testing: The following flags are meant for testing and integration purposes only. @@ -249,7 +251,7 @@ paths: --work-dir WORK_DIR Working directory. (default: /var/lib/letsencrypt) --logs-dir LOGS_DIR Logs directory. (default: /var/log/letsencrypt) --server SERVER ACME Directory Resource URI. (default: - https://acme-v01.api.letsencrypt.org/directory) + https://acme-v02.api.letsencrypt.org/directory) manage: Various subcommands and flags are available for managing your @@ -328,6 +330,7 @@ renew: renew", regardless of if the certificate is renewed. This setting does not apply to important TLS configuration updates. (default: False) + --no-autorenew Disable auto renewal of certificates. (default: True) certificates: List certificates managed by Certbot @@ -448,8 +451,13 @@ plugins: using DNSimple for DNS). (default: False) --dns-dnsmadeeasy Obtain certificates using a DNS TXT record (if you areusing DNS Made Easy for DNS). (default: False) + --dns-gehirn Obtain certificates using a DNS TXT record (if you are + using Gehirn Infrastracture Service for DNS). + (default: False) --dns-google Obtain certificates using a DNS TXT record (if you are using Google Cloud DNS). (default: False) + --dns-linode Obtain certificates using a DNS TXT record (if you are + using Linode for DNS). (default: False) --dns-luadns Obtain certificates using a DNS TXT record (if you are using LuaDNS for DNS). (default: False) --dns-nsone Obtain certificates using a DNS TXT record (if you are @@ -460,6 +468,8 @@ plugins: using BIND for DNS). (default: False) --dns-route53 Obtain certificates using a DNS TXT record (if you are using Route53 for DNS). (default: False) + --dns-sakuracloud Obtain certificates using a DNS TXT record (if you are + using Sakura Cloud for DNS). (default: False) apache: Apache Web Server plugin - Beta @@ -553,6 +563,18 @@ dns-dnsmadeeasy: --dns-dnsmadeeasy-credentials DNS_DNSMADEEASY_CREDENTIALS DNS Made Easy credentials INI file. (default: None) +dns-gehirn: + Obtain certificates using a DNS TXT record (if you are using Gehirn + Infrastracture Service for DNS). + + --dns-gehirn-propagation-seconds DNS_GEHIRN_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 30) + --dns-gehirn-credentials DNS_GEHIRN_CREDENTIALS + Gehirn Infrastracture Service credentials file. + (default: None) + dns-google: Obtain certificates using a DNS TXT record (if you are using Google Cloud DNS for DNS). @@ -570,6 +592,16 @@ dns-google: control#permissions_and_roles for information about therequired permissions.) (default: None) +dns-linode: + Obtain certs using a DNS TXT record (if you are using Linode for DNS). + + --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 960) + --dns-linode-credentials DNS_LINODE_CREDENTIALS + Linode credentials INI file. (default: None) + dns-luadns: Obtain certificates using a DNS TXT record (if you are using LuaDNS for DNS). @@ -599,7 +631,7 @@ dns-ovh: before asking the ACME server to verify the DNS record. (default: 30) --dns-ovh-credentials DNS_OVH_CREDENTIALS - OVH credentials file. (default: None) + OVH credentials INI file. (default: None) dns-rfc2136: Obtain certificates using a DNS TXT record (if you are using BIND for @@ -621,6 +653,17 @@ dns-route53: before asking the ACME server to verify the DNS record. (default: 10) +dns-sakuracloud: + Obtain certificates using a DNS TXT record (if you are using Sakura Cloud + for DNS). + + --dns-sakuracloud-propagation-seconds DNS_SAKURACLOUD_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 90) + --dns-sakuracloud-credentials DNS_SAKURACLOUD_CREDENTIALS + Sakura Cloud credentials file. (default: None) + manual: Authenticate through manual configuration or custom shell scripts. When using shell scripts, an authenticator script must be provided. The diff --git a/letsencrypt-auto b/letsencrypt-auto index d2cfa672d..17e00929b 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.1" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1103,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c @@ -1208,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 67bab66d4..0b7d27be3 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsgc/cACgkQTRfJlc2X -dfLjBgf/bHZn/q+Dqn34uBXHymRSce7UxQn17izcKAt7hZBl4j4sebQ9+0jjuNur -zrW8b0XJ0PsI10GG9qHR3ajC+04pWfRritnK1g4Ycb/pDcUkWo+8uRwr7skAVcvC -oa8ToBS3iUbd3csFl1mu1BGACUHLvVs2cYdDtMuJj8wjsVZ7KnWBGKULAskwmU4Z -VVUxeUrG9f+2kT35meEJUk91FS+4tmqNIVsVlBzf0Q0ZU1iQnV56dMwTqFRzdDJ2 -DBecE0GwuYnKXo2I7kIYaqACQmk9YFh55Sh0K9PbQxyv7YEZXZtkcdqFqyhxy3Nh -EJ2kurFaM3/VmLljc/rW8QW8B3QNbw== -=pkDz +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAltGdAYACgkQTRfJlc2X +dfKUyQf+NakhD3SfMeuJyT1StexEc9iGaAvspNH+Gf6P5v5dZOZnSOdtraR2kQAi +OQE2L5FAajIhpuELpZTAgCEFU1LZpqTvWOb/1Vb06T8DuLIYierh64LkAn0zJY/M +e8PTWyU5dcM6pY0ITvhuIMDAtomV+TzKeD1qHy2hJVTJGttk/yNtT5p8/NYIuH8Z +OWXkNuo/346xvYpTDp2Xpwv79L9JhQsxfEBpKV4IGObpTf+Mfl2f4taroLYEATGU +vrNM39P0cxu/hEHpog74CHPeK99YlBR6+7tMINQ9bYHkdjq2vLYdyopE8mCN16oy +CwITDfR5POwvs+WjU+oEtgQb73kTug== +=3XNY -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9afa86849..17e00929b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0.dev0" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index f266c93e99a6c6b451a0dd066cf33fc069cac53c..cf87eacfc908dc352e0d17756ecd6bc9d7320e63 100644 GIT binary patch literal 256 zcmV+b0ssD!H-fBQ`@sIYn!8z-`j+lz{Q@N0u$U83m47cn~8z5v4(zv^ufnv-raM*`SHDa?$*L zwjYLFp(zsV>M%UIA)#&*&!|W<43f30`QC>x;6{5kHWfag>;fAdx=2aP?uJ%oq~Mt# zwrh6trQ~7|S0C@}LAcZA0h*o^am2HkeuXz7reK3777FD4>jya36?*RxD`;b-E+2?X Gsx%mai+NN4 literal 256 zcmV+b0ssDtYs3=1`HBT)24SsR1u@$W7ZPSU$S&OOtBrFsR+98n78ySvPt!!G z;8VuT@C4ln966cf7@p1|^2x*#OsORv*dICX|C;T?NkAcYvB#;S?b|v(@U=jc^F-3_=^g!<2^ko94v~{-OR8 zs(jE6oLCW6yS;4jMUbxlVGl~J&StH_A-KuM!mMbFXeL^Jnpkr41{l6#rWYsfUp^(i zzO<}YgQBPwyO%n@J11S;c~Voa9yf?}P0YBYWV5x#$dd)Xb67iJaEyXJESY09t?}#b GsB~g1X@A53 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index f53891ccd..f80ee2cb4 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 From 0a6d520d26c8466c69078734dfbff096470ae807 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 14:18:44 -0700 Subject: [PATCH 581/631] Bump version to 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88967c716..88592013c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0a40a2f3..f435bb1a9 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c5381278e..e4ccc719a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c7a3418c6..05649d4d0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index e5c32af9d..911f9e052 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 039c49767..9dd318296 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2f3f95a9c..09b11def0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 3d1ce16c9..2ca3213bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index be7515274..e9ead6546 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e020412b5..3c7402f25 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 3043aafed..327224c9c 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 3ea232bc3..1f92f7dce 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2e63de4f0..0b4241afb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fc76b6e0c..258c7f0f1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 813dc4981..bd54ec4c5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 97039dade..5f0b26f6e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 53c65af0f..b7cfc15b5 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index dd30dae7f..4706f17bd 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 766be39a3..3b0b77f6c 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0' +__version__ = '0.27.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 17e00929b..f733c71b4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.27.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 72cf844ecffd2669488aa677d71b8af0bacd1a14 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Thu, 12 Jul 2018 14:13:13 +0200 Subject: [PATCH 582/631] travis: container env are bad at reading cpu count Signed-off-by: Yoan Blanc --- .travis.yml | 2 +- tox.cover.sh | 4 +++- tox.ini | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 937d24610..3e3f7a021 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=cover NUMPROCESSES=2 FYI="this also tests py27" - sudo: required env: TOXENV=nginx_compat services: docker diff --git a/tox.cover.sh b/tox.cover.sh index c713327c5..6440d1b48 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -8,6 +8,8 @@ # # -e makes sure we fail fast and don't submit coveralls submit +NUMPROCESSES=${NUMPROCESSES:=auto} + if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else @@ -61,7 +63,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "$NUMPROCESSES" --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 482c65c36..d113fd450 100644 --- a/tox.ini +++ b/tox.ini @@ -115,6 +115,7 @@ commands = [testenv:cover] basepython = python2.7 +passenv = NUMPROCESSES commands = {[base]install_packages} ./tox.cover.sh From 9b0d2714c1404190a4e70457da732dd08bbe861e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Jul 2018 16:21:24 -0700 Subject: [PATCH 583/631] Handle existing ACMEv1 and ACMEv2 accounts (#6214) Fixes #6207. As noted by Erica: - we no longer need to check if it exists before linking to it, because we delete properly. - the previously excisting check on if server is in `LE_REUSE_SERVERS` before unlinking is nice, but probably not necessary, especially since we don't officially support people doing weird things with symlinks in our directories, and because we rmdir which will fail if it's not empty anyway. * Create single account symlink. * refactor _delete_accounts_dir_for_server_path * add symlinked account dir deletion * add tests --- certbot/account.py | 81 +++++++++++++++++++++++------------ certbot/tests/account_test.py | 20 +++++++++ 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index f2ed5cfd5..59ceb42e0 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -1,5 +1,6 @@ """Creates ACME accounts for server.""" import datetime +import functools import hashlib import logging import os @@ -191,6 +192,11 @@ class AccountFileStorage(interfaces.AccountStorage): def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_account_dir(self, prev_server_path, server_path, account_id): + prev_account_dir = self._account_dir_path_for_server_path(account_id, prev_server_path) + new_account_dir = self._account_dir_path_for_server_path(account_id, server_path) + os.symlink(prev_account_dir, new_account_dir) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): accounts_dir = self.config.accounts_dir_for_server_path(server_path) if os.path.islink(accounts_dir): @@ -207,7 +213,12 @@ class AccountFileStorage(interfaces.AccountStorage): prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) # we didn't error so we found something, so create a symlink to that - self._symlink_to_accounts_dir(prev_server_path, server_path) + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + # If accounts_dir isn't empty, make an account specific symlink + if os.listdir(accounts_dir): + self._symlink_to_account_dir(prev_server_path, server_path, account_id) + else: + self._symlink_to_accounts_dir(prev_server_path, server_path) return prev_loaded_account else: raise errors.AccountNotFound( @@ -250,49 +261,65 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ - # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) - shutil.rmtree(account_dir_path) + # Step 1: Delete account specific links and the directory + self._delete_account_dir_for_server_path(account_id, self.config.server_path) - # Step 2: remove the directory if it's empty, and linked directories + # Step 2: Remove any accounts links and directories that are now empty if not os.listdir(self.config.accounts_dir): self._delete_accounts_dir_for_server_path(self.config.server_path) + def _delete_account_dir_for_server_path(self, account_id, server_path): + link_func = functools.partial(self._account_dir_path_for_server_path, account_id) + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + shutil.rmtree(nonsymlinked_dir) + def _delete_accounts_dir_for_server_path(self, server_path): - accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + link_func = self.config.accounts_dir_for_server_path + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + os.rmdir(nonsymlinked_dir) + + def _delete_links_and_find_target_dir(self, server_path, link_func): + """Delete symlinks and return the nonsymlinked directory path. + + :param str server_path: file path based on server + :param callable link_func: callable that returns possible links + given a server_path + + :returns: the final, non-symlinked target + :rtype: str + + """ + dir_path = link_func(server_path) # does an appropriate directory link to me? if so, make sure that's gone reused_servers = {} for k in constants.LE_REUSE_SERVERS: reused_servers[constants.LE_REUSE_SERVERS[k]] = k - # is there a next one up? call that and be done - if server_path in reused_servers: - next_server_path = reused_servers[server_path] - next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) - if os.path.islink(next_accounts_dir_path) \ - and os.readlink(next_accounts_dir_path) == accounts_dir_path: - self._delete_accounts_dir_for_server_path(next_server_path) - return + # is there a next one up? + possible_next_link = True + while possible_next_link: + possible_next_link = False + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_dir_path = link_func(next_server_path) + if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path: + possible_next_link = True + server_path = next_server_path + dir_path = next_dir_path # if there's not a next one up to delete, then delete me - # and whatever I link to if applicable - if os.path.islink(accounts_dir_path): - # save my info then delete me - target = os.readlink(accounts_dir_path) - os.unlink(accounts_dir_path) - # then delete whatever I linked to, if appropriate - if server_path in constants.LE_REUSE_SERVERS: - prev_server_path = constants.LE_REUSE_SERVERS[server_path] - prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) - if target == prev_accounts_dir_path: - self._delete_accounts_dir_for_server_path(prev_server_path) - else: - # just delete me - os.rmdir(accounts_dir_path) + # and whatever I link to + while os.path.islink(dir_path): + target = os.readlink(dir_path) + os.unlink(dir_path) + dir_path = target + + return dir_path def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e0ec3d5f8..701478336 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -249,6 +249,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + def test_upgrade_load_single_account(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() @@ -307,6 +315,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + def _set_server_and_stop_symlink(self, server_path): + self._set_server(server_path) + with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: + f.write('bar') + + def test_delete_shared_account_up(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + + def test_delete_shared_account_down(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 84287130326406bd30d058668bd5a69ec4f7bdb3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 13 Jul 2018 14:21:23 -0700 Subject: [PATCH 584/631] Remove linode and ovh links which aren't valid yet. (#6198) * Remove linode links which aren't valid yet. * remove ovh references * keep links which are now valid * keep pypi links --- docs/packaging.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index a86b770c5..c13a14af3 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -65,10 +65,8 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-dnsimple - https://www.archlinux.org/packages/community/any/certbot-dns-dnsmadeeasy - https://www.archlinux.org/packages/community/any/certbot-dns-google -- https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone -- https://www.archlinux.org/packages/community/any/certbot-dns-ovh - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 - https://www.archlinux.org/packages/community/any/certbot-dns-route53 @@ -95,7 +93,6 @@ In Fedora 23+. - https://apps.fedoraproject.org/packages/python-certbot-dns-dnsimple - https://apps.fedoraproject.org/packages/python-certbot-dns-dnsmadeeasy - https://apps.fedoraproject.org/packages/python-certbot-dns-google -- https://apps.fedoraproject.org/packages/python-certbot-dns-linode - https://apps.fedoraproject.org/packages/python-certbot-dns-luadns - https://apps.fedoraproject.org/packages/python-certbot-dns-nsone - https://apps.fedoraproject.org/packages/python-certbot-dns-rfc2136 From fa7cb38e97b9cf7a22d377c5982c311f0d189242 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 07:33:38 -0700 Subject: [PATCH 585/631] Add 0.26.0 changelog (#6205) --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88251e48a..4d0808de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,54 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.26.0 - 2018-07-11 + +### Added + +* A new security enhancement which we're calling AutoHSTS has been added to + Certbot's Apache plugin. This enhancement configures your webserver to send a + HTTP Strict Transport Security header with a low max-age value that is slowly + increased over time. The max-age value is not increased to a large value + until you've successfully managed to renew your certificate. This enhancement + can be requested with the --auto-hsts flag. +* New official DNS plugins have been created for Gehirn Infrastracture Service, + Linode, OVH, and Sakura Cloud. These plugins can be found on our Docker Hub + page at https://hub.docker.com/u/certbot and on PyPI. +* The ability to reuse ACME accounts from Let's Encrypt's ACMEv1 endpoint on + Let's Encrypt's ACMEv2 endpoint has been added. +* Certbot and its components now support Python 3.7. +* Certbot's install subcommand now allows you to interactively choose which + certificate to install from the list of certificates managed by Certbot. +* Certbot now accepts the flag `--no-autorenew` which causes any obtained + certificates to not be automatically renewed when it approaches expiration. +* Support for parsing the TLS-ALPN-01 challenge has been added back to the acme + library. + +### Changed + +* Certbot's default ACME server has been changed to Let's Encrypt's ACMEv2 + endpoint. By default, this server will now be used for both new certificate + lineages and renewals. +* The Nginx plugin is no longer marked labeled as an "Alpha" version. +* The `prepare` method of Certbot's plugins is no longer called before running + "Updater" enhancements that are run on every invocation of `certbot renew`. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with functional changes were: + +* acme +* certbot +* certbot-apache +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-ovh +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/55?closed=1 + ## 0.25.1 - 2018-06-13 ### Fixed From 997496388721c9ea1092d60bd6fce7fa641fac9b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 07:54:36 -0700 Subject: [PATCH 586/631] Handle existing ACMEv1 and ACMEv2 accounts (#6214) (#6215) Fixes #6207. As noted by Erica: - we no longer need to check if it exists before linking to it, because we delete properly. - the previously excisting check on if server is in `LE_REUSE_SERVERS` before unlinking is nice, but probably not necessary, especially since we don't officially support people doing weird things with symlinks in our directories, and because we rmdir which will fail if it's not empty anyway. * Create single account symlink. * refactor _delete_accounts_dir_for_server_path * add symlinked account dir deletion * add tests (cherry picked from commit 9b0d2714c1404190a4e70457da732dd08bbe861e) --- certbot/account.py | 81 +++++++++++++++++++++++------------ certbot/tests/account_test.py | 20 +++++++++ 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index f2ed5cfd5..59ceb42e0 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -1,5 +1,6 @@ """Creates ACME accounts for server.""" import datetime +import functools import hashlib import logging import os @@ -191,6 +192,11 @@ class AccountFileStorage(interfaces.AccountStorage): def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_account_dir(self, prev_server_path, server_path, account_id): + prev_account_dir = self._account_dir_path_for_server_path(account_id, prev_server_path) + new_account_dir = self._account_dir_path_for_server_path(account_id, server_path) + os.symlink(prev_account_dir, new_account_dir) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): accounts_dir = self.config.accounts_dir_for_server_path(server_path) if os.path.islink(accounts_dir): @@ -207,7 +213,12 @@ class AccountFileStorage(interfaces.AccountStorage): prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) # we didn't error so we found something, so create a symlink to that - self._symlink_to_accounts_dir(prev_server_path, server_path) + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + # If accounts_dir isn't empty, make an account specific symlink + if os.listdir(accounts_dir): + self._symlink_to_account_dir(prev_server_path, server_path, account_id) + else: + self._symlink_to_accounts_dir(prev_server_path, server_path) return prev_loaded_account else: raise errors.AccountNotFound( @@ -250,49 +261,65 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ - # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) - shutil.rmtree(account_dir_path) + # Step 1: Delete account specific links and the directory + self._delete_account_dir_for_server_path(account_id, self.config.server_path) - # Step 2: remove the directory if it's empty, and linked directories + # Step 2: Remove any accounts links and directories that are now empty if not os.listdir(self.config.accounts_dir): self._delete_accounts_dir_for_server_path(self.config.server_path) + def _delete_account_dir_for_server_path(self, account_id, server_path): + link_func = functools.partial(self._account_dir_path_for_server_path, account_id) + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + shutil.rmtree(nonsymlinked_dir) + def _delete_accounts_dir_for_server_path(self, server_path): - accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + link_func = self.config.accounts_dir_for_server_path + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + os.rmdir(nonsymlinked_dir) + + def _delete_links_and_find_target_dir(self, server_path, link_func): + """Delete symlinks and return the nonsymlinked directory path. + + :param str server_path: file path based on server + :param callable link_func: callable that returns possible links + given a server_path + + :returns: the final, non-symlinked target + :rtype: str + + """ + dir_path = link_func(server_path) # does an appropriate directory link to me? if so, make sure that's gone reused_servers = {} for k in constants.LE_REUSE_SERVERS: reused_servers[constants.LE_REUSE_SERVERS[k]] = k - # is there a next one up? call that and be done - if server_path in reused_servers: - next_server_path = reused_servers[server_path] - next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) - if os.path.islink(next_accounts_dir_path) \ - and os.readlink(next_accounts_dir_path) == accounts_dir_path: - self._delete_accounts_dir_for_server_path(next_server_path) - return + # is there a next one up? + possible_next_link = True + while possible_next_link: + possible_next_link = False + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_dir_path = link_func(next_server_path) + if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path: + possible_next_link = True + server_path = next_server_path + dir_path = next_dir_path # if there's not a next one up to delete, then delete me - # and whatever I link to if applicable - if os.path.islink(accounts_dir_path): - # save my info then delete me - target = os.readlink(accounts_dir_path) - os.unlink(accounts_dir_path) - # then delete whatever I linked to, if appropriate - if server_path in constants.LE_REUSE_SERVERS: - prev_server_path = constants.LE_REUSE_SERVERS[server_path] - prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) - if target == prev_accounts_dir_path: - self._delete_accounts_dir_for_server_path(prev_server_path) - else: - # just delete me - os.rmdir(accounts_dir_path) + # and whatever I link to + while os.path.islink(dir_path): + target = os.readlink(dir_path) + os.unlink(dir_path) + dir_path = target + + return dir_path def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e0ec3d5f8..701478336 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -249,6 +249,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + def test_upgrade_load_single_account(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() @@ -307,6 +315,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + def _set_server_and_stop_symlink(self, server_path): + self._set_server(server_path) + with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: + f.write('bar') + + def test_delete_shared_account_up(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + + def test_delete_shared_account_down(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 72daec4346e3e43eed1ba4f42d5ab6e3346e8798 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 08:38:50 -0700 Subject: [PATCH 587/631] fix account tests (#6216) --- certbot/tests/account_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 701478336..b2be47d0f 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -295,7 +295,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') - self.storage.find_all() + self.storage.load(self.acc.id) # delete starting at given server_url self._set_server(server_url) @@ -326,7 +326,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_delete_shared_account_down(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') - self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 595e77eccbd58131db7a3e07ad9385f436119dc3 Mon Sep 17 00:00:00 2001 From: Trinopoty Biswas Date: Mon, 16 Jul 2018 21:10:16 +0530 Subject: [PATCH 588/631] Fixed linode API settings page URL (#6208) --- certbot-dns-linode/certbot_dns_linode/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index aaed61450..0c445f45d 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -23,7 +23,7 @@ Credentials Use of this plugin requires a configuration file containing Linode API credentials, obtained from your Linode account's `Applications & API -Tokens page `_. +Tokens page `_. .. code-block:: ini :name: credentials.ini From 704101c75b4bf2a7e3f45553fac2ea066bc2108b Mon Sep 17 00:00:00 2001 From: hal869 <36906232+hal869@users.noreply.github.com> Date: Mon, 16 Jul 2018 10:20:53 -0700 Subject: [PATCH 589/631] update venv3.sh to include dns-rfc2136 plugin (#6226) --- tools/venv3.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/venv3.sh b/tools/venv3.sh index 784fc42e8..07512f370 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -24,6 +24,7 @@ fi -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-ovh \ + -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ -e certbot-nginx \ From a0d68338a2da9ff0d7c751777086dc56b88839b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 16 Jul 2018 16:36:59 -0700 Subject: [PATCH 590/631] Release 0.26.1 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 14 ++++++---- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 86 insertions(+), 84 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88967c716..fe071ad56 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0a40a2f3..7cb15b8ad 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 17e00929b..e097719db 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c5381278e..0741a53af 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c7a3418c6..c036dab8f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index e5c32af9d..a70a65a2e 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 039c49767..69f422c18 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2f3f95a9c..4a0a91e2a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 3d1ce16c9..2daae6a10 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index be7515274..03c3b8bcb 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e020412b5..7e3729e2b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 3043aafed..9da99852d 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 3ea232bc3..259285eef 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2e63de4f0..7e7072fcf 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fc76b6e0c..fb10ebc2f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 813dc4981..4f4e4bc75 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 97039dade..cbe3802d6 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 53c65af0f..f7b134eb8 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index dd30dae7f..0ce8e8147 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 766be39a3..7fa464284 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0' +__version__ = '0.26.1' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index c044c206a..931ea4c62 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.26.0 + "". (default: CertbotACMEClient/0.26.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -475,9 +475,11 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: None) + Path to the Apache 'a2enmod' binary. (default: + a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: None) + Path to the Apache 'a2dismod' binary. (default: + a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -491,13 +493,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you. (Only Ubuntu/Debian currently) (default: False) + you. (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you. (Only - Ubuntu/Debian currently) (default: False) + Ubuntu/Debian currently) (default: True) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index 17e00929b..e097719db 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 0b7d27be3..9f6706931 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- +Version: GnuPG v2 -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAltGdAYACgkQTRfJlc2X -dfKUyQf+NakhD3SfMeuJyT1StexEc9iGaAvspNH+Gf6P5v5dZOZnSOdtraR2kQAi -OQE2L5FAajIhpuELpZTAgCEFU1LZpqTvWOb/1Vb06T8DuLIYierh64LkAn0zJY/M -e8PTWyU5dcM6pY0ITvhuIMDAtomV+TzKeD1qHy2hJVTJGttk/yNtT5p8/NYIuH8Z -OWXkNuo/346xvYpTDp2Xpwv79L9JhQsxfEBpKV4IGObpTf+Mfl2f4taroLYEATGU -vrNM39P0cxu/hEHpog74CHPeK99YlBR6+7tMINQ9bYHkdjq2vLYdyopE8mCN16oy -CwITDfR5POwvs+WjU+oEtgQb73kTug== -=3XNY +iQEcBAABCAAGBQJbTSv8AAoJEE0XyZXNl3Xy12sH/1FgV3SDVG0T1jgKQOYEUwrq +cmpjdav8YPgFOSQDOcyFZG0DNcRfTskZt45IMkBLLnXq2PuPvkppc1+akP81vMoK +NXHHS+PXDMjnBW4NFkexoM06KRF1SyHnvqsOg13w7UW2CjsAgtazGF5BucNCnjPH +XJTwUf4uhKxeUb0Xkva1OPH++oTWz8+SYgWr/iMggkBrK8y04QUUJ6lyCO6MZgcE +3JcECG7CwMK+hW0gCUkCSNZ0NzOBALCd9wCxNGszgkeJXrrW73oUpZmGC5BxIwYY +o6lcF0qo7Jb92t4B3+7JhulMC5JoVoG4lpiXpKQFFCT0P4pZKotIomKNMATmnB4= +=hzUL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 17e00929b..e097719db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index cf87eacfc908dc352e0d17756ecd6bc9d7320e63..d1306d03d5cb08580450b34b2cc50a7d5b35bde8 100644 GIT binary patch literal 256 zcmV+b0ssEZf5kI!7rKt%`BTJd3(dQ}1+=^wm$LZwtN4D@Kt?U1ALJ@F2wf=MBK-J| z2C_km8Z46qIOyMvkA(v9J>+vwvIk@Id*X$TMU_IAT;weL#Iofv&kDQ3yBu2YDx`x? zFUBjgYmFBxeWs=PgXF6Oerz8PWP(3kjE-j7Ci&~=g GfTdGDx_-t0 literal 256 zcmV+b0ssD!H-fBQ`@sIYn!8z-`j+lz{Q@N0u$U83m47cn~8z5v4(zv^ufnv-raM*`SHDa?$*L zwjYLFp(zsV>M%UIA)#&*&!|W<43f30`QC>x;6{5kHWfag>;fAdx=2aP?uJ%oq~Mt# zwrh6trQ~7|S0C@}LAcZA0h*o^am2HkeuXz7reK3777FD4>jya36?*RxD`;b-E+2?X Gsx%mai+NN4 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index f80ee2cb4..feb3f1c3a 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 From e9af000b5b452170199ac525b77787844bd486ed Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 16 Jul 2018 16:37:25 -0700 Subject: [PATCH 591/631] Bump version to 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index fe071ad56..88592013c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7cb15b8ad..f435bb1a9 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 0741a53af..e4ccc719a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c036dab8f..05649d4d0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a70a65a2e..911f9e052 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 69f422c18..9dd318296 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4a0a91e2a..09b11def0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2daae6a10..2ca3213bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 03c3b8bcb..e9ead6546 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 7e3729e2b..3c7402f25 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 9da99852d..327224c9c 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 259285eef..1f92f7dce 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 7e7072fcf..0b4241afb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fb10ebc2f..258c7f0f1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 4f4e4bc75..bd54ec4c5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index cbe3802d6..5f0b26f6e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index f7b134eb8..b7cfc15b5 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0ce8e8147..4706f17bd 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 7fa464284..3b0b77f6c 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.1' +__version__ = '0.27.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index e097719db..765072c3f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 6fd312833f8e6cac9fb28f0e1af96d7d8643acb6 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Tue, 17 Jul 2018 15:29:08 -0700 Subject: [PATCH 592/631] Remove setuptools min version for OVH DNS plugin (#6229) This simplifies packaging for EPEL7. See also #5617. --- certbot-dns-ovh/setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 258c7f0f1..e0ce785a1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'certbot>=0.21.1', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] From 783b6e4746ac95d8aefe3ac9d0616a4176dbd6ca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Jul 2018 17:19:04 -0700 Subject: [PATCH 593/631] Automate EBS cleanup (#6160) * ensure volume cleanup * remove volume cleanup * cleanup function and output --- tests/letstest/multitester.py | 51 +++++++++++++---------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 17740cde8..0ae9636d4 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -128,6 +128,7 @@ def make_instance(instance_name, userdata=""): #userdata contains bash or cloud-init script new_instance = EC2.create_instances( + BlockDeviceMappings=_get_block_device_mappings(ami_id), ImageId=ami_id, SecurityGroups=security_groups, KeyName=keyname, @@ -151,38 +152,21 @@ def make_instance(instance_name, raise return new_instance -def terminate_and_clean(instances): +def _get_block_device_mappings(ami_id): + """Returns the list of block device mappings to ensure cleanup. + + This list sets connected EBS volumes to be deleted when the EC2 + instance is terminated. + """ - Some AMIs specify EBS stores that won't delete on instance termination. - These must be manually deleted after shutdown. - """ - volumes_to_delete = [] - for instance in instances: - for bdmap in instance.block_device_mappings: - if 'Ebs' in bdmap.keys(): - if not bdmap['Ebs']['DeleteOnTermination']: - volumes_to_delete.append(bdmap['Ebs']['VolumeId']) - - for instance in instances: - instance.terminate() - - # can't delete volumes until all attaching instances are terminated - _ids = [instance.id for instance in instances] - all_terminated = False - while not all_terminated: - all_terminated = True - for _id in _ids: - # necessary to reinit object for boto3 to get true state - inst = EC2.Instance(id=_id) - if inst.state['Name'] != 'terminated': - all_terminated = False - time.sleep(5) - - for vol_id in volumes_to_delete: - volume = EC2.Volume(id=vol_id) - volume.delete() - - return volumes_to_delete + # Not all devices use EBS, but the default value for DeleteOnTermination + # when the device does use EBS is true. See: + # * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-blockdev-mapping.html + # * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-blockdev-template.html + return [{'DeviceName': mapping['DeviceName'], + 'Ebs': {'DeleteOnTermination': True}} + for mapping in EC2.Image(ami_id).block_device_mappings + if not mapping.get('Ebs', {}).get('DeleteOnTermination', True)] # Helper Routines @@ -370,10 +354,11 @@ def test_client_process(inqueue, outqueue): def cleanup(cl_args, instances, targetlist): print('Logs in ', LOGDIR) if not cl_args.saveinstances: - print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') + print('Terminating EC2 Instances') if cl_args.killboulder: boulder_server.terminate() - terminate_and_clean(instances) + for instance in instances: + instance.terminate() else: # print login information for the boxes for debugging for ii, target in enumerate(targetlist): From 94cadd33eb4efce19c1023346d5721703e8c88f6 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 17 Jul 2018 20:00:12 -0700 Subject: [PATCH 594/631] test(postfix): env for testing on oldest deps (#6230) Fixes #6124. --- .travis.yml | 2 +- certbot-postfix/certbot_postfix/tests/installer_test.py | 2 +- tox.ini | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e3f7a021..b671c0e8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: - python: "3.5" env: TOXENV=mypy - python: "2.7" - env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' + env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest' sudo: required services: docker - python: "3.4" diff --git a/certbot-postfix/certbot_postfix/tests/installer_test.py b/certbot-postfix/certbot_postfix/tests/installer_test.py index 1bdd2c8b3..37b78bdca 100644 --- a/certbot-postfix/certbot_postfix/tests/installer_test.py +++ b/certbot-postfix/certbot_postfix/tests/installer_test.py @@ -253,7 +253,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): fake_set.reset_mock() installer.deploy_cert("example.com", "cert_path", "key_path", "chain_path", "fullchain_path") - fake_set.assert_not_called() + self.assertFalse(fake_set.called) @certbot_test_util.patch_get_utility() def test_deploy_already_secure(self, mock_util): diff --git a/tox.ini b/tox.ini index d113fd450..0676a0da4 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,12 @@ commands = setenv = {[testenv:py27-oldest]setenv} +[testenv:py27-postfix-oldest] +commands = + {[base]install_and_test} certbot-postfix +setenv = + {[testenv:py27-oldest]setenv} + [testenv:py27_install] basepython = python2.7 commands = From cdc333491be3cc48dd17c5191f815859035bfdc7 Mon Sep 17 00:00:00 2001 From: R3DDY97 Date: Wed, 18 Jul 2018 21:04:09 +0530 Subject: [PATCH 595/631] Gpg2 doc (#5981) --- docs/install.rst | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index ead59350d..f7504baa5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -40,11 +40,17 @@ supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. + +Additional integrity verification of certbot-auto script can be done by verifying its digital signature. +This requires a local installation of gpg2, which comes packaged in many Linux distributions under name gnupg or gnupg2. + + Installing with ``certbot-auto`` requires 512MB of RAM in order to build some of the dependencies. Installing from pre-built OS packages avoids this requirement. You can also temporarily set a swap file. See "Problems with Python virtual environment" below for details. + Alternate installation methods ================================ @@ -64,12 +70,30 @@ download and run it as follows:: user@webserver:~$ chmod a+x ./certbot-auto user@webserver:~$ ./certbot-auto --help -.. hint:: The certbot-auto download is protected by HTTPS, which is pretty good, but if you'd like to - double check the integrity of the ``certbot-auto`` script, you can use these steps for verification before running it:: +To check the integrity of the ``certbot-auto`` script, +you can use these steps:: + + + user@webserver:~$ wget -N https://dl.eff.org/certbot-auto.asc + user@webserver:~$ gpg2 --keyserver pool.sks-keyservers.net --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 + user@webserver:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc certbot-auto + + + +The output of the last command should look something like:: + + + gpg: Signature made Wed 02 May 2018 05:29:12 AM IST + gpg: using RSA key A2CFB51FA275A7286234E7B24D17C995CD9775F2 + gpg: key 4D17C995CD9775F2 marked as ultimately trusted + gpg: checking the trustdb + gpg: marginals needed: 3 completes needed: 1 trust model: pgp + gpg: depth: 0 valid: 2 signed: 2 trust: 0-, 0q, 0n, 0m, 0f, 2u + gpg: depth: 1 valid: 2 signed: 0 trust: 2-, 0q, 0n, 0m, 0f, 0u + gpg: next trustdb check due at 2027-11-22 + gpg: Good signature from "Let's Encrypt Client Team " [ultimate] + - user@server:~$ wget -N https://dl.eff.org/certbot-auto.asc - user@server:~$ gpg2 --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - user@server:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc certbot-auto The ``certbot-auto`` command updates to the latest client release automatically. Since ``certbot-auto`` is a wrapper to ``certbot``, it accepts exactly From daee6e8eb3ca999840013e3469c9706e3328c55f Mon Sep 17 00:00:00 2001 From: Eli Young Date: Thu, 19 Jul 2018 10:29:34 -0700 Subject: [PATCH 596/631] Fix test naming for certbot-dns-sakuracloud (#6231) --- .../certbot_dns_sakuracloud/dns_sakuracloud_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py index 84605d06f..1d9282f9a 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py @@ -38,7 +38,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, self.auth._get_sakuracloud_client = mock.MagicMock(return_value=self.mock_client) -class NS1LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): +class SakuraCloudLexiconClientTest(unittest.TestCase, + dns_test_common_lexicon.BaseLexiconClientTest): DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) From c129ab2965f7ec4edf173b1cd900bd5add13cd2a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 23 Jul 2018 11:46:22 -0700 Subject: [PATCH 597/631] Bump the acme version needed for account reuse (#6250) * Bump the acme version needed for account reuse. Fixes https://github.com/certbot/certbot/issues/6155#issuecomment-407122742. * Update nginx oldest requirements. * bump min acme version * update min acme version --- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 2 +- local-oldest-requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index e70ac0c7f..38ed5debe 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.25.0 +acme[dev]==0.26.0 -e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 4706f17bd..02aaa6581 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -7,7 +7,7 @@ version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.25.0', + 'acme>=0.26.0', 'certbot>=0.22.0', 'mock', 'PyOpenSSL', diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 1f449acae..03226fc84 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ -acme[dev]==0.25.0 +acme[dev]==0.26.0 diff --git a/setup.py b/setup.py index fc87917fb..1827c4d42 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.25.0', + 'acme>=0.26.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From dc913574189b780e5b78f28ffaf5db47d3132436 Mon Sep 17 00:00:00 2001 From: dovreshef Date: Fri, 27 Jul 2018 18:50:53 +0300 Subject: [PATCH 598/631] Raise ConflictError on attempts to create an existing account (#6251) * Raise ConflictError on attempts to create an existing account in ACME V2. Fixes issue #6246 * Allow querying an account without calling new_account in ACMEv2 Fixed issue #6258 --- acme/acme/client.py | 17 +++++++++++++++++ acme/acme/client_test.py | 11 +++++++++++ acme/acme/errors.py | 2 ++ 3 files changed, 30 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index a0bfe460d..bd86657b9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -577,16 +577,33 @@ class ClientV2(ClientBase): :param .NewRegistration new_account: + :raises .ConflictError: in case the account already exists + :returns: Registration Resource. :rtype: `.RegistrationResource` """ response = self._post(self.directory['newAccount'], new_account) + # if account already exists + if response.status_code == 200 and 'Location' in response.headers: + raise errors.ConflictError(response.headers.get('Location')) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) self.net.account = regr return regr + def query_registration(self, regr): + """Query server about registration. + + :param messages.RegistrationResource: Existing Registration + Resource. + + """ + self.net.account = regr + updated_regr = super(ClientV2, self).query_registration(regr) + self.net.account = updated_regr + return updated_regr + def update_registration(self, regr, update=None): """Update registration. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index cd31c4ac3..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -134,6 +134,12 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.acme_version, 2) + def test_query_registration_client_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.response.json.return_value = self.regr.body.to_json() + self.assertEqual(self.regr, client.query_registration(self.regr)) + def test_forwarding(self): self.response.json.return_value = DIRECTORY_V1.to_json() client = self._init() @@ -706,6 +712,11 @@ class ClientV2Test(ClientTestBase): self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + def test_new_account_conflict(self): + self.response.status_code = http_client.OK + self.response.headers['Location'] = self.regr.uri + self.assertRaises(errors.ConflictError, self.client.new_account, self.new_reg) + def test_new_order(self): order_response = copy.deepcopy(self.response) order_response.status_code = http_client.CREATED diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 97fa73614..3a0f8c596 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -110,6 +110,8 @@ class ConflictError(ClientError): In the version of ACME implemented by Boulder, this is used to find an account if you only have the private key, but don't know the account URL. + + Also used in V2 of the ACME client for the same purpose. """ def __init__(self, location): self.location = location From ee7d5052fdf06fd364d9dffec4f89c84de84e2f8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 30 Jul 2018 16:32:31 -0700 Subject: [PATCH 599/631] Update changelog for 0.26.1 release (#6237) * Update changelog for 0.26.1 release --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d0808de6..6d384ad30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.26.1 - 2018-07-17 + +### Fixed + +* Fix a bug that was triggered when users who had previously manually set `--server` to get ACMEv2 certs tried to renew ACMEv1 certs. + +Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: + +* certbot + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/58?closed=1 + ## 0.26.0 - 2018-07-11 ### Added @@ -783,7 +796,7 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac ### Added -* When using the standalone plugin while running Certbot interactively +* When using the standalone plugin while running Certbot interactively and a required port is bound by another process, Certbot will give you the option to retry to grab the port rather than immediately exiting. * You are now able to deactivate your account with the Let's Encrypt From 68f9a1b30042cbfc7b42667829a83d23d9f89f6c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 31 Jul 2018 09:56:03 -0700 Subject: [PATCH 600/631] Files with multiple vhosts are fine. (#6273) --- certbot-apache/certbot_apache/display_ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 097b84b96..db1c3cca4 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -113,8 +113,7 @@ def _vhost_menu(domain, vhosts): code, tag = zope.component.getUtility(interfaces.IDisplay).menu( "We were unable to find a vhost with a ServerName " "or Address of {0}.{1}Which virtual host would you " - "like to choose?\n(note: conf files with multiple " - "vhosts are not yet supported)".format(domain, os.linesep), + "like to choose?".format(domain, os.linesep), choices, force_interactive=True) except errors.MissingCommandlineFlag: msg = ( From 62629213153e40054654b1a71a3138a90091f6e7 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Tue, 31 Jul 2018 19:31:36 +0200 Subject: [PATCH 601/631] bump pytest-xdist to 1.22.5 (#6253) Signed-off-by: Yoan Blanc --- .travis.yml | 2 +- tools/dev_constraints.txt | 2 +- tox.cover.sh | 4 +--- tox.ini | 15 +++++++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b671c0e8d..367e00fea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover NUMPROCESSES=2 FYI="this also tests py27" + env: TOXENV=cover FYI="this also tests py27" - sudo: required env: TOXENV=nginx_compat services: docker diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 5a16b8cba..f14169bfc 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -48,7 +48,7 @@ pylint==1.4.2 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 -pytest-xdist==1.20.1 +pytest-xdist==1.22.5 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 diff --git a/tox.cover.sh b/tox.cover.sh index 6440d1b48..bb56ddf18 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -8,8 +8,6 @@ # # -e makes sure we fail fast and don't submit coveralls submit -NUMPROCESSES=${NUMPROCESSES:=auto} - if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else @@ -63,7 +61,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "$NUMPROCESSES" --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 0676a0da4..9db06f78c 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,7 @@ source_paths = tests/lock_test.py [testenv] +passenv = TRAVIS commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -121,7 +122,6 @@ commands = [testenv:cover] basepython = python2.7 -passenv = NUMPROCESSES commands = {[base]install_packages} ./tox.cover.sh @@ -166,7 +166,9 @@ commands = docker run --rm -it apache-compat -c apache.tar.gz -vvvv whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:nginx_compat] commands = @@ -175,7 +177,9 @@ commands = docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_precise] # At the moment, this tests under Python 2.7 only, as only that version is @@ -185,7 +189,9 @@ commands = docker run --rm -t -i lea whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -198,6 +204,7 @@ whitelist_externals = docker passenv = DOCKER_* + TRAVIS TRAVIS_BRANCH [testenv:le_auto_wheezy] From f2bc876b6eddc248cd4ab021deefd2d7ef85f7b6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 31 Jul 2018 15:56:22 -0700 Subject: [PATCH 602/631] switch to codecov (#6220) --- .travis.yml | 4 ++-- README.rst | 4 ++-- tox.cover.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 367e00fea..b4d702ff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,12 +90,12 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" +install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && coveralls' +after_success: '[ "$TOXENV" == "cover" ] && codecov' notifications: email: false diff --git a/README.rst b/README.rst index a18028aee..0dbe1cdef 100644 --- a/README.rst +++ b/README.rst @@ -107,8 +107,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master - :target: https://coveralls.io/r/certbot/certbot +.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg + :target: https://codecov.io/gh/certbot/certbot :alt: Coverage status .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/ diff --git a/tox.cover.sh b/tox.cover.sh index bb56ddf18..c68e757de 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -6,7 +6,7 @@ # generate separate stats for each package. It should be removed once # those packages are moved to separate repo. # -# -e makes sure we fail fast and don't submit coveralls submit +# -e makes sure we fail fast and don't submit to codecov if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" From f6219ddf196f1de0c4d12e1bb5c1f6d737a6e844 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 22:00:47 +0300 Subject: [PATCH 603/631] Enable Apache VirtualHost for HTTP challenge validation if not enabled already (#6268) If user provides a custom --apache-vhost-root path that's not parsed by Apache per default, Certbot fails the challenge validation. While the VirtualHost on custom path is correctly found, and edited, it's still not seen by Apache. This PR adds a temporary Include directive to the root Apache configuration when writing the challenge tokens to the VirtualHost. --- certbot-apache/certbot_apache/http_01.py | 6 ++++++ .../certbot_apache/tests/http_01_test.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 37545e9cc..22598baca 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -6,6 +6,7 @@ from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-m from certbot import errors from certbot.plugins import common from certbot_apache.obj import VirtualHost # pylint: disable=unused-import +from certbot_apache.parser import get_aug_path logger = logging.getLogger(__name__) @@ -172,4 +173,9 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.parser.add_dir( vhost.path, "Include", self.challenge_conf_post) + if not vhost.enabled: + self.configurator.parser.add_dir( + get_aug_path(self.configurator.parser.loc["default"]), + "Include", vhost.filep) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 98bf412ae..9c729b08c 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -10,6 +10,7 @@ from certbot import achallenges from certbot import errors from certbot.tests import acme_util +from certbot_apache.parser import get_aug_path from certbot_apache.tests import util @@ -134,6 +135,21 @@ class ApacheHttp01Test(util.ApacheTest): def test_perform_3_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=3, minor_version=4) + def test_activate_disabled_vhost(self): + vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="certbot.demo", account_key=self.account_key)] + vhosts[0].enabled = False + self.common_perform_test(achalls, vhosts) + matches = self.config.parser.find_dir( + "Include", vhosts[0].filep, + get_aug_path(self.config.parser.loc["default"])) + self.assertEqual(len(matches), 1) + def combinations_perform_test(self, num_achalls, minor_version): """Test perform with the given achall count and Apache version.""" achalls = self.achalls[:num_achalls] From 7bff0a02e4b86db47015987bbafa3d3a93246829 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 2 Aug 2018 18:17:38 +0300 Subject: [PATCH 604/631] Make Apache control script and binary paths configurable on command line (#6238) This PR adds two new command line parameters, --apache-ctlpath and --apache-binpath both of which are used to construct commands that we shell out for. The way that we previously fetched values either from Certbot configuration object or the dictionary of distribution based constants is now also unified, and the active options are parsed in prepare() to make it easier to override needed values for the distributions needing this behavior. Fixes: #5338 * Added the command line options and parsing * Refactor existing code * Distro override updates * Handle vhost_root from cli * Fix compatibility tests * Add comment about changes to command line arguments * Check None properly * Made help texts consistent * Keep the old defaults * Move to shorter CLI parameter names * No need for specific bin path, nor apache_cmd anymore * Make sure that we use user provided vhost-root value * Fix alt restart commands in overrides * Fix version_cmd defaults in overrides * Fix comparison * Remove cruft, and use configuration object for parser parameter --- certbot-apache/certbot_apache/configurator.py | 115 +++++++++++------- .../certbot_apache/override_arch.py | 4 +- .../certbot_apache/override_centos.py | 14 ++- .../certbot_apache/override_darwin.py | 6 +- .../certbot_apache/override_debian.py | 10 +- .../certbot_apache/override_gentoo.py | 18 ++- .../certbot_apache/override_suse.py | 2 +- certbot-apache/certbot_apache/parser.py | 8 +- .../certbot_apache/tests/autohsts_test.py | 3 + .../certbot_apache/tests/centos_test.py | 2 + .../certbot_apache/tests/configurator_test.py | 33 ++--- .../certbot_apache/tests/debian_test.py | 2 +- .../certbot_apache/tests/gentoo_test.py | 2 +- .../certbot_apache/tests/parser_test.py | 6 +- certbot-apache/certbot_apache/tests/util.py | 49 ++++---- .../configurators/apache/common.py | 3 - 16 files changed, 150 insertions(+), 127 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb77e2e41..da632dc81 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import copy import fnmatch import logging import os @@ -97,48 +98,72 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) - def constant(self, key): - """Get constant for OS_DEFAULTS""" - return self.OS_DEFAULTS.get(key) + def option(self, key): + """Get a value from options""" + return self.options.get(key) + + def _prepare_options(self): + """ + Set the values possibly changed by command line parameters to + OS_DEFAULTS constant dictionary + """ + opts = ["enmod", "dismod", "le_vhost_ext", "server_root", "vhost_root", + "logs_root", "challenge_location", "handle_modules", "handle_sites", + "ctl"] + for o in opts: + # Config options use dashes instead of underscores + if self.conf(o.replace("_", "-")) is not None: + self.options[o] = self.conf(o.replace("_", "-")) + else: + self.options[o] = self.OS_DEFAULTS[o] + + # Special cases + self.options["version_cmd"][0] = self.option("ctl") + self.options["restart_cmd"][0] = self.option("ctl") + self.options["conftest_cmd"][0] = self.option("ctl") @classmethod def add_parser_arguments(cls, add): + # When adding, modifying or deleting command line arguments, be sure to + # include the changes in the list used in method _prepare_options() to + # ensure consistent behavior. add("enmod", default=cls.OS_DEFAULTS["enmod"], - help="Path to the Apache 'a2enmod' binary.") + help="Path to the Apache 'a2enmod' binary") add("dismod", default=cls.OS_DEFAULTS["dismod"], - help="Path to the Apache 'a2dismod' binary.") + help="Path to the Apache 'a2dismod' binary") add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], - help="SSL vhost configuration extension.") + help="SSL vhost configuration extension") add("server-root", default=cls.OS_DEFAULTS["server_root"], - help="Apache server root directory.") + help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") add("logs-root", default=cls.OS_DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", default=cls.OS_DEFAULTS["challenge_location"], - help="Directory path for challenge configuration.") - add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you. " + + help="Directory path for challenge configuration") + add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you. " + + help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - util.add_deprecated_argument(add, argument_name="ctl", nargs=1) + add("ctl", default=cls.OS_DEFAULTS["ctl"], + help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) @@ -169,7 +194,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser = None self.version = version self.vhosts = None - self.vhostroot = None + self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} @@ -201,12 +226,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except ImportError: raise errors.NoInstallationError("Problem in Augeas installation") + self._prepare_options() + # Verify Apache is installed - restart_cmd = self.constant("restart_cmd")[0] - if not util.exe_exists(restart_cmd): - if not path_surgery(restart_cmd): - raise errors.NoInstallationError( - 'Cannot find Apache control command {0}'.format(restart_cmd)) + self._verify_exe_availability(self.option("ctl")) # Make sure configuration is valid self.config_test() @@ -226,12 +249,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - # Parse vhost-root if defined on cli - if not self.conf("vhost-root"): - self.vhostroot = self.constant("vhost_root") - else: - self.vhostroot = os.path.abspath(self.conf("vhost-root")) - self.parser = self.get_parser() # Check for errors in parsing files with Augeas @@ -245,13 +262,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.conf("server-root")) + util.lock_dir_until_exit(self.option("server_root")) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( - "Unable to lock %s", self.conf("server-root")) + "Unable to lock %s", self.option("server_root")) self._prepared = True + def _verify_exe_availability(self, exe): + """Checks availability of Apache executable""" + if not util.exe_exists(exe): + if not path_surgery(exe): + raise errors.NoInstallationError( + 'Cannot find Apache executable {0}'.format(exe)) + def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. If augeas version is recent enough, it will support case insensitive @@ -269,8 +293,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def get_parser(self): """Initializes the ApacheParser""" + # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.conf("vhost-root"), self.version, configurator=self) def _wildcard_domain(self, domain): @@ -1037,7 +1062,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param boolean temp: If the change is temporary """ - if self.conf("handle-modules"): + if self.option("handle_modules"): if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1066,7 +1091,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.constant("le_vhost_ext")`` + ``self.option("le_vhost_ext")`` .. note:: This function saves the configuration @@ -1165,18 +1190,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - # Defined by user on CLI - - fp = os.path.join(os.path.realpath(self.vhostroot), + fp = os.path.join(os.path.realpath(self.option("vhost_root")), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = os.path.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.conf("le_vhost_ext") + return fp[:-(len(".conf"))] + self.option("le_vhost_ext") else: - return fp + self.conf("le_vhost_ext") + return fp + self.option("le_vhost_ext") def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2025,7 +2048,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.conf("logs-root"))) + self.option("logs_root"))) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2037,7 +2060,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.vhostroot, + redirect_filepath = os.path.join(self.option("vhost_root"), redirect_filename) # Register the new file that will be created @@ -2158,18 +2181,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ error = "" try: - util.run_script(self.constant("restart_cmd")) + util.run_script(self.option("restart_cmd")) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.constant("restart_cmd")) - alt_restart = self.constant("restart_cmd_alt") + self.option("restart_cmd")) + alt_restart = self.option("restart_cmd_alt") if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.constant( + util.run_script(self.option( "restart_cmd_alt")) return except errors.SubprocessError as secerr: @@ -2185,7 +2208,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - util.run_script(self.constant("conftest_cmd")) + util.run_script(self.option("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2201,11 +2224,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = util.run_script(self.constant("version_cmd")) + stdout, _ = util.run_script(self.option("version_cmd")) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.constant("version_cmd")) + self.option("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) @@ -2295,7 +2318,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # certbot for unprivileged users via setuid), this function will need # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/override_arch.py b/certbot-apache/certbot_apache/override_arch.py index ea5155a3c..c5620e9f9 100644 --- a/certbot-apache/certbot_apache/override_arch.py +++ b/certbot-apache/certbot_apache/override_arch.py @@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index 0b6b12b96..a4f1b84ec 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -18,25 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "centos-options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in CentOS. + """ + super(CentOSConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) diff --git a/certbot-apache/certbot_apache/override_darwin.py b/certbot-apache/certbot_apache/override_darwin.py index 53741d504..4e2a6acac 100644 --- a/certbot-apache/certbot_apache/override_darwin.py +++ b/certbot-apache/certbot_apache/override_darwin.py @@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/other", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/httpd', '-v'], - apache_cmd="/usr/sbin/httpd", + ctl="apachectl", + version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 02dffc3f7..0caa619d2 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -23,14 +23,14 @@ class DebianConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=True, + handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( @@ -134,11 +134,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.conf("dismod")): + if not util.exe_exists(self.option("dismod")): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), "-f", mod_name]) - util.run_script([self.conf("enmod"), mod_name]) + temp, [self.option("dismod"), "-f", mod_name]) + util.run_script([self.option("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 165e44c96..556e3225e 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -18,25 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/apache2', '-v'], - apache_cmd="apache2ctl", + ctl="apache2ctl", + version_cmd=['apache2ctl', '-v'], restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in Gentoo. + """ + super(GentooConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) @@ -61,7 +69,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + mod_cmd = [self.configurator.option("ctl"), "modules"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py index a67054b5b..83079b92c 100644 --- a/certbot-apache/certbot_apache/override_suse.py +++ b/certbot-apache/certbot_apache/override_suse.py @@ -16,8 +16,8 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 02337f8d4..148f052d0 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -69,7 +69,7 @@ class ApacheParser(object): # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.constant("vhost_files")) + self.configurator.option("vhost_files")) # check to see if there were unparsed define statements if version < (2, 4): @@ -152,7 +152,7 @@ class ApacheParser(object): """Get Defines from httpd process""" variables = dict() - define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + define_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_RUN_CFG"] matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") try: @@ -179,7 +179,7 @@ class ApacheParser(object): # configuration files _ = self.find_dir("Include") - inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + inc_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_INCLUDES"] matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)") if matches: @@ -190,7 +190,7 @@ class ApacheParser(object): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + mod_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_MODULES"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 73da33f15..c5d720dd3 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -119,6 +119,9 @@ class AutoHSTSTest(util.ApacheTest): cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), cur_val) + # Ensure that the value is raised to max + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + maxage.format(constants.AUTOHSTS_STEPS[-1])) # Make permanent self.config.deploy_autohsts(mock_lineage) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 4ee8b5dcf..46b857c3e 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -135,5 +135,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): errors.SubprocessError, errors.SubprocessError] self.assertRaises(errors.MisconfigurationError, self.config.restart) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 350262634..6f1c358c2 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -116,8 +116,9 @@ class MultipleVhostsTest(util.ApacheTest): ApacheConfigurator.add_parser_arguments(mock.MagicMock()) def test_constant(self): - self.assertEqual(self.config.constant("server_root"), "/etc/apache2") - self.assertEqual(self.config.constant("nonexistent"), None) + self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in + self.config.option("server_root")) + self.assertEqual(self.config.option("nonexistent"), None) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -651,22 +652,10 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost_slink.name, "nonsym.link") def test_make_vhost_ssl_nonexistent_vhost_path(self): - def conf_side_effect(arg): - """ Mock function for ApacheConfigurator.conf """ - confvars = { - "vhost-root": "/tmp/nonexistent", - "le_vhost_ext": "-le-ssl.conf", - "handle-sites": True} - return confvars[arg] - - with mock.patch( - "certbot_apache.configurator.ApacheConfigurator.conf" - ) as mock_conf: - mock_conf.side_effect = conf_side_effect - ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) - self.assertEqual(os.path.dirname(ssl_vhost.filep), - os.path.dirname(os.path.realpath( - self.vh_truth[1].filep))) + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) + self.assertEqual(os.path.dirname(ssl_vhost.filep), + os.path.dirname(os.path.realpath( + self.vh_truth[1].filep))) def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -1583,7 +1572,7 @@ class AugeasVhostsTest(util.ApacheTest): broken_vhost) class MultiVhostsTest(util.ApacheTest): - """Test vhosts with illegal names dependent on augeas version.""" + """Test configuration with multiple virtualhosts in a single file.""" # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ @@ -1703,7 +1692,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1739,7 +1728,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1755,7 +1744,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py index fde8d4c35..bb1d64278 100644 --- a/certbot-apache/certbot_apache/tests/debian_test.py +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ super(MultipleVhostsTestDebian, self).setUp() self.config = util.get_apache_configurator( - self.config_path, None, self.config_dir, self.work_dir, + self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth(self.temp_dir, diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index d32551267..0681e30b5 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.modules = set() with mock.patch("certbot.util.get_os_info") as mock_osi: - # Make sure we have the have the CentOS httpd constants + # Make sure we have the have the Gentoo httpd constants mock_osi.return_value = ("gentoo", "123") self.config.parser.update_runtime_variables() diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index f95f1b346..d62fd54e8 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -282,11 +282,11 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): mock_popen.side_effect = OSError - mock_const.return_value = "nonexistent" + mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 6d3cfa109..9329ccb20 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -97,9 +97,10 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc backups = os.path.join(work_dir, "backups") mock_le_config = mock.MagicMock( apache_server_root=config_path, - apache_vhost_root=conf_vhost_path, + apache_vhost_root=None, apache_le_vhost_ext="-le-ssl.conf", apache_challenge_location=config_path, + apache_enmod=None, backup_dir=backups, config_dir=config_dir, http01_port=80, @@ -107,33 +108,25 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) - orig_os_constant = configurator.ApacheConfigurator(mock_le_config, - name="apache", - version=version).constant - - def mock_os_constant(key, vhost_path=vhost_path): - """Mock default vhost path""" - if key == "vhost_root": - return vhost_path - else: - return orig_os_constant(key) - - with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons: - mock_cons.side_effect = mock_os_constant - with mock.patch("certbot_apache.configurator.util.run_script"): - with mock.patch("certbot_apache.configurator.util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - with mock.patch("certbot_apache.parser.ApacheParser." - "update_runtime_variables"): - try: - config_class = entrypoint.OVERRIDE_CLASSES[os_info] - except KeyError: - config_class = configurator.ApacheConfigurator - config = config_class(config=mock_le_config, name="apache", - version=version) - - config.prepare() + with mock.patch("certbot_apache.configurator.util.run_script"): + with mock.patch("certbot_apache.configurator.util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + with mock.patch("certbot_apache.parser.ApacheParser." + "update_runtime_variables"): + try: + config_class = entrypoint.OVERRIDE_CLASSES[os_info] + except KeyError: + config_class = configurator.ApacheConfigurator + config = config_class(config=mock_le_config, name="apache", + version=version) + if not conf_vhost_path: + config_class.OS_DEFAULTS["vhost_root"] = vhost_path + else: + # Custom virtualhost path was requested + config.config.apache_vhost_root = conf_vhost_path + config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.prepare() return config diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 1d2cfdeca..82195264b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy): setattr(self.le_config, "apache_" + k, entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) - # An alias - self.le_config.apache_handle_modules = self.le_config.apache_handle_mods - self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), name="apache") From 4989668e0aaec8ff7d03661846d3340ca6b245af Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 3 Aug 2018 17:55:12 -0400 Subject: [PATCH 605/631] Clarify that configurations can be invalid (#6277) --- certbot/cert_manager.py | 2 +- certbot/renewal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d1205835a..771ca8caf 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -353,7 +353,7 @@ def _describe_certs(config, parsed_certs, parse_failures): notify("Found the following {0}certs:".format(match)) notify(_report_human_readable(config, parsed_certs)) if parse_failures: - notify("\nThe following renewal configuration files " + notify("\nThe following renewal configurations " "were invalid:") notify(_report_lines(parse_failures)) diff --git a/certbot/renewal.py b/certbot/renewal.py index f50131028..ecc8b1f2f 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -359,7 +359,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, notify_error(report(renew_failures, "failure")) if parse_failures: - notify("\nAdditionally, the following renewal configuration files " + notify("\nAdditionally, the following renewal configurations " "were invalid: ") notify(report(parse_failures, "parsefail")) From d8057f0e17dc757fae662dad91a6fedc96ad6a2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 6 Aug 2018 09:45:56 -0700 Subject: [PATCH 606/631] Fix Sphinx (#6070) Fixes #4686. In Sphinx 1.6, they changed how they handle images in latex and PDF files. You can learn more about this by reading the linked issue (or I can answer any questions), but the shortish version is we now need to use the extension sphinx.ext.imgconverter. This is only available in Sphinx 1.6+. I also updated our pinned versions to use the latest Sphinx and a new dependency it pulled in called sphinxcontrib-websupport. To build the latex and PDF docs, you must first run: apt-get install imagemagick latexmk texlive texlive-latex-extra Afterwards, if you create the normal Certbot dev environment using this branch, activate the virtual environment, and from the root of the repo run make -C docs clean latex latexpdf, you'll successfully build the PDF docs. * fix #4686 * bump minimum Sphinx req --- docs/conf.py | 1 + setup.py | 4 ++-- tools/dev_constraints.txt | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 09bb44285..2e6c5a9b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ needs_sphinx = '1.0' # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.imgconverter', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', diff --git a/setup.py b/setup.py index 1827c4d42..a13b7cdb9 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,8 @@ dev3_extras = [ docs_extras = [ 'repoze.sphinx.autointerface', - # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 - 'Sphinx >=1.0,<=1.5.6', + # sphinx.ext.imgconverter + 'Sphinx >=1.6', 'sphinx_rtd_theme', ] diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f14169bfc..ef7804328 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -60,8 +60,9 @@ s3transfer==0.1.11 scandir==1.6 simplegeneric==0.8.1 snowballstemmer==1.2.1 -Sphinx==1.5.6 +Sphinx==1.7.5 sphinx-rtd-theme==0.2.4 +sphinxcontrib-websupport==1.0.1 tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 From b1003b7250fe0b53a683d1a48b732130f0d5aa99 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 16 Aug 2018 21:28:25 +0200 Subject: [PATCH 607/631] Fail fast during tests if python executable is not in the PATH (#6306) --- tests/boulder-integration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2f9130489..fbe8d26aa 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -10,6 +10,9 @@ set -eux +# Check that python executable is available in the PATH. Fail immediatly if not. +command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) + . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx From 0e9dd5e3d21d3907733cac7ba9bdf3f7a27105db Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 21 Aug 2018 08:59:56 -0700 Subject: [PATCH 608/631] Remove visible warnings about missing apachectl. (#6307) --- certbot/plugins/util.py | 4 ++-- certbot/plugins/util_test.py | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..3f03bf375 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -51,6 +51,6 @@ def path_surgery(cmd): return True else: expanded = " expanded" if any(added) else "" - logger.warning("Failed to find executable %s in%s PATH: %s", cmd, - expanded, path) + logger.debug("Failed to find executable %s in%s PATH: %s", cmd, + expanded, path) return False diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..9757d8de7 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -16,9 +16,8 @@ class GetPrefixTest(unittest.TestCase): class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" - @mock.patch("certbot.plugins.util.logger.warning") @mock.patch("certbot.plugins.util.logger.debug") - def test_path_surgery(self, mock_debug, mock_warn): + def test_path_surgery(self, mock_debug): from certbot.plugins.util import path_surgery all_path = {"PATH": "/usr/local/bin:/bin/:/usr/sbin/:/usr/local/sbin/"} with mock.patch.dict('os.environ', all_path): @@ -26,14 +25,12 @@ class PathSurgeryTest(unittest.TestCase): mock_exists.return_value = True self.assertEqual(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) - self.assertEqual(mock_warn.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) no_path = {"PATH": "/tmp/"} with mock.patch.dict('os.environ', no_path): path_surgery("thingy") - self.assertEqual(mock_debug.call_count, 1) - self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("Failed to find" in mock_warn.call_args[0][0]) + self.assertEqual(mock_debug.call_count, 2) + self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) self.assertTrue("/usr/local/bin" in os.environ["PATH"]) self.assertTrue("/tmp" in os.environ["PATH"]) From 6e23b81dba7566267c98e4b946bbc90f755b562d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 29 Aug 2018 14:11:13 -0700 Subject: [PATCH 609/631] Separate integration (#5814) Main piece of #5810. * Rename Certbot integration tests * Remove nginx from certbot tests * allow for running individual integration tests * fail under 65 * Add set -e * Track Nginx coverage and omit it from report later. * Use INTEGRATION_TEST in script * add INTEGRATION_TEST=all * update min certbot percentage --- .travis.yml | 4 +- tests/boulder-integration.sh | 505 +-------------------------- tests/certbot-boulder-integration.sh | 492 ++++++++++++++++++++++++++ 3 files changed, 505 insertions(+), 496 deletions(-) create mode 100755 tests/certbot-boulder-integration.sh diff --git a/.travis.yml b/.travis.yml index b4d702ff1..acdf365bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v2 + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index fbe8d26aa..3e16fcbbc 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -1,499 +1,16 @@ #!/bin/bash -# Simple integration test. Make sure to activate virtualenv beforehand -# (source venv/bin/activate) and that you are running Boulder test -# instance (see ./boulder-fetch.sh). -# -# Environment variables: -# SERVER: Passed as "certbot --server" argument. -# -# Note: this script is called by Boulder integration test suite! -set -eux +set -e -# Check that python executable is available in the PATH. Fail immediatly if not. -command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) - -. ./tests/integration/_common.sh -export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx - -cleanup_and_exit() { - EXIT_STATUS=$? - if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` - then - echo Kill server subprocess, left running by abnormal exit - kill $SERVER_STILL_RUNNING - fi - if [ -f "$HOOK_DIRS_TEST" ]; then - rm -f "$HOOK_DIRS_TEST" - fi - exit $EXIT_STATUS -} - -trap cleanup_and_exit EXIT - -export HOOK_DIRS_TEST="$(mktemp)" -renewal_hooks_root="$config_dir/renewal-hooks" -renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) -renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" -renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" -renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" - -# Creates hooks in Certbot's renewal hook directory that write to a file -CreateDirHooks() { - for hook_dir in $renewal_hooks_dirs; do - mkdir -p $hook_dir - hook_path="$hook_dir/hook.sh" - cat << EOF > "$hook_path" -#!/bin/bash -xe -if [ "\$0" = "$renewal_dir_deploy_hook" ]; then - if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then - echo "Environment variables not properly set!" >&2 - exit 1 +if [ "$INTEGRATION_TEST" = "certbot" ]; then + tests/certbot-boulder-integration.sh +elif [ "$INTEGRATION_TEST" = "nginx" ]; then + certbot-nginx/tests/boulder-integration.sh +else + tests/certbot-boulder-integration.sh + # Most CI systems set this variable to true. + # If the tests are running as part of CI, Nginx should be available. + if ${CI:-false} || type nginx; then + certbot-nginx/tests/boulder-integration.sh fi fi -echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" -EOF - chmod +x "$hook_path" - done -} - -# Asserts that the hooks created by CreateDirHooks have been run once and -# resets the file. -# -# Arguments: -# The number of times the deploy hook should have been run. (It should run -# once for each certificate that was issued in that run of Certbot.) -CheckDirHooks() { - expected="pre\n" - for ((i=0; i<$1; i++)); do - expected=$expected"deploy\n" - done - expected=$expected"post" - - if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then - echo "Unexpected directory hook output!" >&2 - echo "Expected:" >&2 - echo -e "$expected" >&2 - echo "Got:" >&2 - cat "$HOOK_DIRS_TEST" >&2 - exit 1 - fi - - rm -f "$HOOK_DIRS_TEST" - export HOOK_DIRS_TEST="$(mktemp)" -} - -common_no_force_renew() { - certbot_test_no_force_renew \ - --authenticator standalone \ - --installer null \ - "$@" -} - -common() { - common_no_force_renew \ - --renew-by-default \ - "$@" -} - -export HOOK_TEST="/tmp/hook$$" -CheckHooks() { - if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then - expected="wtf.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf2.pre\n" - else - expected=$expected"wtf2.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" - else - expected="wtf2.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf.pre\n" - else - expected=$expected"wtf.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" - fi - - if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then - echo Hooks did not run as expected\; got >&2 - cat "$HOOK_TEST" >&2 - echo -e "Expected\n$expected" >&2 - rm "$HOOK_TEST" - exit 1 - fi - rm "$HOOK_TEST" -} - -# Checks if deploy is in the hook output and deletes the file -DeployInHookOutput() { - CONTENTS=$(cat "$HOOK_TEST") - rm "$HOOK_TEST" - grep deploy <(echo "$CONTENTS") -} - -# Asserts that there is a saved renew_hook for a lineage. -# -# Arguments: -# Name of lineage to check -CheckSavedRenewHook() { - if ! grep renew_hook "$config_dir/renewal/$1.conf"; then - echo "Hook wasn't saved as renew_hook" >&2 - exit 1 - fi -} - -# Asserts the deploy hook was properly run and saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -CheckDeployHook() { - if ! DeployInHookOutput; then - echo "The deploy hook wasn't run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Asserts the renew hook wasn't run but was saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -# Asserts the deploy hook wasn't run and deletes the hook file -CheckRenewHook() { - if DeployInHookOutput; then - echo "The renew hook was incorrectly run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Return success only if input contains exactly $1 lines of text, of -# which $2 different values occur in the first field. -TotalAndDistinctLines() { - total=$1 - distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' -} - -# Cleanup coverage data -coverage erase - -# test for regressions of #4719 -get_num_tmp_files() { - ls -1 /tmp | wc -l -} -num_tmp_files=$(get_num_tmp_files) -common --csr / && echo expected error && exit 1 || true -common --help -common --help all -common --version -if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then - echo "New files or directories created in /tmp!" - exit 1 -fi -CreateDirHooks - -common register -for dir in $renewal_hooks_dirs; do - if [ ! -d "$dir" ]; then - echo "Hook directory not created by Certbot!" >&2 - exit 1 - fi -done - -common unregister - -common register --email ex1@domain.org,ex2@domain.org - -common register --update-registration --email ex1@domain.org - -common register --update-registration --email ex1@domain.org,ex2@domain.org - -common plugins --init --prepare | grep webroot - -# We start a server listening on the port for the -# unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & -python_server_pid=$! - -certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! -certname="le2.wtf" -common --domains le2.wtf --preferred-challenges http-01 run \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -certname="le.wtf" -common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ - --manual-auth-hook ./tests/manual-http-auth.sh \ - --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ - --cert-name $certname \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -common certonly --cert-name newname -d newname.le.wtf - -export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ - OPENSSL_CNF=examples/openssl.cnf -./examples/generate-csr.sh le3.wtf -common auth --csr "$CSR_PATH" \ - --cert-path "${root}/csr/cert.pem" \ - --chain-path "${root}/csr/chain.pem" -openssl x509 -in "${root}/csr/cert.pem" -text -openssl x509 -in "${root}/csr/chain.pem" -text - -common --domains le3.wtf install \ - --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/key.pem" - -CheckCertCount() { - CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` - if [ "$CERTCOUNT" -ne "$2" ] ; then - echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` - exit 1 - fi -} - -CheckCertCount "le.wtf" 1 -# This won't renew (because it's not time yet) -common_no_force_renew renew -CheckCertCount "le.wtf" 1 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed for non-renewal!" >&2; - exit 1 -fi - -rm -rf "$renewal_hooks_root" -# renew using HTTP manual auth hooks -common renew --cert-name le.wtf --authenticator manual -CheckCertCount "le.wtf" 2 - -# test renewal with no executables in hook directories -for hook_dir in $renewal_hooks_dirs; do - touch "$hook_dir/file" - mkdir "$hook_dir/dir" -done -# renew using DNS manual auth hooks -common renew --cert-name dns.le.wtf --authenticator manual -CheckCertCount "dns.le.wtf" 2 - -# test with disabled directory hooks -rm -rf "$renewal_hooks_root" -CreateDirHooks -# This will renew because the expiry is less than 10 years from now -sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" -common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks -CheckCertCount "le.wtf" 3 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed with --no-directory-hooks!" >&2 - exit 1 -fi - -# The 4096 bit setting should persist to the first renewal, but be overridden in the second - -size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` -size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` -size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` -# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes -if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then - echo key sizes violate assumptions: - ls -l "${root}/conf/archive/le.wtf/privkey"* - exit 1 -fi - -# --renew-by-default is used, so renewal should occur -[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" -common renew -CheckCertCount "le.wtf" 4 -CheckHooks -CheckDirHooks 5 - -# test with overlapping directory hooks on the command line -common renew --cert-name le2.wtf \ - --pre-hook "$renewal_dir_pre_hook" \ - --deploy-hook "$renewal_dir_deploy_hook" \ - --post-hook "$renewal_dir_post_hook" -CheckDirHooks 1 - -# test with overlapping directory hooks in the renewal conf files -common renew --cert-name le2.wtf -CheckDirHooks 1 - -# manual-dns-auth.sh will skip completing the challenge for domains that begin -# with fail. -common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ - --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh - -if common certificates | grep "fail\.dns1\.le\.wtf"; then - echo "certificate should not have been issued for domain!" >&2 - exit 1 -fi - -# reuse-key -common --domains reusekey.le.wtf --reuse-key -common renew --cert-name reusekey.le.wtf -CheckCertCount "reusekey.le.wtf" 2 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# The final awk command here exits successfully if its input consists of -# exactly two lines with identical first fields, and unsuccessfully otherwise. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 - -# don't reuse key (just by forcing reissuance without --reuse-key) -common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal -CheckCertCount "reusekey.le.wtf" 3 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# Exactly three lines, of which exactly two identical first fields. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 - -# Nonetheless, all three certificates are different even though two of them -# share the same subject key. -sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 - -# ECDSA -openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" -SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ - -config "${OPENSSL_CNF:-openssl.cnf}" \ - -key "${root}/privkey-p384.pem" \ - -subj "/" \ - -reqexts san \ - -outform der \ - -out "${root}/csr-p384.der" -common auth --csr "${root}/csr-p384.der" \ - --cert-path "${root}/csr/cert-p384.pem" \ - --chain-path "${root}/csr/chain-p384.pem" -openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' - -# OCSP Must Staple -common auth --must-staple --domains "must-staple.le.wtf" -openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' - -# revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke -# revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke -if [ ! -d "$root/conf/live/le1.wtf" ]; then - echo "cert deleted when --no-delete-after-revoke was used!" - exit 1 -fi -common delete --cert-name le1.wtf -# revoke by cert key -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" - -# Get new certs to test revoke with a reason, by account and by cert key -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ - --reason cessationOfOperation -common --domains le2.wtf -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" \ - --reason keyCompromise - -common unregister - -out=$(common certificates) -subdomains="le dns.le newname.le must-staple.le" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if ! echo $out | grep "$domain"; then - echo "$domain not in certificates output!" - exit 1; - fi -done - -# Testing that revocation also deletes by default -subdomains="le1 le2" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if echo $out | grep "$domain"; then - echo "Revoked $domain in certificates output! Should not be!" - exit 1; - fi -done - -# Test that revocation raises correct error if --cert-name and --cert-path don't match -common --domains le1.wtf -common --domains le2.wtf -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true -if ! echo $out | grep "or both must point to the same certificate lineages."; then - echo "Non-interactive revoking with mismatched --cert-name and --cert-path " - echo "did not raise the correct error!" - exit 1 -fi - -# Revoking by matching --cert-name and --cert-path deletes -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" -out=$(common certificates) -if echo $out | grep "le1.wtf"; then - echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" - exit 1 -fi - -# Test that revocation doesn't delete if multiple lineages share an archive dir -common --domains le1.wtf -common --domains le2.wtf -sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" -#common update_symlinks # not needed, but a bit more context for what this test is about -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") -if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then - echo "Deleted a cert that had an overlapping archive dir with another lineage!" - exit 1 -fi - -cert_name="must-staple.le.wtf" -common delete --cert-name $cert_name -archive="$root/conf/archive/$cert_name" -conf="$root/conf/renewal/$cert_name.conf" -live="$root/conf/live/$cert_name" -for path in $archive $conf $live; do - if [ -e $path ]; then - echo "Lineage not properly deleted!" - exit 1 - fi -done - -# Test ACMEv2-only features -if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then - common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh -fi - -coverage report --fail-under 65 --include 'certbot/*' --show-missing - -# Most CI systems set this variable to true. -# If the tests are running as part of CI, Nginx should be available. -if ${CI:-false} || type nginx; -then - . ./certbot-nginx/tests/boulder-integration.sh -fi diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh new file mode 100755 index 000000000..8b8b931e5 --- /dev/null +++ b/tests/certbot-boulder-integration.sh @@ -0,0 +1,492 @@ +#!/bin/bash +# Simple integration test. Make sure to activate virtualenv beforehand +# (source venv/bin/activate) and that you are running Boulder test +# instance (see ./boulder-fetch.sh). +# +# Environment variables: +# SERVER: Passed as "certbot --server" argument. +# +# Note: this script is called by Boulder integration test suite! + +set -eux + +# Check that python executable is available in the PATH. Fail immediatly if not. +command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) + +. ./tests/integration/_common.sh +export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx + +cleanup_and_exit() { + EXIT_STATUS=$? + if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` + then + echo Kill server subprocess, left running by abnormal exit + kill $SERVER_STILL_RUNNING + fi + if [ -f "$HOOK_DIRS_TEST" ]; then + rm -f "$HOOK_DIRS_TEST" + fi + exit $EXIT_STATUS +} + +trap cleanup_and_exit EXIT + +export HOOK_DIRS_TEST="$(mktemp)" +renewal_hooks_root="$config_dir/renewal-hooks" +renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) +renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" +renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" +renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" + +# Creates hooks in Certbot's renewal hook directory that write to a file +CreateDirHooks() { + for hook_dir in $renewal_hooks_dirs; do + mkdir -p $hook_dir + hook_path="$hook_dir/hook.sh" + cat << EOF > "$hook_path" +#!/bin/bash -xe +if [ "\$0" = "$renewal_dir_deploy_hook" ]; then + if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then + echo "Environment variables not properly set!" >&2 + exit 1 + fi +fi +echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" +EOF + chmod +x "$hook_path" + done +} + +# Asserts that the hooks created by CreateDirHooks have been run once and +# resets the file. +# +# Arguments: +# The number of times the deploy hook should have been run. (It should run +# once for each certificate that was issued in that run of Certbot.) +CheckDirHooks() { + expected="pre\n" + for ((i=0; i<$1; i++)); do + expected=$expected"deploy\n" + done + expected=$expected"post" + + if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then + echo "Unexpected directory hook output!" >&2 + echo "Expected:" >&2 + echo -e "$expected" >&2 + echo "Got:" >&2 + cat "$HOOK_DIRS_TEST" >&2 + exit 1 + fi + + rm -f "$HOOK_DIRS_TEST" + export HOOK_DIRS_TEST="$(mktemp)" +} + +common_no_force_renew() { + certbot_test_no_force_renew \ + --authenticator standalone \ + --installer null \ + "$@" +} + +common() { + common_no_force_renew \ + --renew-by-default \ + "$@" +} + +export HOOK_TEST="/tmp/hook$$" +CheckHooks() { + if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then + expected="wtf.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf2.pre\n" + else + expected=$expected"wtf2.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" + else + expected="wtf2.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf.pre\n" + else + expected=$expected"wtf.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" + fi + + if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then + echo Hooks did not run as expected\; got >&2 + cat "$HOOK_TEST" >&2 + echo -e "Expected\n$expected" >&2 + rm "$HOOK_TEST" + exit 1 + fi + rm "$HOOK_TEST" +} + +# Checks if deploy is in the hook output and deletes the file +DeployInHookOutput() { + CONTENTS=$(cat "$HOOK_TEST") + rm "$HOOK_TEST" + grep deploy <(echo "$CONTENTS") +} + +# Asserts that there is a saved renew_hook for a lineage. +# +# Arguments: +# Name of lineage to check +CheckSavedRenewHook() { + if ! grep renew_hook "$config_dir/renewal/$1.conf"; then + echo "Hook wasn't saved as renew_hook" >&2 + exit 1 + fi +} + +# Asserts the deploy hook was properly run and saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +CheckDeployHook() { + if ! DeployInHookOutput; then + echo "The deploy hook wasn't run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Asserts the renew hook wasn't run but was saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +# Asserts the deploy hook wasn't run and deletes the hook file +CheckRenewHook() { + if DeployInHookOutput; then + echo "The renew hook was incorrectly run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + +# Cleanup coverage data +coverage erase + +# test for regressions of #4719 +get_num_tmp_files() { + ls -1 /tmp | wc -l +} +num_tmp_files=$(get_num_tmp_files) +common --csr / && echo expected error && exit 1 || true +common --help +common --help all +common --version +if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then + echo "New files or directories created in /tmp!" + exit 1 +fi +CreateDirHooks + +common register +for dir in $renewal_hooks_dirs; do + if [ ! -d "$dir" ]; then + echo "Hook directory not created by Certbot!" >&2 + exit 1 + fi +done + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org + +common plugins --init --prepare | grep webroot + +# We start a server listening on the port for the +# unrequested challenge to prevent regressions in #3601. +python ./tests/run_http_server.py $http_01_port & +python_server_pid=$! + +certname="le1.wtf" +common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +python ./tests/run_http_server.py $tls_sni_01_port & +python_server_pid=$! +certname="le2.wtf" +common --domains le2.wtf --preferred-challenges http-01 run \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +certname="le.wtf" +common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ + --manual-auth-hook ./tests/manual-http-auth.sh \ + --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +certname="dns.le.wtf" +common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ + --cert-name $certname \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +common certonly --cert-name newname -d newname.le.wtf + +export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ + OPENSSL_CNF=examples/openssl.cnf +./examples/generate-csr.sh le3.wtf +common auth --csr "$CSR_PATH" \ + --cert-path "${root}/csr/cert.pem" \ + --chain-path "${root}/csr/chain.pem" +openssl x509 -in "${root}/csr/cert.pem" -text +openssl x509 -in "${root}/csr/chain.pem" -text + +common --domains le3.wtf install \ + --cert-path "${root}/csr/cert.pem" \ + --key-path "${root}/key.pem" + +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` + if [ "$CERTCOUNT" -ne "$2" ] ; then + echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` + exit 1 + fi +} + +CheckCertCount "le.wtf" 1 +# This won't renew (because it's not time yet) +common_no_force_renew renew +CheckCertCount "le.wtf" 1 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed for non-renewal!" >&2; + exit 1 +fi + +rm -rf "$renewal_hooks_root" +# renew using HTTP manual auth hooks +common renew --cert-name le.wtf --authenticator manual +CheckCertCount "le.wtf" 2 + +# test renewal with no executables in hook directories +for hook_dir in $renewal_hooks_dirs; do + touch "$hook_dir/file" + mkdir "$hook_dir/dir" +done +# renew using DNS manual auth hooks +common renew --cert-name dns.le.wtf --authenticator manual +CheckCertCount "dns.le.wtf" 2 + +# test with disabled directory hooks +rm -rf "$renewal_hooks_root" +CreateDirHooks +# This will renew because the expiry is less than 10 years from now +sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" +common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks +CheckCertCount "le.wtf" 3 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed with --no-directory-hooks!" >&2 + exit 1 +fi + +# The 4096 bit setting should persist to the first renewal, but be overridden in the second + +size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes +if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then + echo key sizes violate assumptions: + ls -l "${root}/conf/archive/le.wtf/privkey"* + exit 1 +fi + +# --renew-by-default is used, so renewal should occur +[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" +common renew +CheckCertCount "le.wtf" 4 +CheckHooks +CheckDirHooks 5 + +# test with overlapping directory hooks on the command line +common renew --cert-name le2.wtf \ + --pre-hook "$renewal_dir_pre_hook" \ + --deploy-hook "$renewal_dir_deploy_hook" \ + --post-hook "$renewal_dir_post_hook" +CheckDirHooks 1 + +# test with overlapping directory hooks in the renewal conf files +common renew --cert-name le2.wtf +CheckDirHooks 1 + +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + +# ECDSA +openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" +SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ + -config "${OPENSSL_CNF:-openssl.cnf}" \ + -key "${root}/privkey-p384.pem" \ + -subj "/" \ + -reqexts san \ + -outform der \ + -out "${root}/csr-p384.der" +common auth --csr "${root}/csr-p384.der" \ + --cert-path "${root}/csr/cert-p384.pem" \ + --chain-path "${root}/csr/chain-p384.pem" +openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' + +# OCSP Must Staple +common auth --must-staple --domains "must-staple.le.wtf" +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' + +# revoke by account key +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke +# revoke renewed +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke +if [ ! -d "$root/conf/live/le1.wtf" ]; then + echo "cert deleted when --no-delete-after-revoke was used!" + exit 1 +fi +common delete --cert-name le1.wtf +# revoke by cert key +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" + +# Get new certs to test revoke with a reason, by account and by cert key +common --domains le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ + --reason cessationOfOperation +common --domains le2.wtf +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" \ + --reason keyCompromise + +common unregister + +out=$(common certificates) +subdomains="le dns.le newname.le must-staple.le" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if ! echo $out | grep "$domain"; then + echo "$domain not in certificates output!" + exit 1; + fi +done + +# Testing that revocation also deletes by default +subdomains="le1 le2" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if echo $out | grep "$domain"; then + echo "Revoked $domain in certificates output! Should not be!" + exit 1; + fi +done + +# Test that revocation raises correct error if --cert-name and --cert-path don't match +common --domains le1.wtf +common --domains le2.wtf +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true +if ! echo $out | grep "or both must point to the same certificate lineages."; then + echo "Non-interactive revoking with mismatched --cert-name and --cert-path " + echo "did not raise the correct error!" + exit 1 +fi + +# Revoking by matching --cert-name and --cert-path deletes +common --domains le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" +out=$(common certificates) +if echo $out | grep "le1.wtf"; then + echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" + exit 1 +fi + +# Test that revocation doesn't delete if multiple lineages share an archive dir +common --domains le1.wtf +common --domains le2.wtf +sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" +#common update_symlinks # not needed, but a bit more context for what this test is about +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") +if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then + echo "Deleted a cert that had an overlapping archive dir with another lineage!" + exit 1 +fi + +cert_name="must-staple.le.wtf" +common delete --cert-name $cert_name +archive="$root/conf/archive/$cert_name" +conf="$root/conf/renewal/$cert_name.conf" +live="$root/conf/live/$cert_name" +for path in $archive $conf $live; do + if [ -e $path ]; then + echo "Lineage not properly deleted!" + exit 1 + fi +done + +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh +fi + +coverage report --fail-under 64 --include 'certbot/*' --show-missing From 405a8b426422fbc0a885e04a8b4b36d1b608258f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 29 Aug 2018 15:15:57 -0700 Subject: [PATCH 610/631] Pin the real oldest requirement for nginx tests. (#6327) --- certbot-nginx/local-oldest-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 38ed5debe..bcd02d197 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.26.0 --e .[dev] +certbot[dev]==0.22.0 From cd2edeff1b0f86cc01759e94546b0155f6a8549f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 13:12:06 -0700 Subject: [PATCH 611/631] Fix test farm tests (#6335) * update CentOS AMI ids * Remove assumption of usable default subnet --- tests/letstest/multitester.py | 60 ++++++++++++++++++++++++++++------- tests/letstest/targets.yaml | 4 +-- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 0ae9636d4..320328d9e 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -103,13 +103,32 @@ LOGDIR = "" #points to logging / working directory # boto3/AWS api globals AWS_SESSION = None EC2 = None +SECURITY_GROUP_NAME = 'certbot-security-group' +SUBNET_NAME = 'certbot-subnet' # Boto3/AWS automation functions #------------------------------------------------------------------------------- -def make_security_group(): +def should_use_subnet(subnet): + """Should we use the given subnet for these tests? + + We should if it is the default subnet for the availability zone or the + subnet is named "certbot-subnet". + + """ + if not subnet.map_public_ip_on_launch: + return False + if subnet.default_for_az: + return True + for tag in subnet.tags: + if tag['Key'] == 'Name' and tag['Value'] == SUBNET_NAME: + return True + return False + +def make_security_group(vpc): + """Creates a security group in the given VPC.""" # will fail if security group of GroupName already exists # cannot have duplicate SGs of the same name - mysg = EC2.create_security_group(GroupName="letsencrypt_test", + mysg = vpc.create_security_group(GroupName=SECURITY_GROUP_NAME, Description='security group for automated testing') mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) @@ -123,14 +142,16 @@ def make_security_group(): def make_instance(instance_name, ami_id, keyname, + security_group_id, + subnet_id, machine_type='t2.micro', - security_groups=['letsencrypt_test'], userdata=""): #userdata contains bash or cloud-init script new_instance = EC2.create_instances( BlockDeviceMappings=_get_block_device_mappings(ami_id), ImageId=ami_id, - SecurityGroups=security_groups, + SecurityGroupIds=[security_group_id], + SubnetId=subnet_id, KeyName=keyname, MinCount=1, MaxCount=1, @@ -294,7 +315,7 @@ def grab_certbot_log(): sudo('if [ -f ./certbot.log ]; then \ cat ./certbot.log; else echo "[nolocallog]"; fi') -def create_client_instances(targetlist): +def create_client_instances(targetlist, security_group_id, subnet_id): "Create a fleet of client instances" instances = [] print("Creating instances: ", end="") @@ -314,6 +335,8 @@ def create_client_instances(targetlist): target['ami'], KEYNAME, machine_type=machine_type, + security_group_id=security_group_id, + subnet_id=subnet_id, userdata=userdata)) print() return instances @@ -418,14 +441,28 @@ print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, AWS_SESSION = boto3.session.Session(profile_name=PROFILE) EC2 = AWS_SESSION.resource('ec2') +print("Determining Subnet") +for subnet in EC2.subnets.all(): + if should_use_subnet(subnet): + subnet_id = subnet.id + vpc_id = subnet.vpc.id + break +else: + print("No usable subnet exists!") + print("Please create a VPC with a subnet named {0}".format(SUBNET_NAME)) + print("that maps public IPv4 addresses to instances launched in the subnet.") + sys.exit(1) + print("Making Security Group") +vpc = EC2.Vpc(vpc_id) sg_exists = False -for sg in EC2.security_groups.all(): - if sg.group_name == 'letsencrypt_test': +for sg in vpc.security_groups.all(): + if sg.group_name == SECURITY_GROUP_NAME: + security_group_id = sg.id sg_exists = True - print(" %s already exists"%'letsencrypt_test') + print(" %s already exists"%SECURITY_GROUP_NAME) if not sg_exists: - make_security_group() + security_group_id = make_security_group(vpc).id time.sleep(30) boulder_preexists = False @@ -446,11 +483,12 @@ else: KEYNAME, machine_type='t2.micro', #machine_type='t2.medium', - security_groups=['letsencrypt_test']) + security_group_id=security_group_id, + subnet_id=subnet_id) try: if not cl_args.boulderonly: - instances = create_client_instances(targetlist) + instances = create_client_instances(targetlist, security_group_id, subnet_id) # Configure and launch boulder server #------------------------------------------------------------------------------- diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 766b4ea09..57ce4811a 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -48,13 +48,13 @@ targets: # CentOS # These Marketplace AMIs must, irritatingly, have their terms manually # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-61bbf104 + - ami: ami-9887c6e7 name: centos7 type: centos virt: hvm user: centos # centos6 requires EPEL repo added - - ami: ami-57cd8732 + - ami: ami-1585c46a name: centos6 type: centos virt: hvm From e178bbfdf54e7ac5160de7a4656c6c19fa8ee4d2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 14:10:05 -0700 Subject: [PATCH 612/631] Release script improvements (#6337) * Add error checking and automatic logging. * Ignore release dir and logs * Don't always require PGP card and fix script cmd. * keep track of default GPG key * Add PGP card sanity check after offline signature * fix typo * I'm tired of pressing y. * Automate running tools/offline-sigrequest.sh. * Update comment and make output more readable. --- .gitignore | 3 +- tools/_release.sh | 247 ++++++++++++++++++++++++++++++++++++++++++++ tools/release.sh | 257 +++++----------------------------------------- 3 files changed, 276 insertions(+), 231 deletions(-) create mode 100755 tools/_release.sh diff --git a/.gitignore b/.gitignore index e744a82a2..9ef645593 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ dist*/ /venv*/ /kgs/ /.tox/ -/releases/ +/releases*/ +/log* letsencrypt.log certbot.log letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 diff --git a/tools/_release.sh b/tools/_release.sh new file mode 100755 index 000000000..ec9bd7461 --- /dev/null +++ b/tools/_release.sh @@ -0,0 +1,247 @@ +#!/bin/bash -xe +# Release packages to PyPI + +if [ "$RELEASE_DIR" = "" ]; then + echo Please run this script through the tools/release.sh wrapper script or set the environment + echo variable RELEASE_DIR to the directory where the release should be built. + exit 1 +fi + +version="$1" +echo Releasing production version "$version"... +nextversion="$2" +RELEASE_BRANCH="candidate-$version" + +if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then + RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" +fi +DEFAULT_GPG_KEY="A2CFB51FA275A7286234E7B24D17C995CD9775F2" +RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-"$DEFAULT_GPG_KEY"} +# Needed to fix problems with git signatures and pinentry +export GPG_TTY=$(tty) + +# port for a local Python Package Index (used in testing) +PORT=${PORT:-1234} + +# subpackages to be released (the way developers think about them) +SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" + +# subpackages to be released (the way the script thinks about them) +SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" +SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" +SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" +# certbot_compatibility_test is not packaged because: +# - it is not meant to be used by anyone else than Certbot devs +# - it causes problems when running pytest - the latter tries to +# run everything that matches test*, while there are no unittests +# there + +tag="v$version" +mv "dist.$version" "dist.$version.$(date +%s).bak" || true +git tag --delete "$tag" || true + +tmpvenv=$(mktemp -d) +virtualenv --no-site-packages -p python2 $tmpvenv +. $tmpvenv/bin/activate +# update setuptools/pip just like in other places in the repo +pip install -U setuptools +pip install -U pip # latest pip => no --pre for dev releases +pip install -U wheel # setup.py bdist_wheel + +# newer versions of virtualenv inherit setuptools/pip/wheel versions +# from current env when creating a child env +pip install -U virtualenv + +root_without_le="$version.$$" +root="$RELEASE_DIR/le.$root_without_le" + +echo "Cloning into fresh copy at $root" # clean repo = no artifacts +git clone . $root +git rev-parse HEAD +cd $root +if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then + git branch -f "$RELEASE_BRANCH" +fi +git checkout "$RELEASE_BRANCH" + +for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +do + sed -i 's/\.dev0//' "$pkg_dir/setup.py" + git add "$pkg_dir/setup.py" +done + +SetVersion() { + ver="$1" + # bumping Certbot's version number is done differently + for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test + do + setup_file="$pkg_dir/setup.py" + if [ $(grep -c '^version' "$setup_file") != 1 ]; then + echo "Unexpected count of version variables in $setup_file" + exit 1 + fi + sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py + done + init_file="certbot/__init__.py" + if [ $(grep -c '^__version' "$init_file") != 1 ]; then + echo "Unexpected count of __version variables in $init_file" + exit 1 + fi + sed -i "s/^__version.*/__version__ = '$ver'/" "$init_file" + + git add $SUBPKGS certbot-compatibility-test +} + +SetVersion "$version" + +echo "Preparing sdists and wheels" +for pkg_dir in . $SUBPKGS_NO_CERTBOT +do + cd $pkg_dir + + python setup.py clean + rm -rf build dist + python setup.py sdist + python setup.py bdist_wheel + + echo "Signing ($pkg_dir)" + for x in dist/*.tar.gz dist/*.whl + do + gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x + done + + cd - +done + + +mkdir "dist.$version" +mv dist "dist.$version/certbot" +for pkg_dir in $SUBPKGS_NO_CERTBOT +do + mv $pkg_dir/dist "dist.$version/$pkg_dir/" +done + +echo "Testing packages" +cd "dist.$version" +# start local PyPI +python -m SimpleHTTPServer $PORT & +# cd .. is NOT done on purpose: we make sure that all subpackages are +# installed from local PyPI rather than current directory (repo root) +virtualenv --no-site-packages ../venv +. ../venv/bin/activate +pip install -U setuptools +pip install -U pip +# Now, use our local PyPI. Disable cache so we get the correct KGS even if we +# (or our dependencies) have conditional dependencies implemented with if +# statements in setup.py and we have cached wheels lying around that would +# cause those ifs to not be evaluated. +pip install \ + --no-cache-dir \ + --extra-index-url http://localhost:$PORT \ + $SUBPKGS +# stop local PyPI +kill $! +cd ~- + +# get a snapshot of the CLI help for the docs +# We set CERTBOT_DOCS to use dummy values in example user-agent string. +CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt +jws --help > acme/docs/jws-help.txt + +cd .. +# freeze before installing anything else, so that we know end-user KGS +# make sure "twine upload" doesn't catch "kgs" +if [ -d kgs ] ; then + echo Deleting old kgs... + rm -rf kgs +fi +mkdir kgs +kgs="kgs/$version" +pip freeze | tee $kgs +pip install pytest +for module in $subpkgs_modules ; do + echo testing $module + pytest --pyargs $module +done +cd ~- + +# pin pip hashes of the things we just built +for pkg in $SUBPKGS_IN_AUTO ; do + echo $pkg==$version \\ + pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' +done > letsencrypt-auto-source/pieces/certbot-requirements.txt +deactivate + +# there should be one requirement specifier and two hashes for each subpackage +expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then + echo Unexpected pip hash output + exit 1 +fi + +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +tools/offline-sigrequest.sh +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + echo "The signature on letsencrypt-auto is not correct." + read -p "Would you like this script to try and sign it again [Y/n]?" response + case $response in + [yY][eE][sS]|[yY]|"") + tools/offline-sigrequest.sh;; + *) + ;; + esac +done + +if [ "$RELEASE_GPG_KEY" = "$DEFAULT_GPG_KEY" ]; then + while ! gpg2 --card-status >/dev/null 2>&1; do + echo gpg cannot find your OpenPGP card + read -p "Please take the card out and put it back in again." + done +fi + +# This signature is not quite as strong, but easier for people to verify out of band +gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto +# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, +# but we can use the right name for certbot-auto.asc from day one +mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc + +# copy leauto to the root, overwriting the previous release version +cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto +cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto + +git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt +git diff --cached +git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" + +cd .. +echo Now in $PWD +name=${root_without_le%.*} +ext="${root_without_le##*.}" +rev="$(git rev-parse --short HEAD)" +echo tar cJvf $name.$rev.tar.xz $name.$rev +echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz +cd ~- + +echo "New root: $root" +echo "Test commands (in the letstest repo):" +echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' +echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' +echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' +echo "In order to upload packages run the following command:" +echo twine upload "$root/dist.$version/*/*" + +if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then + SetVersion "$nextversion".dev0 + letsencrypt-auto-source/build.py + git add letsencrypt-auto-source/letsencrypt-auto + git diff + git commit -m "Bump version to $nextversion" +fi diff --git a/tools/release.sh b/tools/release.sh index 880563b4b..ae3e78dc1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -1,11 +1,5 @@ -#!/bin/bash -xe -# Release dev packages to PyPI - -Usage() { - echo Usage: - echo "$0 [ --production ]" - exit 1 -} +#!/bin/bash -e +# Release packages to PyPI if [ "`dirname $0`" != "tools" ] ; then echo Please run this script from the repo root @@ -13,235 +7,38 @@ if [ "`dirname $0`" != "tools" ] ; then fi CheckVersion() { - # Args: - if ! echo "$2" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then + # Args: + if ! echo "$1" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then echo "$1 doesn't look like 1.2.3" + echo "Usage:" + echo "$0 RELEASE_VERSION NEXT_VERSION" exit 1 fi } -if [ "$1" = "--production" ] ; then - version="$2" - CheckVersion Version "$version" - echo Releasing production version "$version"... - nextversion="$3" - CheckVersion "Next version" "$nextversion" - RELEASE_BRANCH="candidate-$version" -else - version=`grep "__version__" certbot/__init__.py | cut -d\' -f2 | sed s/\.dev0//` - version="$version.dev$(date +%Y%m%d)1" - RELEASE_BRANCH="dev-release" - echo Releasing developer version "$version"... -fi +CheckVersion "$1" +CheckVersion "$2" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi -RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} -# Needed to fix problems with git signatures and pinentry -export GPG_TTY=$(tty) - -# port for a local Python Package Index (used in testing) -PORT=${PORT:-1234} - -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - -# subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" -subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" -# certbot_compatibility_test is not packaged because: -# - it is not meant to be used by anyone else than Certbot devs -# - it causes problems when running pytest - the latter tries to -# run everything that matches test*, while there are no unittests -# there - -tag="v$version" -mv "dist.$version" "dist.$version.$(date +%s).bak" || true -git tag --delete "$tag" || true - -tmpvenv=$(mktemp -d) -virtualenv --no-site-packages -p python2 $tmpvenv -. $tmpvenv/bin/activate -# update setuptools/pip just like in other places in the repo -pip install -U setuptools -pip install -U pip # latest pip => no --pre for dev releases -pip install -U wheel # setup.py bdist_wheel - -# newer versions of virtualenv inherit setuptools/pip/wheel versions -# from current env when creating a child env -pip install -U virtualenv - -root_without_le="$version.$$" -root="./releases/le.$root_without_le" - -echo "Cloning into fresh copy at $root" # clean repo = no artifacts -git clone . $root -git rev-parse HEAD -cd $root -if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then - git branch -f "$RELEASE_BRANCH" -fi -git checkout "$RELEASE_BRANCH" - -for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . -do - sed -i 's/\.dev0//' "$pkg_dir/setup.py" -done -# We only add Certbot's setup.py here because the other files are added in the -# call to SetVersion below. -git add -p setup.py - -SetVersion() { - ver="$1" - # bumping Certbot's version number is done differently - for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test - do - sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py - done - sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py - - # interactive user input - git add -p $SUBPKGS certbot-compatibility-test - -} - -SetVersion "$version" - -echo "Preparing sdists and wheels" -for pkg_dir in . $SUBPKGS_NO_CERTBOT -do - cd $pkg_dir - - python setup.py clean - rm -rf build dist - python setup.py sdist - python setup.py bdist_wheel - - echo "Signing ($pkg_dir)" - for x in dist/*.tar.gz dist/*.whl - do - gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x - done - - cd - -done - - -mkdir "dist.$version" -mv dist "dist.$version/certbot" -for pkg_dir in $SUBPKGS_NO_CERTBOT -do - mv $pkg_dir/dist "dist.$version/$pkg_dir/" -done - -echo "Testing packages" -cd "dist.$version" -# start local PyPI -python -m SimpleHTTPServer $PORT & -# cd .. is NOT done on purpose: we make sure that all subpackages are -# installed from local PyPI rather than current directory (repo root) -virtualenv --no-site-packages ../venv -. ../venv/bin/activate -pip install -U setuptools -pip install -U pip -# Now, use our local PyPI. Disable cache so we get the correct KGS even if we -# (or our dependencies) have conditional dependencies implemented with if -# statements in setup.py and we have cached wheels lying around that would -# cause those ifs to not be evaluated. -pip install \ - --no-cache-dir \ - --extra-index-url http://localhost:$PORT \ - $SUBPKGS -# stop local PyPI -kill $! -cd ~- - -# get a snapshot of the CLI help for the docs -# We set CERTBOT_DOCS to use dummy values in example user-agent string. -CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt -jws --help > acme/docs/jws-help.txt - -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -pip install pytest -for module in $subpkgs_modules ; do - echo testing $module - pytest --pyargs $module -done -cd ~- - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt -deactivate - -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output +if [ "$RELEASE_GPG_KEY" = "" ] && ! gpg2 --card-status >/dev/null 2>&1; then + echo OpenPGP card not found! + echo Please insert your PGP card and run this script again. exit 1 fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# and that it's signed correctly -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" -done - -# This signature is not quite as strong, but easier for people to verify out of band -gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt -git diff --cached -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" - -cd .. -echo Now in $PWD -name=${root_without_le%.*} -ext="${root_without_le##*.}" -rev="$(git rev-parse --short HEAD)" -echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz -cd ~- - -echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' -echo "In order to upload packages run the following command:" -echo twine upload "$root/dist.$version/*/*" - -if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then - SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto - git diff - git commit -m "Bump version to $nextversion" +if ! command -v script >/dev/null 2>&1; then + echo The command script was not found. + echo Please install it. + exit 1 +fi + +export RELEASE_DIR="./releases" +mv "$RELEASE_DIR" "$RELEASE_DIR.$(date +%s).bak" || true +LOG_PATH="log" +mv "$LOG_PATH" "$LOG_PATH.$(date +%s).bak" || true + +# Work with both Linux and macOS versions of script +if script --help | grep -q -- '--command'; then + script --command "tools/_release.sh $1 $2" "$LOG_PATH" +else + script "$LOG_PATH" tools/_release.sh "$1" "$2" fi From 19149a0d578249487e2a137b6a69514fdbd395e8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Sep 2018 15:41:59 -0700 Subject: [PATCH 613/631] Release 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++---------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 22 +++++++++------- letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/letsencrypt-auto.sig | 5 ++-- .../pieces/certbot-requirements.txt | 24 ++++++++--------- 26 files changed, 92 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88592013c..7d11449a0 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f435bb1a9..3b5cb7313 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index e097719db..a04bdaf25 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e4ccc719a..5ebfc9f10 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 05649d4d0..fc8fde114 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 911f9e052..9aa9fed33 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9dd318296..24721960b 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 09b11def0..551369f79 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2ca3213bf..ad5ae08af 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index e9ead6546..746040f09 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 3c7402f25..21f9f99ce 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 327224c9c..a241bdcfb 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 1f92f7dce..6860975cc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 0b4241afb..a98419b73 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index e0ce785a1..6342bb11f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bd54ec4c5..f82e1508c 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 5f0b26f6e..2fe473337 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b7cfc15b5..5ebab483a 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 02aaa6581..6cd2af84e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 3b0b77f6c..d6fee2e83 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.27.0.dev0' +__version__ = '0.27.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 931ea4c62..4fea2d4d1 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.26.1 + "". (default: CertbotACMEClient/0.27.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -475,16 +475,15 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary (default: a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: + Path to the Apache 'a2dismod' binary (default: a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT - SSL vhost configuration extension. (default: -le- + SSL vhost configuration extension (default: -le- ssl.conf) --apache-server-root APACHE_SERVER_ROOT - Apache server root directory. (default: /etc/apache2) + Apache server root directory (default: /etc/apache2) --apache-vhost-root APACHE_VHOST_ROOT Apache server VirtualHost configuration root (default: None) @@ -492,14 +491,17 @@ apache: Apache server logs directory (default: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION - Directory path for challenge configuration. (default: + Directory path for challenge configuration (default: /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES - Let installer handle enabling required modules for - you. (Only Ubuntu/Debian currently) (default: True) + Let installer handle enabling required modules for you + (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES - Let installer handle enabling sites for you. (Only + Let installer handle enabling sites for you (Only Ubuntu/Debian currently) (default: True) + --apache-ctl APACHE_CTL + Full path to Apache control script (default: + apache2ctl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index e097719db..a04bdaf25 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9f6706931..5a6fb41f6 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJbTSv8AAoJEE0XyZXNl3Xy12sH/1FgV3SDVG0T1jgKQOYEUwrq -cmpjdav8YPgFOSQDOcyFZG0DNcRfTskZt45IMkBLLnXq2PuPvkppc1+akP81vMoK -NXHHS+PXDMjnBW4NFkexoM06KRF1SyHnvqsOg13w7UW2CjsAgtazGF5BucNCnjPH -XJTwUf4uhKxeUb0Xkva1OPH++oTWz8+SYgWr/iMggkBrK8y04QUUJ6lyCO6MZgcE -3JcECG7CwMK+hW0gCUkCSNZ0NzOBALCd9wCxNGszgkeJXrrW73oUpZmGC5BxIwYY -o6lcF0qo7Jb92t4B3+7JhulMC5JoVoG4lpiXpKQFFCT0P4pZKotIomKNMATmnB4= -=hzUL +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluQW1IACgkQTRfJlc2X +dfI4zAf+NhtBZkuJuyxOAOiHGwic/Wjdu8g+D2gNlqVD+aUco99mrbyYL5/mB/N6 +q27jav+WjJGK+gCBFrddIbPjJXdrWErf93MmkuVKwpBMyQVwuj9hfWfXnB6K7Kse +8mOOPxWKusq5ykLmujPaYG7Fcfxf5DkmxD/hlwFW7zxOQsIARnfUuW2UST58s6q8 +muG/I92DasOihyRLBxZkd930/CocXl9SJVdC2Aus9zc/2ClQsSPFkPGqC3P9ijC4 +DdU4jJF04LTHtbgXHbN/VIH/Hlu4s++uRyRpoWikbOFwlERrkmsc+1/zi3V2l6a2 +nQQzZE3qwk6fL6NXrJPftcw5Ge1uTA== +=yCtI -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 765072c3f..a04bdaf25 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0.dev0" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index d1306d03d..4de632983 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,3 +1,2 @@ -ÍÅ3pºŽàùSÄj Í»¾´¼—²øö«ø~Ô@F-¢ä*6](Ý"üø²A‹,“8èß…ñ=äsO²cô{â…ŽE•B—\ä,üIJå1Ï -»Â»Zï*¤ƒO/Æ+³k+}¦¥ùƒä«~ld‚?]ŒŽfÚ(r?JÐ2Ô”ò †4=¿ K+K¾è#Ó@š_›[)4ºV8;Õ Ž-øKöWørÝýp…ý¤¤wߊ2»Û®x,g§Ú§w·áê×ù8½0;£uC^Ì“ÚÓB¯§€yIÚÄ Ã¹X/ b£øºìÏ -¸®s‹Z¢ö€¥S \ No newline at end of file +‡ÀÓ•°¦«m’" $`áêÊ$¯Ãí€uÞCÎ? »ß]e鹌N]ûùmCÜT*Tg¦ß3”5Î@{¼ƒÔ5Œ«1AE‰Ù +â,ê¼ßeÉâAÞMçT¡/€¡òÈÐú·ÝÙSKyUعÔln€L.†Ñ‹iT;ÜM°gÁÊ!ЊtfC, UáÙ‚GžÂš¹Îûë]¸V\:X§ýA¢-§AКèänN RÿHL8…`»(çÔŒ?m€GÉ…•Ví}•i’ààÿ¢XícÎcÑÆšª‰Š‚ÙˆqëRÎÎIð=©/µs-»>ZI²à#ÑNw( §òº¦”.¼Jey \ No newline at end of file diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index feb3f1c3a..273649da9 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb From e28f3da9749643b41e5ab564829629c6f6e66ae9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Sep 2018 15:42:01 -0700 Subject: [PATCH 614/631] Bump version to 0.28.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7d11449a0..85492d9a3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3b5cb7313..aa5908f9e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 5ebfc9f10..8dac3e047 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index fc8fde114..b823cf98f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9aa9fed33..3daf933cb 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 24721960b..feb2a0d58 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 551369f79..b6efcf360 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ad5ae08af..c268eaa8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 746040f09..fc147f85c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 21f9f99ce..86d36bcb3 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index a241bdcfb..1d196403f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 6860975cc..a5c06d90e 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a98419b73..474093a5b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 6342bb11f..15fe3d6d7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index f82e1508c..c009ef032 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2fe473337..2bae0c3d0 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 5ebab483a..9f8bfbbdb 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6cd2af84e..3c8a66ee5 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index d6fee2e83..ab23926c9 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.27.0' +__version__ = '0.28.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index a04bdaf25..4b55022db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.28.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 2708d28157622f62a19efc53d4fc1fb6530d06b1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 17:13:30 -0700 Subject: [PATCH 615/631] Update changelog for 0.27.0 (#6338) --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d384ad30..6ab875941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.27.0 - 2018-09-05 + +### Added + +* The Apache plugin now accepts the parameter --apache-ctl which can be + used to configure the path to the Apache control script. + +### Changed + +* When using `acme.client.ClientV2` (or + `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a + newer version of the ACME protocol), an `acme.errors.ConflictError` will be + raised if you try to create an ACME account with a key that has already been + used. Previously, a JSON parsing error was raised in this scenario when using + the library with Let's Encrypt's ACMEv2 endpoint. + +### Fixed + +* When Apache is not installed, Certbot's Apache plugin no longer prints + messages about being unable to find apachectl to the terminal when the plugin + is not selected. +* If you're using the Apache plugin with the --apache-vhost-root flag set to a + directory containing a disabled virtual host for the domain you're requesting + a certificate for, the virtual host will now be temporarily enabled if + necessary to pass the HTTP challenge. +* The documentation for the Certbot package can now be built using Sphinx 1.6+. +* You can now call `query_registration` without having to first call + `new_account` on `acme.client.ClientV2` objects. +* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`. +* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura + Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-ovh +* certbot-dns-sakuracloud + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/57?closed=1 + ## 0.26.1 - 2018-07-17 ### Fixed From 0c66de47cf38b83b5a11804c011e54bfca309dd8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 5 Sep 2018 18:05:42 -0700 Subject: [PATCH 616/631] Remind people to modify changelog when submitting PRs (#6341) * Remind people to modify changelog when submitting PRs * Update pull_request_template.md --- pull_request_template.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 pull_request_template.md diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 000000000..c071d4135 --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1 @@ +Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. From 05ad5392551f027a365c7ad70d7d3ab901d8f4a7 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 5 Sep 2018 18:05:48 -0700 Subject: [PATCH 617/631] git ignore pytest cache (#6340) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9ef645593..54545e883 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ tests/letstest/venv/ # pytest cache .cache .mypy_cache/ +.pytest_cache/ # docker files .docker From d39a354a659cbc3ee7ff177ea6bd42724b16c71e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 10:17:51 -0700 Subject: [PATCH 618/631] Create master section for incremental changes (#6342) --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ab875941..561ee49a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.28.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + ## 0.27.0 - 2018-09-05 ### Added From 4e2faffe8936004429c7ee4622e788671dd6afe2 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 6 Sep 2018 15:00:20 -0700 Subject: [PATCH 619/631] fix(apache): s/handle_mods/handle_modules (#6347) fixes #6344 * fix(apache): s/handle_mods/handle_modules * test(apache): ensure all keys defined in OS_DEFAULTS overrides * changelog udpate --- CHANGELOG.md | 3 ++- certbot-apache/certbot_apache/override_suse.py | 2 +- .../certbot_apache/tests/configurator_test.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 561ee49a6..8d2ff6323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* +* Fixed parameter name in OpenSUSE overrides for default parameters in the + Apache plugin. Certbot on OpenSUSE works again. ## 0.27.0 - 2018-09-05 diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py index 83079b92c..3d0043afe 100644 --- a/certbot-apache/certbot_apache/override_suse.py +++ b/certbot-apache/certbot_apache/override_suse.py @@ -23,7 +23,7 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 6f1c358c2..0fb89c95a 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,6 +115,18 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use + from certbot_apache.entrypoint import OVERRIDE_CLASSES + for cls in OVERRIDE_CLASSES.values(): + cls.add_parser_arguments(mock.MagicMock()) + + def test_all_configurators_defaults_defined(self): + from certbot_apache.entrypoint import OVERRIDE_CLASSES + from certbot_apache.configurator import ApacheConfigurator + parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + for cls in OVERRIDE_CLASSES.values(): + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + def test_constant(self): self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in self.config.option("server_root")) From 101eae4e05022c058bb7e9bd46bdbea6e3f573f3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 17:21:31 -0700 Subject: [PATCH 620/631] Update CHANGELOG.md for 0.27.1 release (#6350) --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d2ff6323..6d80fe730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,23 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed +* + +## 0.27.1 - 2018-09-06 + +### Fixed + * Fixed parameter name in OpenSUSE overrides for default parameters in the Apache plugin. Certbot on OpenSUSE works again. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* certbot-apache + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/60?closed=1 ## 0.27.0 - 2018-09-05 From b50abddb5f144d0570ce57dfa87804e781037175 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 17:49:24 -0700 Subject: [PATCH 621/631] Candidate 0.27.1 (#6351) * fix(apache): s/handle_mods/handle_modules (#6347) (#6349) fixes #6344 * fix(apache): s/handle_mods/handle_modules * test(apache): ensure all keys defined in OS_DEFAULTS overrides * changelog udpate (cherry picked from commit 4e2faffe8936004429c7ee4622e788671dd6afe2) * Release 0.27.1 * Bump version to 0.28.0 --- certbot-auto | 26 +++++++++--------- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/certbot-auto b/certbot-auto index a04bdaf25..076c45e39 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4fea2d4d1..4ed9f0731 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.0 + "". (default: CertbotACMEClient/0.27.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the diff --git a/letsencrypt-auto b/letsencrypt-auto index a04bdaf25..076c45e39 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 5a6fb41f6..747d98e2d 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluQW1IACgkQTRfJlc2X -dfI4zAf+NhtBZkuJuyxOAOiHGwic/Wjdu8g+D2gNlqVD+aUco99mrbyYL5/mB/N6 -q27jav+WjJGK+gCBFrddIbPjJXdrWErf93MmkuVKwpBMyQVwuj9hfWfXnB6K7Kse -8mOOPxWKusq5ykLmujPaYG7Fcfxf5DkmxD/hlwFW7zxOQsIARnfUuW2UST58s6q8 -muG/I92DasOihyRLBxZkd930/CocXl9SJVdC2Aus9zc/2ClQsSPFkPGqC3P9ijC4 -DdU4jJF04LTHtbgXHbN/VIH/Hlu4s++uRyRpoWikbOFwlERrkmsc+1/zi3V2l6a2 -nQQzZE3qwk6fL6NXrJPftcw5Ge1uTA== -=yCtI +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X +dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ +nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn +ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa +I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk +AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm +8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== +=QRnC -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 4b55022db..09e728aa4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4de632983e5a48c69b74f50d28f12b787fe3d374..b717e359b2a2c65f6cb588a007f5024572c1e4ff 100644 GIT binary patch literal 256 zcmV+b0ssEcvxB0LY3lA!OPssV3H<~y?AaAqjuWWrmMIj>00(?0Ek&mFUj?a=WHkII z-RNTHeD@0(kwW#&HKEPltP&F}Il@J^#^yCaE=%~OTz&S0VZ;xAiFO*AjYgLt@?f^0 z5%fBEr@1)1d9JccQ6BE>@C=b&m*{iB+J6X1CVdnJgQ4FUmIMh?O(XLrLz zjb^9RY>s^Yo_H0;@vfC^I1)~)4x~+{{pTYk(%Y}AY2=R!zDwo_S`Vp=%04-BF^8$F zx6n%y(2yohnk6h!P4Oc-;Nb&}2~R5^lWYM-Sp*8Vav}^+f?3H>xwNIK(r-rHpKXnf GZUdUtC3&a- literal 256 zcmV+b0ssDoz|)nmrmJm|B7YzxVBzY@B(KBmfOXzO&Oac#-(6+txr|O-`}u7{+*B%5 zXQtmXln6D>KzqD{)HNH7tA8;;21N&n*$Uz;>b&1&$>KrYP3KghFMy%)$QvBc`nTQL zQ%iYO*truD)NF2mOo1+jjepULX;eGhO|WOd$|2B-bY?>=43Aae*@8!&!kW3x`|Ac> zxK>;`Sf~9#qAjOE(3R_%S2X_6h_;QtSz zSnXrZW6{Q%s)>q%*@$uLQXI|#&Pnh+sV}v2ExSHi84pRa;3LsacPI^~^17y!F1$)* G4|x*$B7(XA diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 273649da9..b9cd42694 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f From 5d1c6d28d50c539cf616d79634cd579252ca9a3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 7 Sep 2018 12:18:59 -0700 Subject: [PATCH 622/631] Update DNS plugin docs. (#6358) --- docs/using.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 946c12bc6..76db22c84 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -190,10 +190,11 @@ If you'd like to obtain a wildcard certificate from Let's Encrypt or run ``certbot`` on a machine other than your target webserver, you can use one of Certbot's DNS plugins. -These plugins are still in the process of being packaged -by many distributions and cannot currently be installed with ``certbot-auto``. -If, however, you are comfortable installing the certificates yourself, -you can run these plugins with :ref:`Docker `. +These plugins are not included in a default Certbot installation and must be +installed separately. While the DNS plugins cannot currently be used with +``certbot-auto``, they are available in many OS package managers and as Docker +images. Visit https://certbot.eff.org to learn the best way to use the DNS +plugins on your system. Once installed, you can find documentation on how to use each plugin at: From 85a859d63fb0617166aae58f362a27c709bc85eb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 8 Sep 2018 16:34:27 +0200 Subject: [PATCH 623/631] Make Certbot runnable on Windows (#6296) * Add and use a compatibility layer to allow certbot to be run on windows. * Fix path comparison * Corrections on compat and util for tests * Less intrusive way to parse prefix in webroot plugin working for both linux and windows. * Disable pylint import-error for some optional imports in compat.py * Ensure path is normalized before prefixes are generated in webroot plugin * Same prefixes in linux and windows, in fact root path is not needed in webroot plugin * Check that user has administrative rights before continuing on windows (necessary for symlink creation) * More straightforward way to test administrative rights on windows * Try to resolve import error in travis ci * OK. We go for full introspection to trick the ci. * Move the administrative rights control to the certbot entrypoint * Add comment for a really non trivial code. * Allow some commands to be run on a shell without admin rights * Avoid races conditions on windows for lock files * Add sphinx doc to the compat functions. * Remove irrelevant Windows error in the lock mechanism. * Some corrections on compat --- certbot/account.py | 5 +- certbot/cert_manager.py | 5 +- certbot/client.py | 3 +- certbot/compat.py | 140 ++++++++++++++++++++++++++++ certbot/crypto_util.py | 5 +- certbot/display/util.py | 9 +- certbot/lock.py | 23 +---- certbot/log.py | 3 +- certbot/main.py | 12 ++- certbot/plugins/util.py | 7 +- certbot/plugins/util_test.py | 6 +- certbot/plugins/webroot.py | 6 +- certbot/reverter.py | 7 +- certbot/tests/configuration_test.py | 19 ++-- certbot/tests/display/util_test.py | 8 +- certbot/tests/lock_test.py | 2 +- certbot/tests/main_test.py | 5 +- certbot/tests/util.py | 1 - certbot/tests/util_test.py | 5 +- 19 files changed, 204 insertions(+), 67 deletions(-) create mode 100644 certbot/compat.py diff --git a/certbot/account.py b/certbot/account.py index 59ceb42e0..76135c3aa 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -17,6 +17,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -140,7 +141,7 @@ class AccountFileStorage(interfaces.AccountStorage): """ def __init__(self, config): self.config = config - util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(), self.config.strict_permissions) def _account_dir_path(self, account_id): @@ -323,7 +324,7 @@ class AccountFileStorage(interfaces.AccountStorage): def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) - util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), + util.make_or_verify_dir(account_dir_path, 0o700, compat.os_geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 771ca8caf..2a67f8765 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -8,6 +8,7 @@ import traceback import zope.component from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -104,7 +105,7 @@ def lineage_for_certname(cli_config, certname): """Find a lineage object with name certname.""" configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) try: renewal_file = storage.renewal_file_for_certname(cli_config, certname) except errors.CertStorageError: @@ -374,7 +375,7 @@ def _search_lineages(cli_config, func, initial_rv, *args): """ configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) rv = initial_rv for renewal_file in storage.renewal_conf_files(cli_config): diff --git a/certbot/client.py b/certbot/client.py index 4d4915a27..e634b6bd9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -24,6 +24,7 @@ import certbot from certbot import account from certbot import auth_handler from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import eff @@ -447,7 +448,7 @@ class Client(object): """ for path in cert_path, chain_path, fullchain_path: util.make_or_verify_dir( - os.path.dirname(path), 0o755, os.geteuid(), + os.path.dirname(path), 0o755, compat.os_geteuid(), self.config.strict_permissions) diff --git a/certbot/compat.py b/certbot/compat.py new file mode 100644 index 000000000..1dc89dfd8 --- /dev/null +++ b/certbot/compat.py @@ -0,0 +1,140 @@ +""" +Compatibility layer to run certbot both on Linux and Windows. + +The approach used here is similar to Modernizr for Web browsers. +We do not check the plateform type to determine if a particular logic is supported. +Instead, we apply a logic, and then fallback to another logic if first logic +is not supported at runtime. + +Then logic chains are abstracted into single functions to be exposed to certbot. +""" +import os +import select +import sys +import errno +import ctypes + +from certbot import errors + +try: + # Linux specific + import fcntl # pylint: disable=import-error +except ImportError: + # Windows specific + import msvcrt # pylint: disable=import-error + +UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ + 'certificates', 'enhance', 'revoke', 'delete', + 'register', 'unregister', 'config_changes', 'plugins'] +def raise_for_non_administrative_windows_rights(subcommand): + """ + On Windows, raise if current shell does not have the administrative rights. + Do nothing on Linux. + + :param str subcommand: The subcommand (like 'certonly') passed to the certbot client. + + :raises .errors.Error: If the provided subcommand must be run on a shell with + administrative rights, and current shell does not have these rights. + + """ + # Why not simply try ctypes.windll.shell32.IsUserAnAdmin() and catch AttributeError ? + # Because windll exists only on a Windows runtime, and static code analysis engines + # do not like at all non existent objects when run from Linux (even if we handle properly + # all the cases in the code). + # So we access windll only by reflection to trick theses engines. + if hasattr(ctypes, 'windll') and subcommand not in UNPRIVILEGED_SUBCOMMANDS_ALLOWED: + windll = getattr(ctypes, 'windll') + if windll.shell32.IsUserAnAdmin() == 0: + raise errors.Error( + 'Error, "{0}" subcommand must be run on a shell with administrative rights.' + .format(subcommand)) + +def os_geteuid(): + """ + Get current user uid + + :returns: The current user uid. + :rtype: int + + """ + try: + # Linux specific + return os.geteuid() + except AttributeError: + # Windows specific + return 0 + +def readline_with_timeout(timeout, prompt): + """ + Read user input to return the first line entered, or raise after specified timeout. + + :param float timeout: The timeout in seconds given to the user. + :param str prompt: The prompt message to display to the user. + + :returns: The first line entered by the user. + :rtype: str + + """ + try: + # Linux specific + # + # Call to select can only be done like this on UNIX + rlist, _, _ = select.select([sys.stdin], [], [], timeout) + if not rlist: + raise errors.Error( + "Timed out waiting for answer to prompt '{0}'".format(prompt)) + return rlist[0].readline() + except OSError: + # Windows specific + # + # No way with select to make a timeout to the user input on Windows, + # as select only supports socket in this case. + # So no timeout on Windows for now. + return sys.stdin.readline() + +def lock_file(fd): + """ + Lock the file linked to the specified file descriptor. + + :param int fd: The file descriptor of the file to lock. + + """ + if 'fcntl' in sys.modules: + # Linux specific + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + # Windows specific + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + +def release_locked_file(fd, path): + """ + Remove, close, and release a lock file specified by its file descriptor and its path. + + :param int fd: The file descriptor of the lock file. + :param str path: The path of the lock file. + + """ + # Linux specific + # + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(path) + except OSError as err: + if err.errno == errno.EACCES: + # Windows specific + # We will not be able to remove a file before closing it. + # To avoid race conditions described for Linux, we will not delete the lockfile, + # just close it to be reused on the next Certbot call. + pass + else: + raise + finally: + os.close(fd) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 943e2c87f..6193a8fbf 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -25,6 +25,7 @@ from OpenSSL import SSL # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import errors from certbot import interfaces from certbot import util @@ -60,7 +61,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): config = zope.component.getUtility(interfaces.IConfig) # Save file - util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(key_dir, 0o700, compat.os_geteuid(), config.strict_permissions) key_f, key_path = util.unique_file( os.path.join(key_dir, keyname), 0o600, "wb") @@ -91,7 +92,7 @@ def init_save_csr(privkey, names, path): privkey.pem, names, must_staple=config.must_staple) # Save CSR - util.make_or_verify_dir(path, 0o755, os.geteuid(), + util.make_or_verify_dir(path, 0o755, compat.os_geteuid(), config.strict_permissions) csr_f, csr_filename = util.unique_file( os.path.join(path, "csr-certbot.pem"), 0o644, "wb") diff --git a/certbot/display/util.py b/certbot/display/util.py index e157a1123..9a813d4b7 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -1,12 +1,12 @@ """Certbot display.""" import logging import os -import select import sys import textwrap import zope.interface +from certbot import compat from certbot import constants from certbot import interfaces from certbot import errors @@ -79,13 +79,8 @@ def input_with_timeout(prompt=None, timeout=36000.0): sys.stdout.write(prompt) sys.stdout.flush() - # select can only be used like this on UNIX - rlist, _, _ = select.select([sys.stdin], [], [], timeout) - if not rlist: - raise errors.Error( - "Timed out waiting for answer to prompt '{0}'".format(prompt)) + line = compat.readline_with_timeout(timeout, prompt) - line = rlist[0].readline() if not line: raise EOFError return line.rstrip('\n') diff --git a/certbot/lock.py b/certbot/lock.py index 5f59cc090..3ff46518d 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,9 +1,9 @@ """Implements file locks for locking files and directories in UNIX.""" import errno -import fcntl import logging import os +from certbot import compat from certbot import errors logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ class LockFile(object): """ try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + compat.lock_file(fd) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): logger.debug( @@ -118,22 +118,7 @@ class LockFile(object): def release(self): """Remove, close, and release the lock file.""" - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - # - # Calling os.remove on a file that's in use doesn't work on - # Windows, but neither does locking with fcntl. try: - os.remove(self._path) + compat.release_locked_file(self._fd, self._path) finally: - try: - os.close(self._fd) - finally: - self._fd = None + self._fd = None diff --git a/certbot/log.py b/certbot/log.py index 89626af99..b883936f3 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -23,6 +23,7 @@ import traceback from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import util @@ -133,7 +134,7 @@ def setup_log_file_handler(config, logfile, fmt): # TODO: logs might contain sensitive data such as contents of the # private key! #525 util.set_up_core_dir( - config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) + config.logs_dir, 0o700, compat.os_geteuid(), config.strict_permissions) log_file_path = os.path.join(config.logs_dir, logfile) try: handler = logging.handlers.RotatingFileHandler( diff --git a/certbot/main.py b/certbot/main.py index 2cba8745b..214378da8 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -19,6 +19,7 @@ from certbot import account from certbot import cert_manager from certbot import cli from certbot import client +from certbot import compat from certbot import configuration from certbot import constants from certbot import crypto_util @@ -1289,16 +1290,16 @@ def make_or_verify_needed_dirs(config): """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) hook_dirs = (config.renewal_pre_hooks_dir, config.renewal_deploy_hooks_dir, config.renewal_post_hooks_dir,) for hook_dir in hook_dirs: util.make_or_verify_dir(hook_dir, - uid=os.geteuid(), + uid=compat.os_geteuid(), strict=config.strict_permissions) @@ -1333,6 +1334,7 @@ def main(cli_args=sys.argv[1:]): :raises errors.Error: error if plugin command is not supported """ + log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() @@ -1346,6 +1348,10 @@ def main(cli_args=sys.argv[1:]): config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) + # On windows, shell without administrative right cannot create symlinks required by certbot. + # So we check the rights before continuing. + compat.raise_for_non_administrative_windows_rights(config.verb) + try: log.post_arg_parse_setup(config) make_or_verify_needed_dirs(config) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index 3f03bf375..5c682c3ff 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -9,18 +9,19 @@ logger = logging.getLogger(__name__) def get_prefixes(path): """Retrieves all possible path prefixes of a path, in descending order of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + (linux) /a/b/c returns ['/a/b/c', '/a/b', '/a', '/'] + (windows) C:\\a\\b\\c returns ['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:'] :param str path: the path to break into prefixes :returns: all possible path prefixes of given path in descending order :rtype: `list` of `str` """ - prefix = path + prefix = os.path.normpath(path) prefixes = [] while len(prefix) > 0: prefixes.append(prefix) prefix, _ = os.path.split(prefix) - # break once we hit '/' + # break once we hit the root path if prefix == prefixes[-1]: break return prefixes diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 9757d8de7..b2f9c79ea 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -9,9 +9,9 @@ class GetPrefixTest(unittest.TestCase): """Tests for certbot.plugins.get_prefixes.""" def test_get_prefix(self): from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) + self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes('/'), ['/']) + self.assertEqual(get_prefixes('a'), ['a']) class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 5d0d7d586..529094705 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -170,7 +170,9 @@ to serve all files under specified web root ({0}).""" old_umask = os.umask(0o022) try: stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + # We ignore the last prefix in the next iteration, + # as it does not correspond to a folder path ('/' or 'C:') + for prefix in sorted(util.get_prefixes(self.full_roots[name])[:-1], key=len): try: # This is coupled with the "umask" call above because # os.mkdir's "mode" parameter may not always work: @@ -180,7 +182,7 @@ to serve all files under specified web root ({0}).""" # Set owner as parent directory if possible try: os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: + except (OSError, AttributeError) as exception: logger.info("Unable to change owner and uid of webroot directory") logger.debug("Error was: %s", exception) except OSError as exception: diff --git a/certbot/reverter.py b/certbot/reverter.py index 683c0cc32..5d56615fd 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -10,6 +10,7 @@ import traceback import six import zope.component +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -65,7 +66,7 @@ class Reverter(object): self.config = config util.make_or_verify_dir( - config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + config.backup_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) def revert_temporary_config(self): @@ -219,7 +220,7 @@ class Reverter(object): """ util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) op_fd, existing_filepaths = self._read_and_append( @@ -433,7 +434,7 @@ class Reverter(object): cp_dir = self.config.in_progress_dir util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) return cp_dir diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 59fb2cea9..eb8bcff89 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -48,18 +48,23 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_constants.TEMP_CHECKPOINT_DIR = 't' self.assertEqual( - self.config.accounts_dir, os.path.join( - self.config.config_dir, 'acc/acme-server.org:443/new')) + os.path.normpath(self.config.accounts_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'acc/acme-server.org:443/new'))) self.assertEqual( - self.config.backup_dir, os.path.join(self.config.work_dir, 'backups')) + os.path.normpath(self.config.backup_dir), + os.path.normpath(os.path.join(self.config.work_dir, 'backups'))) self.assertEqual( - self.config.csr_dir, os.path.join(self.config.config_dir, 'csr')) + os.path.normpath(self.config.csr_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'csr'))) self.assertEqual( - self.config.in_progress_dir, os.path.join(self.config.work_dir, '../p')) + os.path.normpath(self.config.in_progress_dir), + os.path.normpath(os.path.join(self.config.work_dir, '../p'))) self.assertEqual( - self.config.key_dir, os.path.join(self.config.config_dir, 'keys')) + os.path.normpath(self.config.key_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'keys'))) self.assertEqual( - self.config.temp_checkpoint_dir, os.path.join(self.config.work_dir, 't')) + os.path.normpath(self.config.temp_checkpoint_dir), + os.path.normpath(os.path.join(self.config.work_dir, 't'))) def test_absolute_paths(self): from certbot.configuration import NamespaceConfig diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 1dfc21c30..f5cf29047 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -34,7 +34,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" stdin = six.StringIO(expected + "\n") - with mock.patch("certbot.display.util.select.select") as mock_select: + with mock.patch("certbot.compat.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -321,11 +321,7 @@ class FileOutputDisplayTest(unittest.TestCase): class NoninteractiveDisplayTest(unittest.TestCase): - """Test non-interactive display. - - These tests are pretty easy! - - """ + """Test non-interactive display. These tests are pretty easy!""" def setUp(self): super(NoninteractiveDisplayTest, self).setUp() self.mock_stdout = mock.MagicMock() diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index e1a4f8c8a..51469e8c1 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -89,7 +89,7 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.lock.fcntl.lockf') + @mock.patch('certbot.compat.fcntl.lockf') def test_unexpected_lockf_err(self, mock_lockf): msg = 'hi there' mock_lockf.side_effect = IOError(msg) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index cc4e6c293..47ca235f1 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -19,6 +19,7 @@ from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli +from certbot import compat from certbot import constants from certbot import configuration from certbot import crypto_util @@ -1577,7 +1578,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for core_dir in (self.config.config_dir, self.config.work_dir,): mock_util.set_up_core_dir.assert_any_call( core_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), self.config.strict_permissions + compat.os_geteuid(), self.config.strict_permissions ) hook_dirs = (self.config.renewal_pre_hooks_dir, @@ -1586,7 +1587,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for hook_dir in hook_dirs: # default mode of 755 is used mock_util.make_or_verify_dir.assert_any_call( - hook_dir, uid=os.geteuid(), + hook_dir, uid=compat.os_geteuid(), strict=self.config.strict_permissions) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8434d11de..d505ea76c 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -362,7 +362,6 @@ def lock_and_call(func, lock_path): child.join() assert child.exitcode == 0 - def hold_lock(cv, lock_path): # pragma: no cover """Acquire a file lock at lock_path and wait to release it. diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 0e280f3ab..689b4108d 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -10,6 +10,7 @@ import mock import six from six.moves import reload_module # pylint: disable=import-error +from certbot import compat from certbot import errors import certbot.tests.util as test_util @@ -116,7 +117,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): @mock.patch('certbot.util.lock_dir_until_exit') def test_success(self, mock_lock): new_dir = os.path.join(self.tempdir, 'new') - self._call(new_dir, 0o700, os.geteuid(), False) + self._call(new_dir, 0o700, compat.os_geteuid(), False) self.assertTrue(os.path.exists(new_dir)) self.assertEqual(mock_lock.call_count, 1) @@ -124,7 +125,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, - self.tempdir, 0o700, os.geteuid(), False) + self.tempdir, 0o700, compat.os_geteuid(), False) class MakeOrVerifyDirTest(test_util.TempDirTestCase): From 251355cade365b77c79582a8b7f14454c601a10a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 11 Sep 2018 15:44:26 -0700 Subject: [PATCH 624/631] Add better error handling around release signatures (#6353) * Better error handling around sig after offline-sig * Add error handling around first sig with git. * Don't fail if offline-sig fails. --- tools/_release.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index ec9bd7461..dab4eec3a 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -15,8 +15,7 @@ RELEASE_BRANCH="candidate-$version" if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" fi -DEFAULT_GPG_KEY="A2CFB51FA275A7286234E7B24D17C995CD9775F2" -RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-"$DEFAULT_GPG_KEY"} +RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -185,7 +184,7 @@ fi letsencrypt-auto-source/build.py # and that it's signed correctly -tools/offline-sigrequest.sh +tools/offline-sigrequest.sh || true while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ letsencrypt-auto-source/letsencrypt-auto.sig \ letsencrypt-auto-source/letsencrypt-auto ; do @@ -193,21 +192,19 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Would you like this script to try and sign it again [Y/n]?" response case $response in [yY][eE][sS]|[yY]|"") - tools/offline-sigrequest.sh;; + tools/offline-sigrequest.sh || true;; *) ;; esac done -if [ "$RELEASE_GPG_KEY" = "$DEFAULT_GPG_KEY" ]; then - while ! gpg2 --card-status >/dev/null 2>&1; do - echo gpg cannot find your OpenPGP card - read -p "Please take the card out and put it back in again." - done -fi - # This signature is not quite as strong, but easier for people to verify out of band -gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto +while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do + echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." + echo "Make sure your OpenPGP card is in your computer if you are using one." + echo "You may need to take the card out and put it back in again." + read -p "Press enter to try signing again." +done # We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, # but we can use the right name for certbot-auto.asc from day one mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc @@ -218,7 +215,12 @@ cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt git diff --cached -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do + echo "Unable to sign the release commit using git." + echo "You may have to configure git to use gpg2 by running:" + echo 'git config --global gpg.program $(command -v gpg2)' + read -p "Press enter to try signing again." +done git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" cd .. From 8f7209de145735565b21d66b5796035a3bce7615 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 12 Sep 2018 16:35:43 -0700 Subject: [PATCH 625/631] Silence spammy integration test cases. (#5934) --- tests/certbot-boulder-integration.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 8b8b931e5..8a8e60798 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -185,10 +185,10 @@ get_num_tmp_files() { ls -1 /tmp | wc -l } num_tmp_files=$(get_num_tmp_files) -common --csr / && echo expected error && exit 1 || true -common --help -common --help all -common --version +common --csr / > /dev/null && echo expected error && exit 1 || true +common --help > /dev/null +common --help all > /dev/null +common --version > /dev/null if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then echo "New files or directories created in /tmp!" exit 1 From a3b858db34a2512d184f8d90220fd02d3c310c7e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 13 Sep 2018 00:38:37 +0100 Subject: [PATCH 626/631] Exclude one-time use parameters. Fixes #6118 (#6223) * Exclude one-time use parameters. Fixes #6118 * Fix error. * Delete items inplace, rather than creating new list. * Fix stupid mistake. * Use .index() for stability. * Try previous idea while resetting the index. * Shorter comment for pylint. * More readable approach * Fix whitespace --- certbot-nginx/certbot_nginx/parser.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 7d1da2e73..a7ca2a735 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -395,12 +395,17 @@ class NginxParser(object): addr.ipv6only = False for directive in enclosing_block[new_vhost.path[-1]][1]: if len(directive) > 0 and directive[0] == 'listen': - if 'default_server' in directive: - del directive[directive.index('default_server')] - if 'default' in directive: - del directive[directive.index('default')] - if 'ipv6only=on' in directive: - del directive[directive.index('ipv6only=on')] + # Exclude one-time use parameters which will cause an error if repeated. + # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen + exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', + 'ipv6only', 'reuseport', 'so_keepalive')) + + for param in exclude: + # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 + keys = [x.split('=')[0] for x in directive] + if param in keys: + del directive[keys.index(param)] return new_vhost From b32ec6ed301bd591124a1a60abc7710b3b95b5d7 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Wed, 12 Sep 2018 16:40:10 -0700 Subject: [PATCH 627/631] Remove CHANGES.rst (#6162) The change log is now being tracked in CHANGELOG.md, so CHANGES.rst is no longer necessary. --- CHANGES.rst | 8 -------- Dockerfile | 2 +- Dockerfile-old | 2 +- MANIFEST.in | 2 +- certbot-compatibility-test/Dockerfile | 2 +- setup.py | 3 +-- 6 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 CHANGES.rst diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 3b92c125f..000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,8 +0,0 @@ -ChangeLog -========= - -To see the changes in a given release, view the issues closed in a given -release's GitHub milestone: - - - `Past releases `_ - - `Upcoming releases `_ diff --git a/Dockerfile b/Dockerfile index 28cd6b323..d1296b30f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot -COPY CHANGES.rst README.rst setup.py src/ +COPY CHANGELOG.md README.rst setup.py src/ COPY acme src/acme COPY certbot src/certbot diff --git a/Dockerfile-old b/Dockerfile-old index 7bce82e0c..da48c7fc8 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ # all above files are necessary for setup.py and venv setup, however, # package source code directory has to be copied separately to a diff --git a/MANIFEST.in b/MANIFEST.in index 434a156b7..7f529c7a7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.rst -include CHANGES.rst +include CHANGELOG.md include CONTRIBUTING.md include LICENSE.txt include linter_plugin.py diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index fe55a68a6..803b4a1b9 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... diff --git a/setup.py b/setup.py index a13b7cdb9..f8f5feadc 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ init_fn = os.path.join(here, 'certbot', '__init__.py') meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) -changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] # This package relies on PyOpenSSL, requests, and six, however, it isn't @@ -79,7 +78,7 @@ setup( name='certbot', version=version, description="ACME client", - long_description=readme, # later: + '\n\n' + changes + long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", author_email='client-dev@letsencrypt.org', From 38b1d9d6bab6bef9f9f8499a9a6f8dcd921133ba Mon Sep 17 00:00:00 2001 From: David Beitey Date: Wed, 12 Sep 2018 23:48:50 +0000 Subject: [PATCH 628/631] More detailed error logging for nginx plugin (#6175) This makes errors more useful when Nginx can't be found or when Nginx's configuration can't be found. Previously, a generic `NoInstallationError` isn't descriptive enough to explain _what_ wasn't installed or what failed without going digging into the source code. --- certbot-nginx/certbot_nginx/configurator.py | 4 +++- certbot-nginx/certbot_nginx/parser.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index d31a54c17..d526381a2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -136,7 +136,9 @@ class NginxConfigurator(common.Installer): """ # Verify Nginx is installed if not util.exe_exists(self.conf('ctl')): - raise errors.NoInstallationError + raise errors.NoInstallationError( + "Could not find a usable 'nginx' binary. Ensure nginx exists, " + "the binary is executable, and your PATH is set correctly.") # Make sure configuration is valid self.config_test() diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index a7ca2a735..a5cf2892e 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -222,7 +222,7 @@ class NginxParser(object): return os.path.join(self.root, name) raise errors.NoInstallationError( - "Could not find configuration root") + "Could not find Nginx root configuration file (nginx.conf)") def filedump(self, ext='tmp', lazy=True): """Dumps parsed configurations into files. From e501322ff1e4f0349bdd6ecc3dd526613cd7f00d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 14 Sep 2018 00:20:22 +0200 Subject: [PATCH 629/631] Connect AppVeyor to the certbot git repository (#6361) --- appveyor.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..67ad67c16 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,12 @@ +# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2 +branches: + only: + - master + - /^\d+\.\d+\.x$/ # version branches like X.X.X + - /^test-.*$/ + +build: off + +test_script: + - ps: Write-Host "Hello, world!" + \ No newline at end of file From 3ef43e4d88783fe23c0ff44e5edfc75ebe4891c8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 18 Sep 2018 12:52:11 -0700 Subject: [PATCH 630/631] Update parser to match new Nginx functionality (#6381) Previously, Nginx did not allow `${` to start a variable name. Now it's allowed to. This means we'll be more permissible than Nginx when people are on older versions of Nginx, but it's unlikely anyone was relying on this to fail in the first place, so that's probably ok. --- CHANGELOG.md | 6 +++--- certbot-nginx/certbot_nginx/nginxparser.py | 2 +- .../certbot_nginx/tests/nginxparser_test.py | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d80fe730..9abe047c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,15 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* - +* Match Nginx parser update in allowing variable names to start with `${`. + ## 0.27.1 - 2018-09-06 ### Fixed * Fixed parameter name in OpenSUSE overrides for default parameters in the Apache plugin. Certbot on OpenSUSE works again. - + Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 8818bc040..bfb75adcc 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -26,7 +26,7 @@ class RawNginxParser(object): dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\') squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\') quoted = dquoted | squoted - head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space) + head_tokenchars = Regex(r"(\$\{)|[^{};\s'\"]") # if (last_space) tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars)) paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars)) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index dd31ebac8..7fc4576c3 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -271,6 +271,8 @@ class TestRawNginxParser(unittest.TestCase): location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ { alias /data/w3/images/$1; } + + proxy_set_header X-Origin-URI ${scheme}://${http_host}/$request_uri; """ parsed = loads(test) self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'], @@ -281,7 +283,8 @@ class TestRawNginxParser(unittest.TestCase): [['return', '403']]], [['if', '($args', '~', 'post=140)'], [['rewrite', '^', 'http://example.com/']]], [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'], - [['alias', '/data/w3/images/$1']]]] + [['alias', '/data/w3/images/$1']]], + ['proxy_set_header', 'X-Origin-URI', '${scheme}://${http_host}/$request_uri']] ) def test_edge_cases(self): @@ -289,10 +292,6 @@ class TestRawNginxParser(unittest.TestCase): parsed = loads(r'"hello\""; # blah "heh heh"') self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']]) - # empty var as block - parsed = loads(r"${}") - self.assertEqual(parsed, [[['$'], []]]) - # if with comment parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah ) }""") @@ -342,10 +341,9 @@ class TestRawNginxParser(unittest.TestCase): ]) # variable weirdness - parsed = loads("directive $var;") - self.assertEqual(parsed, [['directive', '$var']]) + parsed = loads("directive $var ${var} $ ${};") + self.assertEqual(parsed, [['directive', '$var', '${var}', '$', '${}']]) self.assertRaises(ParseException, loads, "server {server_name test.com};") - self.assertRaises(ParseException, loads, "directive ${var};") self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']]) self.assertRaises(ParseException, loads, "blag${dfgdf{g};") From efd2ed1bdb38cba36058be46a0208677012389c4 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 19 Sep 2018 02:35:28 +0200 Subject: [PATCH 631/631] Correct OVH integration tests on machines without internet access (#6380) * Correct OVH integration tests on machines without internet. * Update changelog --- CHANGELOG.md | 1 + certbot-dns-ovh/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9abe047c5..25aff0a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed * Match Nginx parser update in allowing variable names to start with `${`. +* Correct OVH integration tests on machines without internet access. ## 0.27.1 - 2018-09-06 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 15fe3d6d7..1f3acbf62 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -11,7 +11,7 @@ version = '0.28.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=2.7.3', # Correct OVH integration tests 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index ef7804328..00ecee03e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.2.1 +dns-lexicon==2.7.3 dnspython==1.15.0 docutils==0.14 execnet==1.5.0