From 8857302347afacff6673fcabadde713126c012ca Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 4 Jun 2014 14:41:18 -0700 Subject: [PATCH 001/639] check-starttls and process-google-starttls-domains.py --- check-starttls.py | 94 ++++++++++++++++++++++++++++++ process-google-starttls-domains.py | 18 ++++++ 2 files changed, 112 insertions(+) create mode 100755 check-starttls.py create mode 100755 process-google-starttls-domains.py diff --git a/check-starttls.py b/check-starttls.py new file mode 100755 index 000000000..15bc7ce64 --- /dev/null +++ b/check-starttls.py @@ -0,0 +1,94 @@ +#!/usr/bin/python +import sys +import os +import errno +import smtplib +import socket +import subprocess +import re + +import dns.resolver +from M2Crypto import X509 + +def mkdirp(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def extract_names(pem): + leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) + + """Extracts a list of DNS names associated with the leaf cert.""" + subj = leaf.get_subject() + # Certs have a "subject" identified by a Distingushed Name (DN). + # Host certs should also have a Common Name (CN) with a DNS name. + common_names = subj.get_entries_by_nid(subj.nid['CN']) + common_names = [name.get_data().as_text() for name in common_names] + try: + # The SAN extension allows one cert to cover multiple domains + # and permits DNS wildcards. + # http://www.digicert.com/subject-alternative-name.htm + # The field is a comma delimited list, e.g.: + # >>> twitter_cert.get_ext('subjectAltName').get_value() + # 'DNS:www.twitter.com, DNS:twitter.com' + alt_names = leaf.get_ext('subjectAltName').get_value() + alt_names = alt_names.split(',') + alt_names = [name.partition(':') for name in alt_names] + alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] + except: + alt_names = [] + return set(common_names + alt_names) + +def tls_connect(mx_host, mail_domain): + # smtplib doesn't let us access certificate information, + # so shell out to openssl. + output = subprocess.check_output( + """openssl s_client \ + -CApath /usr/share/ca-certificates/mozilla/ \ + -starttls smtp -connect %s:25 -showcerts 0: + print "iiii ", len(cert) + print extract_names(cert[0]) + #lines = output.split("\n") + #for i in range(0, len(lines)): + #line = lines[i] + #if re.search("Subject:.* CN=(.*)", line): + #m = re.search("Subject:.* CN=(.*)", line) + #print "CN=", m.group(1) + #elif re.search("Subject Alternative Name:", line): + #dns = re.findall("DNS:([^,]*),", lines[i+1]) + #for d in dns: + #print d + +# try: +# smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) +# smtpserver.ehlo() +# smtpserver.starttls() +# print "Success: %s" % mx_host +# except socket.error as e: +# print "Connection to %s failed: %s" % (mx_host, e.strerror) +# pass + +def check(mail_domain): + mkdirp(mail_domain) + answers = dns.resolver.query(mail_domain, 'MX') + for rdata in answers: + mx_host = str(rdata.exchange) + print 'Host', rdata.exchange, 'has preference', rdata.preference + tls_connect(mx_host, mail_domain) + +if len(sys.argv) == 1: + print("Please pass at least one mail domain as an argument") + +for domain in sys.argv[1:]: + check(domain) diff --git a/process-google-starttls-domains.py b/process-google-starttls-domains.py new file mode 100755 index 000000000..0a0201875 --- /dev/null +++ b/process-google-starttls-domains.py @@ -0,0 +1,18 @@ +#!/usr/bin/python +import csv +import codecs +import sys +from collections import defaultdict + +csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') +d = defaultdict(set) +for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in csvreader: + if direction == "outbound": + try: + d[address_suffix].add(float(fraction_encrypted)) + except ValueError: + pass + +for address_suffix, fraction_encrypted in d.iteritems(): + if min(fraction_encrypted) >= 0.50: + print min(fraction_encrypted), address_suffix From f0b9ef2716f356dbf033f4aec71c365103429a8d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 10:43:01 -0700 Subject: [PATCH 002/639] Add a Vagrantfile and the list of golden domains. --- Vagrantfile | 27 +++++++++++++++++++++++++++ golden-domains.txt | 22 ++++++++++++++++++++++ vagrant-bootstrap.sh | 10 ++++++++++ 3 files changed, 59 insertions(+) create mode 100644 Vagrantfile create mode 100644 golden-domains.txt create mode 100755 vagrant-bootstrap.sh diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..e485988f1 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,27 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "hashicorp/precise32" + + config.vm.define "sender" do |sender| + sender.vm.network "private_network", ip: "192.168.33.5" + end + config.vm.define "valid" do |valid| + valid.vm.network "private_network", ip: "192.168.33.7" + end + config.vm.provision :shell, path: "vagrant-bootstrap.sh" + + config.vm.provider "virtualbox" do |vb| + # vb.gui = true + vb.customize ["modifyvm", :id, "--memory", "256"] + end +end diff --git a/golden-domains.txt b/golden-domains.txt new file mode 100644 index 000000000..9a0c95948 --- /dev/null +++ b/golden-domains.txt @@ -0,0 +1,22 @@ +ymail.com +yandex.ru +yahoo.co.uk +wp.pl +web.de +vtext.com +ukr.net +t-online.de +sompo-japan.co.jp +sbcglobal.net +salesforce.com +rogers.com +rocketmail.com +rambler.ru +marktplaats.nl +interia.pl +gmx.net +gmx.de +facebook.com +craigslist.org +bigpond.com +aol.com diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh new file mode 100755 index 000000000..ac9837198 --- /dev/null +++ b/vagrant-bootstrap.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export DEBIAN_FRONTEND=noninteractive + +apt-get update -q +apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postfix +cat > /etc/hosts << Date: Thu, 5 Jun 2014 11:05:08 -0700 Subject: [PATCH 003/639] Add design doc --- README.md | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..4728ea352 --- /dev/null +++ b/README.md @@ -0,0 +1,188 @@ +# STARTTLS Everywhere + +## Authors + +Jacob Hoffman-Andrews , Peter Eckersley + +## Background + +Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. + +To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. + +Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. + +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. + +STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. + +## Goals + +* Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. +* Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. +* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring + +## Non-goals + +* Prevent fully-targeted exploits of vulnerabilities on endpoints or on mail hosts. +* Refuse delivery on the recipient side if sender does not negotiate TLS (this may be a future project). +* Develop a fully-decentralized solution. +* Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. + +## Threat model + +Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. + +## Detailed design + +Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: + + (a) a configuration file format to convey STARTTLS support for recipient domains, + + (b) Python code (config-generator) to transform (a) into configuration files for popular MTAs., and + + (c) a method to create and securely distribute files of type (a) for major email domains that that agree to be included, plus any other domains that proactively request to be included. + +## File Format + +The basic file format will be JSON with comments ([](http://blog.getify.com/json-comments/))http://blog.getify.com/json-comments/). Example: + +* { +* // Canonical URL [](https://eff.org/starttls-everywhere/config)[https://eff.org/s](https://eff.org/that-email-thing)tarttls-everywhere/config -- redirects to latest version +* "timestamp": 1401093333 +* "author": "Electronic Frontier Foundation [](https://eff.org)https://eff.org", +* "expires": 1401414363, // epoch seconds +* "nexthop-domains": { +* "gmail.com": { +* "accept-mx-domains": ["google.com", "gmail.com"] +* } +* "yahoo.com": { +* "accept-mx-domains": ["yahoodns.net"] +* } +* "eff.org": { +* "accept-mx-domains": ["eff.org"] +* } +* } +* "mx-domains": { +* "eff.org": { +* "require-tls": true, +* "min-tls-version": "TLSv1.1", +* "enforce-mode": "enforce" +* } +* "google.com": { +* "require-valid-certificate": true, +* "min-tls-version": "TLSv1.1", +* "accept-pinset": "google", +* "enforce-mode": "log-only", +* // error-notification domains * +* "error-notification": "[](https://googlemail.com/post/reports/here)https://g[o](https://g)[o](https://go)[g](https://goo)[l](https://goog)[e](https://googl)[m](https://google)[a](https://googlem)[il](https://googlema)[.co](https://googlemail)[m](https://googlemail.co)[/](https://googlemail.com)post/[r](https://googlemail.com/xhr/)[e](https://googlemail.com/xhr/r)[por](https://googlemail.com/xhr/re)[t](https://googlemail.com/xhr/repor)[s](https://googlemail.com/xhr/report)[/](https://googlemail.com/xhr/reports/go)[he](https://googlemail.com/xhr/reports/go/)[r](https://googlemail.com/xhr/reports/go/he)[e](https://googlemail.com/xhr/reports/go/her)["](https://googlemail.com/xhr/reports/go/here) +* }, +* "yahoodns.net": { +* "require-valid-certificate": true, +* } +* } +* // Similar to +* // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json +* "pinsets": [ +* { +* "name": "google", +* "static_spki_hashes": [ +* "GoogleBackup2048", +* "GoogleG2" +* ] +* } +* ], +* "spki_hashes": { + +* Are base64 encoded hashes already widely used/defined? If not, I'd lean towards hex here in this JSON file, since we hopefully have gzip to reduce the encoding overhead anyway, and this file should optimize for admin ease of use. +* These are what's used for certificate pinning in Chrome. Most likely we'll have people use the same tooling to generate these SPKI hashes for us, so I think using the same format and encoding makes sense. It's also easy on admins: "We'll just use the same set of pins we use for HTTPS." +* The other thing I forgot to mention: There isn't, AFAIK, a good way to get the SPKI hash other than using the Chrome tool. It's not one of the fields output by openssl x509, for example. + +* "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", +* "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" +* } +* } + +A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. + +The _timestamp_ field is an integer number of epoch seconds. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. + +There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. + +Option 1: Plain JSON distributed with a signature using gpg --clearsign. Config-generator should validate the signature against a known GPG public key before extracting. The public key is part of the permanent system configuration, like the fetch URL. + +Option 2: Git is a revision control system built on top of an authenticated, history-preserving file system. Let's use it as an authenticated, history preserving file system: valid versions of recipient policy files may be fetched and verified via signed git tags. [Here's an example shell recipe to do this.](https://gist.github.com/jsha/6230206e89759cc6e00d) + +Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. + +**nexthop-domains** + +The _nexthop-domains _field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _nexthop-domains _section. + +Currently the only property defined for _nexthop-domains _is _accept-mx-domains_, a list. If an MX lookup for a listed nexthop domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. + +The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. + +**mx-domains** + +The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain _can be a subdomain of any other _mx-domain _in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. + +Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. + +If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ + +_Require-valid-certificate _defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: + +1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. +2. The certificate is unexpired. +3. There is a valid chain from the certificate to a root certificate included in [Mozilla's trust store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/included/) (available as [Debian package ca-certificates](https://packages.debian.org/sid/ca-certificates)). + +The _accept-pinset_ field references an entry in the pinsets list, which has the same format and semantics as [Chrome's pinning list](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json). Most _mx-domain_s should specify a pinset that describes trust roots rather than leaf certificates, but both are possible. Pinning will only be added at the request of mail operators because it requires operators be careful when issuing new leaf certificates. + +## Pinning and hostname verification + +Like Chrome (and soon Firefox) we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. + +The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. + +We do not attempt to solve the self-signed certificate problem. For mail hosts with self-signed certificates, we can require TLS but will not require validation of the certificates. Such hosts should be encouraged to upgrade to a CA-signed certificate that can be validated by senders. + +## Creating configuration + +We have three options for creating the configuration file: + +1. Ask mail operators to submit policies for their domains which we incorporate. +2. Manually curate a set of policies for the top N mail domains. +3. Programmatically create a set of policies by connecting to the top N mail domains. + +For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. + +For option (3) we'd be likely to pull in bad policies that could result in failed delivery. + +We'll initially launch a demo using option (2), do some initial deployments to prove viability and delivery rate impact, and then start reaching out to operators to do option (1). + +## Distribution + +The configuration file will be provided at a long-term maintained URL. It will be signed using a key held offline on an airgapped machine or smartcard. + +Since recipient mail servers may abruptly stop supporting TLS, we will request that mail operators set up auto-updating of the configuration file, with signature verification. This allows us to minimize the delivery impact of such events. However, config-generator should not auto-update its own code, since that would amount to auto-deployment of third party code, which some operators may not wish to do. + +We may choose to implement a form of immutable log along the lines of certificate transparency. This would be appealing if we chose to use this mechanism to distribute expected leaf keys as a primary authentication mechanism, but as described in "Pinning and hostname verification," that's not a viable option. Instead we will rely on the CA ecosystem to do primary authentication, so an immutable log for this system is probably overkill, engineering-wise. + +## Python code + +Config-generator should parse input JSON and produce output configs for various mail servers. It should not be possible for any input JSON to cause arbitrary code execution or even any MTA config directives beyond the ones that specifically impact the decision to deliver or bounce based on TLS support. For instance, it must not be possible for config-generator to output a directive to forward mail from one domain to another. Config-generator will have the option to directly pull the latest config from a URL, or from a file on local disk distributed regularly from another system that has outside network access. + +Config-generator will be manually updated by mail operators. + +## Testing + +We will create a reproducible test configuration that can be run locally and exercises each of the major cases: Enforce mode vs log mode; Enforced TLS negotiation, enforced MX hostname match, and enforced valid certificates. + +Additionally, for ongoing monitoring of third-party deployments, we will create a canary mail domain that intentionally fails one of the tests but is included in the configuration file. For instance, starttls-canary.org would be listed in the configuration as requiring STARTTLS, but would not actually offer STARTTLS. Each time a mail operator commits to configuring STARTTLS Everywhere, we would request an account on their email domain from which to send automated daily email to starttls-canary.org. We should expect bounces. If such mail is successfully delivered to starttls-canary.org, that would indicate a configuration failure on the sending host, and we would manually notify the operator. + +## Failure reporting + +For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job and send notices only when STARTTLS Everywhere-related failures exceed 0.1% for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. + +For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. From aa417eec15ab6e9442e33cf687238fbc48041d1c Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:10:07 -0700 Subject: [PATCH 004/639] Formatting issue in design doc --- README.md | 106 ++++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 4728ea352..4686d5baf 100644 --- a/README.md +++ b/README.md @@ -45,63 +45,59 @@ Senders need to know which target hosts are known to support STARTTLS, and how t ## File Format -The basic file format will be JSON with comments ([](http://blog.getify.com/json-comments/))http://blog.getify.com/json-comments/). Example: +The basic file format will be JSON with comments (http://blog.getify.com/json-comments/). Example: -* { -* // Canonical URL [](https://eff.org/starttls-everywhere/config)[https://eff.org/s](https://eff.org/that-email-thing)tarttls-everywhere/config -- redirects to latest version -* "timestamp": 1401093333 -* "author": "Electronic Frontier Foundation [](https://eff.org)https://eff.org", -* "expires": 1401414363, // epoch seconds -* "nexthop-domains": { -* "gmail.com": { -* "accept-mx-domains": ["google.com", "gmail.com"] -* } -* "yahoo.com": { -* "accept-mx-domains": ["yahoodns.net"] -* } -* "eff.org": { -* "accept-mx-domains": ["eff.org"] -* } -* } -* "mx-domains": { -* "eff.org": { -* "require-tls": true, -* "min-tls-version": "TLSv1.1", -* "enforce-mode": "enforce" -* } -* "google.com": { -* "require-valid-certificate": true, -* "min-tls-version": "TLSv1.1", -* "accept-pinset": "google", -* "enforce-mode": "log-only", -* // error-notification domains * -* "error-notification": "[](https://googlemail.com/post/reports/here)https://g[o](https://g)[o](https://go)[g](https://goo)[l](https://goog)[e](https://googl)[m](https://google)[a](https://googlem)[il](https://googlema)[.co](https://googlemail)[m](https://googlemail.co)[/](https://googlemail.com)post/[r](https://googlemail.com/xhr/)[e](https://googlemail.com/xhr/r)[por](https://googlemail.com/xhr/re)[t](https://googlemail.com/xhr/repor)[s](https://googlemail.com/xhr/report)[/](https://googlemail.com/xhr/reports/go)[he](https://googlemail.com/xhr/reports/go/)[r](https://googlemail.com/xhr/reports/go/he)[e](https://googlemail.com/xhr/reports/go/her)["](https://googlemail.com/xhr/reports/go/here) -* }, -* "yahoodns.net": { -* "require-valid-certificate": true, -* } -* } -* // Similar to -* // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json -* "pinsets": [ -* { -* "name": "google", -* "static_spki_hashes": [ -* "GoogleBackup2048", -* "GoogleG2" -* ] -* } -* ], -* "spki_hashes": { + { + // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version + "timestamp": 1401093333 + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1401414363, // epoch seconds + "nexthop-domains": { + "gmail.com": { + "accept-mx-domains": ["google.com", "gmail.com"] + } + "yahoo.com": { + "accept-mx-domains": ["yahoodns.net"] + } + "eff.org": { + "accept-mx-domains": ["eff.org"] + } + } + "mx-domains": { + "eff.org": { + "require-tls": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "enforce" + } + "google.com": { + "require-valid-certificate": true, + "min-tls-version": "TLSv1.1", + "accept-pinset": "google", + "enforce-mode": "log-only", + // error-notification domains * + "error-notification": "https://google.com/post/reports/here" + }, + "yahoodns.net": { + "require-valid-certificate": true, + } + } + // Similar to + // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json + "pinsets": [ + { + "name": "google", + "static_spki_hashes": [ + "GoogleBackup2048", + "GoogleG2" + ] + } + ], + "spki_hashes": { + "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", + "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" + } + } -* Are base64 encoded hashes already widely used/defined? If not, I'd lean towards hex here in this JSON file, since we hopefully have gzip to reduce the encoding overhead anyway, and this file should optimize for admin ease of use. -* These are what's used for certificate pinning in Chrome. Most likely we'll have people use the same tooling to generate these SPKI hashes for us, so I think using the same format and encoding makes sense. It's also easy on admins: "We'll just use the same set of pins we use for HTTPS." -* The other thing I forgot to mention: There isn't, AFAIK, a good way to get the SPKI hash other than using the Chrome tool. It's not one of the fields output by openssl x509, for example. - -* "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", -* "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" -* } -* } A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. From 3d9d5607bd51caca2202d3c79f418fcb4c0554c9 Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:34:12 -0700 Subject: [PATCH 005/639] Formatting issue #2 in design doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4686d5baf..d47ba0f70 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co } } // Similar to - // [](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json)https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json + // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json "pinsets": [ { "name": "google", From 4b5b9f164fcab8cb2ee4601cef804211d74bbdc8 Mon Sep 17 00:00:00 2001 From: jsha Date: Thu, 5 Jun 2014 11:51:19 -0700 Subject: [PATCH 006/639] nexthop-domains -> address-domains --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d47ba0f70..cc510ecfa 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Most email transferred between SMTP servers (aka MTAs) is transmitted in the cle To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. -Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. +Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. @@ -52,7 +52,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "timestamp": 1401093333 "author": "Electronic Frontier Foundation https://eff.org", "expires": 1401414363, // epoch seconds - "nexthop-domains": { + "address-domains": { "gmail.com": { "accept-mx-domains": ["google.com", "gmail.com"] } @@ -111,11 +111,11 @@ Option 2: Git is a revision control system built on top of an authenticated, his Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. -**nexthop-domains** +**address-domains** -The _nexthop-domains _field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _nexthop-domains _section. +The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. -Currently the only property defined for _nexthop-domains _is _accept-mx-domains_, a list. If an MX lookup for a listed nexthop domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. +Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. From 6fb51d54227554445ef0772eb66e8d945d4d13e8 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 14:11:08 -0700 Subject: [PATCH 007/639] Example shouldn't include hashes from Chrome source. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cc510ecfa..3ba644d7f 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,11 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" + "accept-pinset": "eff", } "google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", - "accept-pinset": "google", "enforce-mode": "log-only", // error-notification domains * "error-notification": "https://google.com/post/reports/here" @@ -85,16 +85,16 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json "pinsets": [ { - "name": "google", + "name": "eff", "static_spki_hashes": [ - "GoogleBackup2048", - "GoogleG2" + "EFFBackup2048", + "EFF" ] } ], "spki_hashes": { - "GoogleBackup2048": "sha1/vq7OyjSnqOco9nyMCDGdy77eijM=", - "GoogleG2": "sha1/Q9rWMO5T+KmAym79hfRqo3mQ4Oo=" + "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "EFF": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" } } From 714cb17dcb8a7f5aa1c78f52e7a1ebec4928fcb1 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 14:26:45 -0700 Subject: [PATCH 008/639] Clarify example usage of pinsets by including a CA --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ba644d7f..604d2dad0 100644 --- a/README.md +++ b/README.md @@ -88,13 +88,14 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "name": "eff", "static_spki_hashes": [ "EFFBackup2048", - "EFF" + "StartCom Class 2 Primary Intermediate Server CA" ] } ], "spki_hashes": { + // Not real SPKI hashes, just examples "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "EFF": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + "StartCom Class 2 Primary Intermediate Server CA": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" } } From ed0c024209a243cf4759647b93dff7281d000293 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 5 Jun 2014 16:01:07 -0700 Subject: [PATCH 009/639] Improved provisioning and certificate checking. --- Vagrantfile | 7 ++--- check-starttls.py | 74 +++++++++++++++++++++++--------------------- vagrant-bootstrap.sh | 21 +++++++++++-- 3 files changed, 60 insertions(+), 42 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index e485988f1..b990d1311 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,18 +5,15 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "hashicorp/precise32" config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" + sender.vm.hostname = "sender.example.com" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" + valid.vm.hostname = "valid-example-recipient.com" end config.vm.provision :shell, path: "vagrant-bootstrap.sh" diff --git a/check-starttls.py b/check-starttls.py index 15bc7ce64..72f0d3430 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -35,7 +35,7 @@ def extract_names(pem): # >>> twitter_cert.get_ext('subjectAltName').get_value() # 'DNS:www.twitter.com, DNS:twitter.com' alt_names = leaf.get_ext('subjectAltName').get_value() - alt_names = alt_names.split(',') + alt_names = alt_names.split(', ') alt_names = [name.partition(':') for name in alt_names] alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] except: @@ -43,48 +43,52 @@ def extract_names(pem): return set(common_names + alt_names) def tls_connect(mx_host, mail_domain): - # smtplib doesn't let us access certificate information, - # so shell out to openssl. - output = subprocess.check_output( - """openssl s_client \ - -CApath /usr/share/ca-certificates/mozilla/ \ - -starttls smtp -connect %s:25 -showcerts /dev/null + """ % mx_host, shell=True) + except subprocess.CalledProcessError: + print "Failed s_client" + return - # Save a copy of the certificate for later analysis - with open(os.path.join(mail_domain, mx_host), "w") as f: - f.write(output) + # Save a copy of the certificate for later analysis + with open(os.path.join(mail_domain, mx_host), "w") as f: + f.write(output) - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) - if len(cert) > 0: - print "iiii ", len(cert) - print extract_names(cert[0]) - #lines = output.split("\n") - #for i in range(0, len(lines)): - #line = lines[i] - #if re.search("Subject:.* CN=(.*)", line): - #m = re.search("Subject:.* CN=(.*)", line) - #print "CN=", m.group(1) - #elif re.search("Subject Alternative Name:", line): - #dns = re.findall("DNS:([^,]*),", lines[i+1]) - #for d in dns: - #print d + cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) + if len(cert) > 0: + names = extract_names(cert[0]) + print "%s certificate -> %s" % (mx_host, ", ".join(names)) + if mx_host in names: + print " MATCH" + else: + print " NOMATCH" -# try: -# smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) -# smtpserver.ehlo() -# smtpserver.starttls() -# print "Success: %s" % mx_host -# except socket.error as e: -# print "Connection to %s failed: %s" % (mx_host, e.strerror) -# pass +def supports_starttls(mx_host): + try: + smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) + smtpserver.ehlo() + smtpserver.starttls() + return True + print "Success: %s" % mx_host + except socket.error as e: + print "Connection to %s failed: %s" % (mx_host, e.strerror) + return False + except smtplib.SMTPException: + print "No STARTTLS support on %s" % mx_host + return False def check(mail_domain): mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: - mx_host = str(rdata.exchange) - print 'Host', rdata.exchange, 'has preference', rdata.preference + mx_host = str(rdata.exchange).rstrip(".") tls_connect(mx_host, mail_domain) if len(sys.argv) == 1: diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index ac9837198..215abacfd 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -3,8 +3,25 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -q -apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postfix -cat > /etc/hosts <<> /etc/hosts < /etc/dnsmasq.conf +# Not sure why restart is necessary, but otherwise dnsmasq doesn't use +# /etc/hosts to answer queries. +/etc/init.d/dnsmasq restart + +if [ "`hostname`" = "sender" ]; then + crontab < Date: Thu, 5 Jun 2014 16:51:38 -0700 Subject: [PATCH 010/639] Start work on forcing TLS to valid --- Vagrantfile | 2 + vagrant-bootstrap.sh | 2 +- vm-postfix-config/dynamicmaps.cf | 6 ++ vm-postfix-config/main.cf | 39 ++++++++ vm-postfix-config/master.cf | 113 +++++++++++++++++++++++ vm-postfix-config/starttls-everywhere.cf | 1 + vm-postfix-config/tls_policy | 1 + 7 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 vm-postfix-config/dynamicmaps.cf create mode 100644 vm-postfix-config/main.cf create mode 100644 vm-postfix-config/master.cf create mode 100644 vm-postfix-config/starttls-everywhere.cf create mode 100644 vm-postfix-config/tls_policy diff --git a/Vagrantfile b/Vagrantfile index b990d1311..ab5d43307 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,4 +21,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # vb.gui = true vb.customize ["modifyvm", :id, "--memory", "256"] end + + config.vm.synced_folder "vm-postfix-config", "/etc/postfix" end diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 215abacfd..9ec1e3d11 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -4,7 +4,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -q apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - postfix dnsmasq mutt + postfix dnsmasq mutt vim # Provide hostnames so the boxes can talk to each other. DNSMasq will also serve # results to each box based on these contents. diff --git a/vm-postfix-config/dynamicmaps.cf b/vm-postfix-config/dynamicmaps.cf new file mode 100644 index 000000000..1c48bdcde --- /dev/null +++ b/vm-postfix-config/dynamicmaps.cf @@ -0,0 +1,6 @@ +# Postfix dynamic maps configuration file. +# +#type location of .so file open function (mkmap func) +#==== ================================ ============= ============ +tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open +sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config/main.cf b/vm-postfix-config/main.cf new file mode 100644 index 000000000..b9f265058 --- /dev/null +++ b/vm-postfix-config/main.cf @@ -0,0 +1,39 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +myhostname = sender.example.com +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = sender.example.com, localhost.example.com, , localhost +relayhost = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all diff --git a/vm-postfix-config/master.cf b/vm-postfix-config/master.cf new file mode 100644 index 000000000..3df29d8a9 --- /dev/null +++ b/vm-postfix-config/master.cf @@ -0,0 +1,113 @@ +# +# Postfix master process configuration file. For details on the format +# of the file, see the master(5) manual page (command: "man 5 master"). +# +# Do not forget to execute "postfix reload" after editing this file. +# +# ========================================================================== +# service type private unpriv chroot wakeup maxproc command + args +# (yes) (yes) (yes) (never) (100) +# ========================================================================== +smtp inet n - - - - smtpd +#smtp inet n - - - 1 postscreen +#smtpd pass - - - - - smtpd +#dnsblog unix - - - - 0 dnsblog +#tlsproxy unix - - - - 0 tlsproxy +#submission inet n - - - - smtpd +# -o syslog_name=postfix/submission +# -o smtpd_tls_security_level=encrypt +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_client_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#smtps inet n - - - - smtpd +# -o syslog_name=postfix/smtps +# -o smtpd_tls_wrappermode=yes +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_client_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#628 inet n - - - - qmqpd +pickup fifo n - - 60 1 pickup +cleanup unix n - - - 0 cleanup +qmgr fifo n - n 300 1 qmgr +#qmgr fifo n - n 300 1 oqmgr +tlsmgr unix - - - 1000? 1 tlsmgr +rewrite unix - - - - - trivial-rewrite +bounce unix - - - - 0 bounce +defer unix - - - - 0 bounce +trace unix - - - - 0 bounce +verify unix - - - - 1 verify +flush unix n - - 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - - - - smtp +relay unix - - - - - smtp +# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 +showq unix n - - - - showq +error unix - - - - - error +retry unix - - - - - error +discard unix - - - - - discard +local unix - n n - - local +virtual unix - n n - - virtual +lmtp unix - - - - - lmtp +anvil unix - - - - 1 anvil +scache unix - - - - 1 scache +# +# ==================================================================== +# Interfaces to non-Postfix software. Be sure to examine the manual +# pages of the non-Postfix software to find out what options it wants. +# +# Many of the following services use the Postfix pipe(8) delivery +# agent. See the pipe(8) man page for information about ${recipient} +# and other message envelope options. +# ==================================================================== +# +# maildrop. See the Postfix MAILDROP_README file for details. +# Also specify in main.cf: maildrop_destination_recipient_limit=1 +# +maildrop unix - n n - - pipe + flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} +# +# ==================================================================== +# +# Recent Cyrus versions can use the existing "lmtp" master.cf entry. +# +# Specify in cyrus.conf: +# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 +# +# Specify in main.cf one or more of the following: +# mailbox_transport = lmtp:inet:localhost +# virtual_transport = lmtp:inet:localhost +# +# ==================================================================== +# +# Cyrus 2.1.5 (Amos Gouaux) +# Also specify in main.cf: cyrus_destination_recipient_limit=1 +# +#cyrus unix - n n - - pipe +# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} +# +# ==================================================================== +# Old example of delivery via Cyrus. +# +#old-cyrus unix - n n - - pipe +# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} +# +# ==================================================================== +# +# See the Postfix UUCP_README file for configuration details. +# +uucp unix - n n - - pipe + flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) +# +# Other external delivery methods. +# +ifmail unix - n n - - pipe + flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) +bsmtp unix - n n - - pipe + flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient +scalemail-backend unix - n n - 2 pipe + flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} +mailman unix - n n - - pipe + flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py + ${nexthop} ${user} + diff --git a/vm-postfix-config/starttls-everywhere.cf b/vm-postfix-config/starttls-everywhere.cf new file mode 100644 index 000000000..98b1bfb3a --- /dev/null +++ b/vm-postfix-config/starttls-everywhere.cf @@ -0,0 +1 @@ +smtp_tls_policy_maps = hash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/tls_policy b/vm-postfix-config/tls_policy new file mode 100644 index 000000000..a40e02ffd --- /dev/null +++ b/vm-postfix-config/tls_policy @@ -0,0 +1 @@ +valid-example-recipient.com encrypt From 7bd06a4d353400307c22672fe54127d034219451 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 5 Jun 2014 17:04:28 -0700 Subject: [PATCH 011/639] Work in progress --- vm-postfix-config/main.cf | 3 +++ vm-postfix-config/starttls-everywhere.cf | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 vm-postfix-config/starttls-everywhere.cf diff --git a/vm-postfix-config/main.cf b/vm-postfix-config/main.cf index b9f265058..d21969005 100644 --- a/vm-postfix-config/main.cf +++ b/vm-postfix-config/main.cf @@ -37,3 +37,6 @@ mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all + +#STARTTLS EVERYWHERE MAGIC STARTS HERE +smtp_tls_policy_maps = hash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/starttls-everywhere.cf b/vm-postfix-config/starttls-everywhere.cf deleted file mode 100644 index 98b1bfb3a..000000000 --- a/vm-postfix-config/starttls-everywhere.cf +++ /dev/null @@ -1 +0,0 @@ -smtp_tls_policy_maps = hash:/etc/postfix/tls_policy From 30938260d4a4c586da082bc44193bb270d86443f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 5 Jun 2014 19:45:21 -0700 Subject: [PATCH 012/639] Split postfix configs across the VMs, and start making them do things --- Vagrantfile | 3 ++- {vm-postfix-config => vm-postfix-config-sender}/dynamicmaps.cf | 0 {vm-postfix-config => vm-postfix-config-sender}/main.cf | 2 +- {vm-postfix-config => vm-postfix-config-sender}/master.cf | 0 vm-postfix-config-sender/tls_policy | 1 + vm-postfix-config/tls_policy | 1 - 6 files changed, 4 insertions(+), 3 deletions(-) rename {vm-postfix-config => vm-postfix-config-sender}/dynamicmaps.cf (100%) rename {vm-postfix-config => vm-postfix-config-sender}/main.cf (95%) rename {vm-postfix-config => vm-postfix-config-sender}/master.cf (100%) create mode 100644 vm-postfix-config-sender/tls_policy delete mode 100644 vm-postfix-config/tls_policy diff --git a/Vagrantfile b/Vagrantfile index ab5d43307..c919f16a0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -10,10 +10,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" sender.vm.hostname = "sender.example.com" + config.vm.synced_folder "vm-postfix-config-sender", "/etc/postfix" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" valid.vm.hostname = "valid-example-recipient.com" + config.vm.synced_folder "vm-postfix-config-valid", "/etc/postfix" end config.vm.provision :shell, path: "vagrant-bootstrap.sh" @@ -22,5 +24,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.customize ["modifyvm", :id, "--memory", "256"] end - config.vm.synced_folder "vm-postfix-config", "/etc/postfix" end diff --git a/vm-postfix-config/dynamicmaps.cf b/vm-postfix-config-sender/dynamicmaps.cf similarity index 100% rename from vm-postfix-config/dynamicmaps.cf rename to vm-postfix-config-sender/dynamicmaps.cf diff --git a/vm-postfix-config/main.cf b/vm-postfix-config-sender/main.cf similarity index 95% rename from vm-postfix-config/main.cf rename to vm-postfix-config-sender/main.cf index d21969005..2d926a657 100644 --- a/vm-postfix-config/main.cf +++ b/vm-postfix-config-sender/main.cf @@ -39,4 +39,4 @@ recipient_delimiter = + inet_interfaces = all #STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = hash:/etc/postfix/tls_policy +smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy diff --git a/vm-postfix-config/master.cf b/vm-postfix-config-sender/master.cf similarity index 100% rename from vm-postfix-config/master.cf rename to vm-postfix-config-sender/master.cf diff --git a/vm-postfix-config-sender/tls_policy b/vm-postfix-config-sender/tls_policy new file mode 100644 index 000000000..7792bad7f --- /dev/null +++ b/vm-postfix-config-sender/tls_policy @@ -0,0 +1 @@ +valid-example-recipient.com encrypt protocols=TLSv1.2 diff --git a/vm-postfix-config/tls_policy b/vm-postfix-config/tls_policy deleted file mode 100644 index a40e02ffd..000000000 --- a/vm-postfix-config/tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com encrypt From fa5acdf674f36c211927fe1695e68d8dfd5ec785 Mon Sep 17 00:00:00 2001 From: jsha Date: Fri, 6 Jun 2014 13:44:03 -0700 Subject: [PATCH 013/639] Simplify SPKI hash usage --- README.md | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 604d2dad0..0cc4f5f05 100644 --- a/README.md +++ b/README.md @@ -68,35 +68,21 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" - "accept-pinset": "eff", + "accept-spki-hashes": [ + "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + ] } "google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", - // error-notification domains * "error-notification": "https://google.com/post/reports/here" }, "yahoodns.net": { "require-valid-certificate": true, } } - // Similar to - // https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json - "pinsets": [ - { - "name": "eff", - "static_spki_hashes": [ - "EFFBackup2048", - "StartCom Class 2 Primary Intermediate Server CA" - ] - } - ], - "spki_hashes": { - // Not real SPKI hashes, just examples - "EFFBackup2048": "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "StartCom Class 2 Primary Intermediate Server CA": "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - } } From ce0a6a1814ba1f0d222088a5a569e8a9bd945e16 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 14:04:38 -0700 Subject: [PATCH 014/639] Add example config.json --- config.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 config.json diff --git a/config.json b/config.json new file mode 100644 index 000000000..d660587f1 --- /dev/null +++ b/config.json @@ -0,0 +1,17 @@ +{ + // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to + // latest version + "timestamp": 1401093333 + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1404677353, // epoch seconds + "address-domains": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "valid-example-recipient.com" ] + } + } + "mx-domains": { + "valid-example-recipient.com": { + "min-tls-version": "TLSv1.1" + } + } +} From 372c96d9fd4fc529ab82804d79904118a3d582e2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 14:06:08 -0700 Subject: [PATCH 015/639] Update Postfix configuration and mail-send-loop --- vagrant-bootstrap.sh | 6 +- vm-postfix-config-sender/main.cf | 4 + vm-postfix-config-sender/tls_policy | 2 +- vm-postfix-config-valid/dynamicmaps.cf | 6 ++ vm-postfix-config-valid/main.cf | 42 +++++++++ vm-postfix-config-valid/master.cf | 113 +++++++++++++++++++++++++ vm-postfix-config-valid/tls_policy | 1 + 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 vm-postfix-config-valid/dynamicmaps.cf create mode 100644 vm-postfix-config-valid/main.cf create mode 100644 vm-postfix-config-valid/master.cf create mode 100644 vm-postfix-config-valid/tls_policy diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 9ec1e3d11..1df09a9a6 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -20,8 +20,8 @@ echo selfmx > /etc/dnsmasq.conf /etc/init.d/dnsmasq restart if [ "`hostname`" = "sender" ]; then - crontab < Date: Fri, 6 Jun 2014 15:54:22 -0700 Subject: [PATCH 016/639] Some tweaks to the config format --- README.md | 21 +++++++++++---------- config.json | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0cc4f5f05..31495a8f5 100644 --- a/README.md +++ b/README.md @@ -49,22 +49,23 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co { // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version - "timestamp": 1401093333 + "timestamp": "2014-06-06T14:30:16+00:00", + // "timestamp": 1401414363, : also acceptable "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1401414363, // epoch seconds - "address-domains": { + "expires": "2014-06-06T14:30:16+00:00", + "acceptable-mxs": { "gmail.com": { - "accept-mx-domains": ["google.com", "gmail.com"] + "accept-mx-domains": ["*.google.com", "*.gmail.com"] } "yahoo.com": { - "accept-mx-domains": ["yahoodns.net"] + "accept-mx-domains": ["*.yahoodns.net"] } "eff.org": { - "accept-mx-domains": ["eff.org"] + "accept-mx-domains": ["*.eff.org"] } } - "mx-domains": { - "eff.org": { + "security-policies": { + "*.eff.org": { "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" @@ -73,13 +74,13 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" ] } - "google.com": { + "*.google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", "error-notification": "https://google.com/post/reports/here" }, - "yahoodns.net": { + "*.yahoodns.net": { "require-valid-certificate": true, } } diff --git a/config.json b/config.json index d660587f1..8d6a73d63 100644 --- a/config.json +++ b/config.json @@ -1,17 +1,17 @@ { - // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to - // latest version - "timestamp": 1401093333 + "comment": "Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version", + "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404677353, // epoch seconds - "address-domains": { - "valid-example-recipient.com": { - "accept-mx-domains": [ "valid-example-recipient.com" ] - } - } - "mx-domains": { - "valid-example-recipient.com": { + "expires": 1404677353, "comment 2:": "epoch seconds", + "tls-policies": { + "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } } + "acceptable-mxs": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "*.valid-example-recipient.com" ] + } + }, + } From fcd1a982010ff6fdba113ae208aad4b93750f83e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 6 Jun 2014 15:54:33 -0700 Subject: [PATCH 017/639] Break ground on a config parser --- config-parser.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 config-parser.py diff --git a/config-parser.py b/config-parser.py new file mode 100755 index 000000000..69a09a745 --- /dev/null +++ b/config-parser.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import sys +import json +from datetime import datetime + +def parse_timestamp(ts): + try: + int(ts) + dt = datetime.fromtimestamp(ts) + return dt + except: + raise ValueError, "Invalid timestamp integer: " + `ts` + + +class Config: + def __init__(self, cfg_file_name = "config.json"): + f = open(cfg_file_name) + cfg = json.loads(f.read()) + for atr, val in cfg.items(): + #print atr,val + # Parse and verify each attribute of the structure + if atr.startswith("comment"): + continue + if atr == "author": + if type(val) not in [str, unicode]: + raise TypeError, "Author must be a string: " + `val` + elif atr == "timestamp": + self.timestamp = parse_timestamp(val) + elif atr == "expires": + self.expires = parse_timestamp(val) + elif atr == "address-domains": + if type(val) != dict: + raise TypeError, "address-domains " + `val` + else: + sys.stderr.write("Uknown attribute: " + `atr` + "\n") + +if __name__ == "__main__": + c = Config() From e534a43d1ad2d9a75a5d585034383077997d6e2d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 6 Jun 2014 16:07:38 -0700 Subject: [PATCH 018/639] Make sender actually attempt TLS on outbound connections. --- vm-postfix-config-sender/main.cf | 3 +-- vm-postfix-config-sender/tls_policy | 2 +- vm-postfix-config-valid/main.cf | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vm-postfix-config-sender/main.cf b/vm-postfix-config-sender/main.cf index c4e994063..cf74d122f 100644 --- a/vm-postfix-config-sender/main.cf +++ b/vm-postfix-config-sender/main.cf @@ -40,7 +40,6 @@ inet_interfaces = all #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy -smtpd_tls_loglevel = 1 -smtpd_tls_received_header = yes smtp_tls_loglevel = 1 +smtp_tls_security_level = may diff --git a/vm-postfix-config-sender/tls_policy b/vm-postfix-config-sender/tls_policy index f8d6a4968..af948c5e7 100644 --- a/vm-postfix-config-sender/tls_policy +++ b/vm-postfix-config-sender/tls_policy @@ -1 +1 @@ -valid-example-recipient.com encrypt protocols=TLSv1.1 +#valid-example-recipient.com encrypt protocols=TLSv1.1 diff --git a/vm-postfix-config-valid/main.cf b/vm-postfix-config-valid/main.cf index 140b6f35d..a5f7ce575 100644 --- a/vm-postfix-config-valid/main.cf +++ b/vm-postfix-config-valid/main.cf @@ -38,5 +38,9 @@ mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all +# STARTLS Everywhere recommended best-practice settings +smtpd_tls_session_cache_timeout = 3600s +smtpd_tls_received_header = yes + #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy From 839c5230483cff26b31ed689ff0408f1b7ad7b01 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 8 Jun 2014 06:22:22 -0700 Subject: [PATCH 019/639] Fix typos --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 8d6a73d63..1a9034545 100644 --- a/config.json +++ b/config.json @@ -7,11 +7,11 @@ "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } - } + }, "acceptable-mxs": { "valid-example-recipient.com": { "accept-mx-domains": [ "*.valid-example-recipient.com" ] } - }, + } } From c033905b162650e41995bec30c9e79bf66642945 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 8 Jun 2014 06:22:32 -0700 Subject: [PATCH 020/639] Now validating most of the config json --- config-parser.py | 58 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/config-parser.py b/config-parser.py index 69a09a745..dbc244b25 100755 --- a/config-parser.py +++ b/config-parser.py @@ -3,6 +3,8 @@ import sys import json from datetime import datetime +import string + def parse_timestamp(ts): try: @@ -12,14 +14,30 @@ def parse_timestamp(ts): except: raise ValueError, "Invalid timestamp integer: " + `ts` +legal = string.letters + string.digits + ".-" +known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list +def looks_like_a_domain(s): + "Return true if string looks like a domain, as best we can tell..." + global known_tlds + try: + domain = s.lower() + assert domain[0].islower() + assert all([c in legal for c in domain]) + tld = s.split(".")[-1] + if tld not in known_tlds: + # XXX perform DNS query to determine that this TLD exists + pass + return True + except: + return False class Config: def __init__(self, cfg_file_name = "config.json"): f = open(cfg_file_name) - cfg = json.loads(f.read()) - for atr, val in cfg.items(): + self.cfg = json.loads(f.read()) + for atr, val in self.cfg.items(): #print atr,val - # Parse and verify each attribute of the structure + # Verify each attribute of the structure if atr.startswith("comment"): continue if atr == "author": @@ -29,11 +47,39 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "address-domains": - if type(val) != dict: - raise TypeError, "address-domains " + `val` + elif atr == "tls-policies": + self.tls_policies = {} + for domain,policies in self.check_tls_policy_domains(val): + if type(policies) != dict: + raise TypeError, domain + "'s policies should be a dict: " + `policies` + self.tls_policies[domain] = {} # being here enforces TLS at all + for policy, value in policies.items(): + if policy == "min-tls-version": + reasonable = ["TLS", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + if not value in reasonable: + raise ValueError, "Not a valid TLS version string: " + `value` + self.tls_policies[domain]["min-tls-version"] = str(value) + elif atr == "acceptable-mxs": + pass else: sys.stderr.write("Uknown attribute: " + `atr` + "\n") + print self.tls_policies + + def check_tls_policy_domains(self, val): + if type(val) != dict: + raise TypeError, "tls-policies should be a dict" + `val` + for domain, policies in val.items(): + try: + assert type(domain) == unicode + d = str(domain) # convert from unicode + except: + raise TypeError, "tls-policy domain not a string" + `domain` + if not d.startswith("*."): + raise ValueError, "tls-policy domains must start with *.; try *."+d + d = d.partition("*.")[2] + if not looks_like_a_domain(d): + raise ValueError, "tls-policy for something that a domain? " + d + yield (d, policies) if __name__ == "__main__": c = Config() From 79924108c7beae6c513215d102d4fffba511d794 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 10:12:09 -0700 Subject: [PATCH 021/639] Reorder JSON file to emphasize MX policies over address-domain -> MX domain mapping. --- README.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0cc4f5f05..e252681fc 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,11 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "timestamp": 1401093333 "author": "Electronic Frontier Foundation https://eff.org", "expires": 1401414363, // epoch seconds - "address-domains": { - "gmail.com": { - "accept-mx-domains": ["google.com", "gmail.com"] - } - "yahoo.com": { - "accept-mx-domains": ["yahoodns.net"] - } - "eff.org": { - "accept-mx-domains": ["eff.org"] - } - } "mx-domains": { - "eff.org": { + "*.yahoodns.net": { + "require-valid-certificate": true, + } + "*.eff.org": { "require-tls": true, "min-tls-version": "TLSv1.1", "enforce-mode": "enforce" @@ -73,15 +65,25 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" ] } - "google.com": { + "*.google.com": { "require-valid-certificate": true, "min-tls-version": "TLSv1.1", "enforce-mode": "log-only", "error-notification": "https://google.com/post/reports/here" }, - "yahoodns.net": { - "require-valid-certificate": true, - } + } + // Since the MX lookup is not secure, we list valid responses to protect + // against DNS spoofing. + "address-domains": { + "yahoo.com": { + "accept-mx-domains": ["*.yahoodns.net"] + } + "gmail.com": { + "accept-mx-domains": ["*.google.com"] + } + "eff.org": { + "accept-mx-domains": ["*.eff.org"] + } } } From 0d43d2988a062d819663b282668dc81bff440be6 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 13:08:01 -0700 Subject: [PATCH 022/639] Update check-starttls.py to generate starttls everywhere config. --- check-starttls.py | 104 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/check-starttls.py b/check-starttls.py index 72f0d3430..042d2e50d 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -6,6 +6,7 @@ import smtplib import socket import subprocess import re +import json import dns.resolver from M2Crypto import X509 @@ -19,9 +20,9 @@ def mkdirp(path): else: raise def extract_names(pem): + """Return a list of DNS subject names from PEM-encoded leaf cert.""" leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) - """Extracts a list of DNS names associated with the leaf cert.""" subj = leaf.get_subject() # Certs have a "subject" identified by a Distingushed Name (DN). # Host certs should also have a Common Name (CN) with a DNS name. @@ -43,16 +44,16 @@ def extract_names(pem): return set(common_names + alt_names) def tls_connect(mx_host, mail_domain): + """Attempt a STARTTLS connection with openssl and save the output.""" if supports_starttls(mx_host): # smtplib doesn't let us access certificate information, # so shell out to openssl. try: output = subprocess.check_output( """openssl s_client \ - -CApath /usr/share/ca-certificates/mozilla/ \ -starttls smtp -connect %s:25 -showcerts /dev/null - """ % mx_host, shell=True) + """ % mx_host, shell = True) except subprocess.CalledProcessError: print "Failed s_client" return @@ -61,14 +62,53 @@ def tls_connect(mx_host, mail_domain): with open(os.path.join(mail_domain, mx_host), "w") as f: f.write(output) - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", output, flags = re.DOTALL) - if len(cert) > 0: - names = extract_names(cert[0]) - print "%s certificate -> %s" % (mx_host, ", ".join(names)) - if mx_host in names: - print " MATCH" - else: - print " NOMATCH" +def valid_cert(filename): + """Return true if the certificate is valid. + + Note: CApath must have hashed symlinks to the trust roots. + TODO: Include the -attime flag based on file modification time.""" + + if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: + return False + try: + output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ + -untrusted "%s" \ + "%s" + """ % (filename, filename), shell = True) + return True + except subprocess.CalledProcessError: + return False + +def check_certs(mail_domain): + names = set() + for mx_hostname in os.listdir(mail_domain): + filename = os.path.join(mail_domain, mx_hostname) + if not valid_cert(filename): + return "" + else: + new_names = extract_names_from_openssl_output(filename) + names.update(new_names) + names.add(filename.rstrip(".")) + if len(names) >= 1: + return common_suffix(names) + else: + return "" + +def common_suffix(hosts): + num_components = min(len(h.split(".")) for h in hosts) + longest_suffix = "" + for i in range(1, num_components + 1): + suffixes = set(".".join(h.split(".")[-i:]) for h in hosts) + if len(suffixes) == 1: + longest_suffix = suffixes.pop() + else: + return longest_suffix + return longest_suffix + +def extract_names_from_openssl_output(certificates_file): + openssl_output = open(certificates_file, "r").read() + cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", openssl_output, flags = re.DOTALL) + return extract_names(cert[0]) def supports_starttls(mx_host): try: @@ -84,15 +124,47 @@ def supports_starttls(mx_host): print "No STARTTLS support on %s" % mx_host return False -def check(mail_domain): +def min_tls_version(mail_domain): + protocols = [] + for mx_hostname in os.listdir(mail_domain): + filename = os.path.join(mail_domain, mx_hostname) + contents = open(filename).read() + protocol = re.findall("Protocol : (.*)", contents)[0] + protocols.append(protocol) + return min(protocols) + +def collect(mail_domain): mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: mx_host = str(rdata.exchange).rstrip(".") tls_connect(mx_host, mail_domain) -if len(sys.argv) == 1: - print("Please pass at least one mail domain as an argument") +if __name__ == '__main__': + """Consume a target list of domains and output a configuration file for those domains.""" + if len(sys.argv) == 1: + print("Please pass at least one mail domain as an argument") -for domain in sys.argv[1:]: - check(domain) + config = { + "address-domains": { + }, + "mx-domains": { + } + } + for domain in sys.argv[1:]: + #collect(domain) + if len(os.listdir(domain)) == 0: + continue + suffix = check_certs(domain) + min_version = min_tls_version(domain) + if suffix != "": + suffix_match = "*." + suffix + config["address-domains"][domain] = { + "accept-mx-domains": [suffix_match] + } + config["mx-domains"][suffix_match] = { + "require-tls": True, + "min-tls-version": min_version + } + + print json.dumps(config, indent=2) From bdbc46fc84599dd141b9bb8585f11b2d168aac1a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 9 Jun 2014 13:10:43 -0700 Subject: [PATCH 023/639] Add candidate starttls-everywhere config json --- starttls-everywhere.json | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 starttls-everywhere.json diff --git a/starttls-everywhere.json b/starttls-everywhere.json new file mode 100644 index 000000000..4c91866d5 --- /dev/null +++ b/starttls-everywhere.json @@ -0,0 +1,144 @@ +{ + "mx-domains": { + "*.mx.aol.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.psmtp.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.ukr.net": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.interia.pl": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.gmx.net": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.web.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.marktplaats.nl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.wp.pl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.yahoodns.net": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + "*.t-online.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.rambler.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + "*.t.facebook.com": { + "min-tls-version": "TLSv1", + "require-tls": true + } + }, + "address-domains": { + "wp.pl": { + "accept-mx-domains": [ + "*.wp.pl" + ] + }, + "yahoo.co.uk": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "rocketmail.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "web.de": { + "accept-mx-domains": [ + "*.web.de" + ] + }, + "sbcglobal.net": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "aol.com": { + "accept-mx-domains": [ + "*.mx.aol.com" + ] + }, + "facebook.com": { + "accept-mx-domains": [ + "*.t.facebook.com" + ] + }, + "sompo-japan.co.jp": { + "accept-mx-domains": [ + "*.psmtp.com" + ] + }, + "salesforce.com": { + "accept-mx-domains": [ + "*.psmtp.com" + ] + }, + "rambler.ru": { + "accept-mx-domains": [ + "*.rambler.ru" + ] + }, + "t-online.de": { + "accept-mx-domains": [ + "*.t-online.de" + ] + }, + "gmx.net": { + "accept-mx-domains": [ + "*.gmx.net" + ] + }, + "gmx.de": { + "accept-mx-domains": [ + "*.gmx.net" + ] + }, + "ukr.net": { + "accept-mx-domains": [ + "*.ukr.net" + ] + }, + "rogers.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "ymail.com": { + "accept-mx-domains": [ + "*.yahoodns.net" + ] + }, + "marktplaats.nl": { + "accept-mx-domains": [ + "*.marktplaats.nl" + ] + }, + "interia.pl": { + "accept-mx-domains": [ + "*.interia.pl" + ] + } + } +} From d0bcc1305964ad8e756d85c9c50a007f0ffe0933 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 10 Jun 2014 08:08:17 -0700 Subject: [PATCH 024/639] Break ground on an postfix config wrangling engine --- mta-config-generator.py | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 mta-config-generator.py diff --git a/mta-config-generator.py b/mta-config-generator.py new file mode 100644 index 000000000..0444c1e73 --- /dev/null +++ b/mta-config-generator.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +import string + +DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" + +def parse_line(self, line): + "return the and right hand sides of stripped, non-comment postfix config line" + # lines are like: + # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + left, sep, right = line.partition("=") + if not sep: + return None + return (left.strip(), right.strip()) + +#def get_cf_values(lines, var): + +class MTAConfigGenerator: + def __init__(self, stlse_config): + self.c = stlse_config + +class ExistingConfigError(ValueError): pass + +class PostfixConfigGenerator(MTAConfigGenerator): + def __init__(self, stlse_config): + MTAConfigGenerator.__init__(self, stlse_config) + this.postfix_cf_file = this.find_postfix_cf() + this.wrangle_existing_config() + + def ensure_cf_var(self, var, ideal, also_acceptable): + """ + Ensure that existing postfix config @var is in the list of @acceptable + values; if not, set it to the ideal value. """ + + acceptable = [ideal] + also_acceptable + + l = [line for line in cf if line.startswith("stmpd_use_tls")] + if not any(l): + this.additions.append("smtpd_use_tls = yes") + else: + values = [right for left, right in map(parse_line, l)] + if len(set(values)) > 1: + raise ExistingConfigError, "Conflicting existing config values " + `l` + if values[0] != "yes": + + def wrangle_existing_config(self): + "Try to ensure/mutate that the config file is in a sane state." + this.additions = [] + fn = find_postfix_cf() + raw_cf = open(fn).readlines() + cf = map(string.strip, raw_cf) + this.cf = [line for line in cf if line and not line.startswith("#")] + + # Check we're currently accepting inbound STARTTLS sensibly + this.ensure_cf_var("smtpd_use_tls", "yes", []) + # Ideally we use it opportunistically in the outbound direction + this.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + # Maximum verbosity lets us collect failure information + this.ensure_cf_var("smtp_tls_loglevel", "1", []) + # Inject a reference to our per-domain policy map + this.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + + this.maybe_add_config_lines() + + def maybe_add_config_lines(self): + if not this.additions: + return + this.additions[:0]=["","# New config lines added by STARTTLS Everywhere",""] + new_cf_lines = "\n".join(this.additions) + print "Adding to %s:" % fn + print new_cf_lines + if raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" + new_cf = "".join(raw_cf) + sep + new_cf_lines + f = open(fn, "w").write(new_cf) + f.close() + + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return "/etc/postfix/main.cf" From a03db04ff44335e1911e4a79787cdff025bb711f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 10 Jun 2014 08:08:40 -0700 Subject: [PATCH 025/639] WIP implementing deletion of existing cf lines --- mta-config-generator.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mta-config-generator.py b/mta-config-generator.py index 0444c1e73..5d79423ef 100644 --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -4,10 +4,11 @@ import string DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" -def parse_line(self, line): +def parse_line(self, line_data): "return the and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + num,line = line_data left, sep, right = line.partition("=") if not sep: return None @@ -34,22 +35,28 @@ class PostfixConfigGenerator(MTAConfigGenerator): acceptable = [ideal] + also_acceptable - l = [line for line in cf if line.startswith("stmpd_use_tls")] + l = [num,line for num,line in enumerate(cf) if line.startswith(var)] if not any(l): this.additions.append("smtpd_use_tls = yes") else: values = [right for left, right in map(parse_line, l)] if len(set(values)) > 1: + if this.fixup: + this.deletions.append( raise ExistingConfigError, "Conflicting existing config values " + `l` if values[0] != "yes": - def wrangle_existing_config(self): - "Try to ensure/mutate that the config file is in a sane state." + def wrangle_existing_config(self, fixup=false): + """ + Try to ensure/mutate that the config file is in a sane state. + Fixup means we'll delete existing lines if necessary to get there. + """ this.additions = [] fn = find_postfix_cf() raw_cf = open(fn).readlines() - cf = map(string.strip, raw_cf) - this.cf = [line for line in cf if line and not line.startswith("#")] + this.cf = map(string.strip, raw_cf) + #this.cf = [line for line in cf if line and not line.startswith("#")] + this.fixup = fixup # Check we're currently accepting inbound STARTTLS sensibly this.ensure_cf_var("smtpd_use_tls", "yes", []) From 21e841fd13c028e1db9443ae962b53353ae59dde Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 10 Jun 2014 15:08:52 -0400 Subject: [PATCH 026/639] Move synced folders into a common one. Also, create certificates. --- Vagrantfile | 3 +- check-starttls.py | 2 +- vagrant-bootstrap.sh | 6 +- vagrant-shared/certificates/ca.crt | 22 ++++ vagrant-shared/certificates/ca.key | 27 +++++ vagrant-shared/certificates/certificates | 1 + vagrant-shared/certificates/valid.crt | 21 ++++ vagrant-shared/certificates/valid.csr | 18 +++ vagrant-shared/certificates/valid.key | 27 +++++ .../postfix-config-sender-tls_policy | 0 .../postfix-config-sender.cf | 0 .../postfix-config-valid-example-recipient.cf | 0 vm-postfix-config-sender/dynamicmaps.cf | 6 - vm-postfix-config-sender/master.cf | 113 ------------------ vm-postfix-config-valid/dynamicmaps.cf | 6 - vm-postfix-config-valid/master.cf | 113 ------------------ vm-postfix-config-valid/tls_policy | 1 - 17 files changed, 123 insertions(+), 243 deletions(-) create mode 100644 vagrant-shared/certificates/ca.crt create mode 100644 vagrant-shared/certificates/ca.key create mode 120000 vagrant-shared/certificates/certificates create mode 100644 vagrant-shared/certificates/valid.crt create mode 100644 vagrant-shared/certificates/valid.csr create mode 100644 vagrant-shared/certificates/valid.key rename vm-postfix-config-sender/tls_policy => vagrant-shared/postfix-config-sender-tls_policy (100%) rename vm-postfix-config-sender/main.cf => vagrant-shared/postfix-config-sender.cf (100%) rename vm-postfix-config-valid/main.cf => vagrant-shared/postfix-config-valid-example-recipient.cf (100%) delete mode 100644 vm-postfix-config-sender/dynamicmaps.cf delete mode 100644 vm-postfix-config-sender/master.cf delete mode 100644 vm-postfix-config-valid/dynamicmaps.cf delete mode 100644 vm-postfix-config-valid/master.cf delete mode 100644 vm-postfix-config-valid/tls_policy diff --git a/Vagrantfile b/Vagrantfile index c919f16a0..3ce93c917 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -10,13 +10,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "sender" do |sender| sender.vm.network "private_network", ip: "192.168.33.5" sender.vm.hostname = "sender.example.com" - config.vm.synced_folder "vm-postfix-config-sender", "/etc/postfix" end config.vm.define "valid" do |valid| valid.vm.network "private_network", ip: "192.168.33.7" valid.vm.hostname = "valid-example-recipient.com" - config.vm.synced_folder "vm-postfix-config-valid", "/etc/postfix" end + config.vm.synced_folder "vagrant-shared", "/vagrant" config.vm.provision :shell, path: "vagrant-bootstrap.sh" config.vm.provider "virtualbox" do |vb| diff --git a/check-starttls.py b/check-starttls.py index 042d2e50d..cd8dfea42 100755 --- a/check-starttls.py +++ b/check-starttls.py @@ -152,7 +152,7 @@ if __name__ == '__main__': } } for domain in sys.argv[1:]: - #collect(domain) + collect(domain) if len(os.listdir(domain)) == 0: continue suffix = check_certs(domain) diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 1df09a9a6..13593a0cc 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -23,5 +23,9 @@ if [ "`hostname`" = "sender" ]; then (while sleep 10; do echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com done) & -EOF + ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy fi + +ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf +ln -sf "/vagrant/certificates" /etc/certificates +postfix reload diff --git a/vagrant-shared/certificates/ca.crt b/vagrant-shared/certificates/ca.crt new file mode 100644 index 000000000..dce966d32 --- /dev/null +++ b/vagrant-shared/certificates/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDmzCCAoOgAwIBAgIJAIheS+k2UDobMA0GCSqGSIb3DQEBBQUAMGQxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxDDAKBgNVBAoMA0VGRjEt +MCsGA1UECwwkU1RBUlRUTFMgRXZlcnl3aGVyZSB0ZXN0IGNlcnRpZmljYXRlMB4X +DTE0MDYxMDE4MDg0OVoXDTE5MDYxMDE4MDg1MFowZDELMAkGA1UEBhMCVVMxCzAJ +BgNVBAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRT +VEFSVFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDDYmUQUjaX6L3n9fNks2yQUFhKUBR1NYenx3w2 +DnaZiwpLzI3igJPMQfBdyJGLM1jXZZcpvpgt6yN4OMOLNS2QKBY20gDoQIh0Jmaj +KCoXUbX30H1FfTn+pyU02UpuFsFN3TAk5bQ/BQUYOlMouCowyZ25mnEzzHLeRHKH +Gi2uCH59T53rcgDwjq88pKMVUlndixkOKpeXZkTL++Edg0b5SUpRuzMs6kFmwLoQ +x4xG5lgaAHyu2/9KXhcqielE95s5FNGfi9U2q3nkmpa4dM266DWfkibfvCBP3hiO +Ks+IN+Hyohi2SY7NoDN6hMKcuUNlAK/xD+foA4Ck3R45HII3AgMBAAGjUDBOMB0G +A1UdDgQWBBRPlrBeFMPb27oWESmXykOW1XWLLDAfBgNVHSMEGDAWgBRPlrBeFMPb +27oWESmXykOW1XWLLDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBY +lBQEuRKo927jJFzgvdTKJGIAzxBnC5vg+qKeZkduwxeVEwB13HPP1syvdpAK5dkl +JGuHCF/Yc39HX1OZv7huVRMnyrSKpMt25uqUHirH6Db17HRCS4ZA6rARJfxS6RWu +J31lfXvGRr0hI4yw3XpKGM/c8Gkzji6PsYn4TCPnXjJbRj1GjHFaAuIeyoO0zQjo +OuHnzxs4bIDaV32NHNetuDKSO1GNbenxRiiN1HvQ1vfhzpqerRRaPCHrW4eUcynQ +AAeuC+Ek925t9mH7Ni/kZ0eN7XUwvg3c2lm+LeV+ICP+NWv9r92kfXDZNMhlhNsf +Y8+m1Y9WL3CLj99voNQS +-----END CERTIFICATE----- diff --git a/vagrant-shared/certificates/ca.key b/vagrant-shared/certificates/ca.key new file mode 100644 index 000000000..3ebf4159e --- /dev/null +++ b/vagrant-shared/certificates/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAw2JlEFI2l+i95/XzZLNskFBYSlAUdTWHp8d8Ng52mYsKS8yN +4oCTzEHwXciRizNY12WXKb6YLesjeDjDizUtkCgWNtIA6ECIdCZmoygqF1G199B9 +RX05/qclNNlKbhbBTd0wJOW0PwUFGDpTKLgqMMmduZpxM8xy3kRyhxotrgh+fU+d +63IA8I6vPKSjFVJZ3YsZDiqXl2ZEy/vhHYNG+UlKUbszLOpBZsC6EMeMRuZYGgB8 +rtv/Sl4XKonpRPebORTRn4vVNqt55JqWuHTNuug1n5Im37wgT94YjirPiDfh8qIY +tkmOzaAzeoTCnLlDZQCv8Q/n6AOApN0eORyCNwIDAQABAoIBAQCwNXASDSNBU1zZ +8v3kZtDVQjCuLJSWtIU4cnd6RQb/KN9LRxr7GJyyzREbc4SXduJ7uBphQov6daMS +jJcGWBpUdWK7ZB//Vhv6LJvKL7HuP/oNmhEwd2SzXkj25bTznkANmhsOW794Sm2y +0P8orRcX0u0Vc8z+Ozepby+e2qQx29FXznberRv2rXmMeeQqF+sc2MBu34SlcLnK +KVSe7SmDc+DhWJ3XiPoEpiOTbv4EynndXC85owdse/eN/EahFtrfzqm6jDWVga8A +xA/7RD/Urc2L2IsZOB92xOk/tGs5Df8ZrHavzbSo+i0pzYAKvIxr+G2Krl1Tl/Lh +IXwjWPLZAoGBAOsbdxeZUHH6pk497ICIcmW/NDhAY/mSA6vBWZWLcYF7OU/wMzZ6 +Wlx9v6oz8tBTzQj/Xkpv0ZLIeQqGTuSQzQ7EcrOiTz4PIpM5y/nf/S0oSXwJ7mJw +Sx8w3XHqHEPCT9G83o7EV7xHuRy8/zQzJ9dkFxznA9sRcO7RhE/tsrMNAoGBANS/ +Ql140oyDqyABIlxswyvfJ8Ll4kmb52yNGaCJPSfrAVmSoN15AkbDkpVKFb6hsL6u +xqVPyeVxard4twddrV3GSvTibkGw4fZ8x0FDgDGvCgJ36e4NHb7jQVwwGfNXKMAP +qvYtnE28eAM+zrDHryEhgp4k59zApphr+IU7JQlTAoGAc1h5ODm+rvzTBMX6tyC6 +R1Lkcsicg//wDx8ALY9JM8ZZ2u80oQCsPn5vPzjXYwAKMuTexNRRVJtITzKPmDG2 +eQ1GXP0/tWnFg8eyXDhZRQNj8hgJPYBsSrQ1oMLD9TZq5LKt2gtYJAZoOkI7Tsfe +Px1a/ZIVYTAQYQqnyHMM3i0CgYAQNirWeJiCwJ3PqIZ3yInu0+hxv5bIySqPaQkk +5JBWdF/79WJwvgHgZpLK8YRKrIONZEAa5MObylK5fGdmFktZs/yOQJrqQpJVeBiu +7nfcUVxP59dZnoI/w419euTfWCrwx8DdVYhtnAkBJk4VxoGf4q/TYTiR59RKFSAw +9trRpQKBgDjFQ9mXJUUKoc61P3PIrSYOlnx89c0sznrwm2eeLLAwr/VQ+vNDGRkC +6kXbVLuTxRaFnLSZTlUd0nSXPbJEM5GII48zbGZ7t/ysPtA9390xk1efTO/ryf/j +pn4FBvpgXaneRs4ExyknLgFfJRUEu17fRbyRPZr0bUlTkKg/ZS5M +-----END RSA PRIVATE KEY----- diff --git a/vagrant-shared/certificates/certificates b/vagrant-shared/certificates/certificates new file mode 120000 index 000000000..d162b42af --- /dev/null +++ b/vagrant-shared/certificates/certificates @@ -0,0 +1 @@ +/vagrant/certificates \ No newline at end of file diff --git a/vagrant-shared/certificates/valid.crt b/vagrant-shared/certificates/valid.crt new file mode 100644 index 000000000..f5fe4c213 --- /dev/null +++ b/vagrant-shared/certificates/valid.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlUCAQEwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRTVEFS +VFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwHhcNMTQwNjEwMTgxMTQ3 +WhcNMTkwNjEwMTgxMTQ3WjCBlDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQsw +CQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMTQwMgYDVQQLDCtTVEFSVFRMUyBFdmVy +eXdoZXJlIHRlc3QgY2VydGlmaWNhdGUgKGxlYWYpMScwJQYDVQQDDB5teC52YWxp +ZC1leGFtcGxlLXJlY2lwaWVudC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDZ2NdAwI0LUmXrV7bItKhP8NRzypkYHgApXaoZ6iB6TJnYb1FZyv2Y +wyBdsQQ8zcIgKRyl0rKvNPgHt3riM7nSoB16II6fIQDuR2UnXkhtDOfUd7Ye0wKX +l3A+qoEge42/TfRvzPC6IMa1KH7+dwayprIjLxFUzfMl6GqA/5auLZZtSsV6Pix3 +jXZYFPUHPoBJEyNo/bizcdvSZS/7Kwzfc64l7JLA/OGtRbQpcMAOpRRXCx32nt3N +2L1OXz8Q4It+J20oN8Vfin1a7GbAGdktRbz2bV8bmb0ux1NOnddigPUHRRYMIaKN +W7Nrxp2YQpqmsqBmVEVPA903yRc0ZvuRAgMBAAEwDQYJKoZIhvcNAQEFBQADggEB +AMJ3neFM+to/tFhTiAfUnIrQOKyfk+zYN8gC99HD2SUVbu+Cu1qCNJWxbE3bdxxX +y960yX+Or8E4nG9sWbFbzlGEzz0F8DAPEh4UCtb/5MRkhW158Y9LtbheQAQpZolQ +Sma6lnngCmSDr/OpDW34oM8S7+3VOawYv1e1ruCMqizBwilgcsGq2hY1hWca6LMP +X2FHQ+m4mYBV9wJ9WuKMs9uADz9cfMYuA/wMzNeMZyM82w+nSQGZ+mX7YPu+WBJM +k9OcFrUWxrC/uOQfsyqdjLvFjg4/b49jnusn9mZ/KbtLVmkz5P+5kwSn4kcd5Rlw +LLmEGiXiyEjo8UmEVyumrIQ= +-----END CERTIFICATE----- diff --git a/vagrant-shared/certificates/valid.csr b/vagrant-shared/certificates/valid.csr new file mode 100644 index 000000000..a58aab677 --- /dev/null +++ b/vagrant-shared/certificates/valid.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC2jCCAcICAQAwgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UE +BwwCU0YxDDAKBgNVBAoMA0VGRjE0MDIGA1UECwwrU1RBUlRUTFMgRXZlcnl3aGVy +ZSB0ZXN0IGNlcnRpZmljYXRlIChsZWFmKTEnMCUGA1UEAwwebXgudmFsaWQtZXhh +bXBsZS1yZWNpcGllbnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9RWcr9mMMgXbEE +PM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2HtMCl5dwPqqB +IHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrFej4sd412WBT1 +Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd9p7dzdi9Tl8/ +EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UWDCGijVuza8ad +mEKaprKgZlRFTwPdN8kXNGb7kQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAAhe +pMLNDAaA9fXZ/yqW4ov1pBKYZL1R1RX+YgZqUPoAlrTCiJ0UGIuPcTDGJDMdpZ7x +9gSqwsXrvzXafuHI4GuFjnbY5SIv8zvc+nMXib7IMlyMUcuSBZP8W0sl3ZGWnnpk +legC10c3+I9TfQ7Tl0mpdyf/6yLhM1plxLcIy5bguLJjbBK9JhKNfc84rivqrxUI ++cUqWU13WjMzWdKS6rK5m/Bfleg+jyZ11xYY0QfwNGwuPjfiEBjCs2iqJvdLFRem +FoDq3XBsrH5XohSwWZ6UrZY7wARkmHsYJeYTHulb3MkDPCQKlbU+2SMIpbNk43+/ +fN/ctxWXg3vd/q6eJiE= +-----END CERTIFICATE REQUEST----- diff --git a/vagrant-shared/certificates/valid.key b/vagrant-shared/certificates/valid.key new file mode 100644 index 000000000..b5e8371cd --- /dev/null +++ b/vagrant-shared/certificates/valid.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9R +Wcr9mMMgXbEEPM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2 +HtMCl5dwPqqBIHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrF +ej4sd412WBT1Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd +9p7dzdi9Tl8/EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UW +DCGijVuza8admEKaprKgZlRFTwPdN8kXNGb7kQIDAQABAoIBAHAZgUq0yt+UmxWr +oUdOj33zc5/SFU2vwm2G4U1MiUHlwRT602Xdavn9Dt6nhIK1bruV7EP4VDKMk0WF +SRq1e13DPuflcP65wPzciFTl02cqSPGwWGssMh1HtF7K5n+MlLhoqOwPDaD51MbL +++192lh8Jxar1cNJ52EOZB/VZfhiUZ88JAITWam6clId8KS4cy7SvQ9haptcPlfl +wIeCti2eI4nvC0IR0SCEqWAax6XqypA5k2fjQJklcnBK1R+H1muc00J5lMIz4U8q +qFb5RgxIDMcEeQZWmzfBjU5hO0q/51QyWFFFuBL00MkT4djCHjO/IPYc6DQoAoZR +TbMCWwECgYEA+i3W44tSY96Fdn1gDyYhDor9GPN0XwmRBHpl44TxBLG48I6cGLYI +l0InEh7pu7/0fw+cWylx2OLrl2HeGlWilEOpP2ucJcswcNibATtgSdnu7PjC4eAW +46a5+2hbFk/NONU5VTGClJHCvKzYPWMrAIkRT//WdkIVs23i7bRcOjkCgYEA3upr +cnMhnwVmZrGAIm+DtiDJCc4uVXeqLD2j9csCVNCw/CQAjJ3/5VlQDSjzerjijw/V +uoA+Su8xMoMCg6mAihqdxBMrbwdlh7vje23RaxbeQkIY5JIVAsdDr+UtwofaX3Zx +j3RW5jNeouVlvtExyKZZhyMwuh8d59m4V65/rBkCgYBH+f40EvZOQ0v0jhef5CFo +lLZCgnB9kzwEpM5BihLpfdQuaWkhduW71s102i321UAbejtKwv69HnQXZpHG09Jl +g53i4CvZd77lCHx3+0Q1mxyxUtSGtbkAIAyr9xcVsTni2v2WtBrUcacsLzI7XxeV +HNo9QObLuTGTIM9EAjryiQKBgQCIZEBX56/jn6c3IFX5O+gH8OlxEXFyI+TAavq+ +Mnd7s7EGpXSclTP0fYAofSz0otkklZi9Iyh6Kv4cHOLV8klOtthfFyeVKJ5rvX+D +jv76miRlwBGBEQzABXIZ1oz4IK1xiYQUNSfSdA3sd5WYemEOlxHiSJrQ1qcyrBlJ +tOAzSQKBgQDjLQIZ0rmSAMUCGi7WlDle/pnf6sUEFUPWUqzHOJbJk/CZIIf1IT6o +Evep9eeP6QjiupusY/uSlgjjjj7yxZU5q3VV/0cQl6QhwxG9wqpa8x32wwaM1WBZ +rJgsuSjcEoluAqfX/bqRtumNtQ213DSADzSCpOetlJb+ARANk7YULg== +-----END RSA PRIVATE KEY----- diff --git a/vm-postfix-config-sender/tls_policy b/vagrant-shared/postfix-config-sender-tls_policy similarity index 100% rename from vm-postfix-config-sender/tls_policy rename to vagrant-shared/postfix-config-sender-tls_policy diff --git a/vm-postfix-config-sender/main.cf b/vagrant-shared/postfix-config-sender.cf similarity index 100% rename from vm-postfix-config-sender/main.cf rename to vagrant-shared/postfix-config-sender.cf diff --git a/vm-postfix-config-valid/main.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf similarity index 100% rename from vm-postfix-config-valid/main.cf rename to vagrant-shared/postfix-config-valid-example-recipient.cf diff --git a/vm-postfix-config-sender/dynamicmaps.cf b/vm-postfix-config-sender/dynamicmaps.cf deleted file mode 100644 index 1c48bdcde..000000000 --- a/vm-postfix-config-sender/dynamicmaps.cf +++ /dev/null @@ -1,6 +0,0 @@ -# Postfix dynamic maps configuration file. -# -#type location of .so file open function (mkmap func) -#==== ================================ ============= ============ -tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open -sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config-sender/master.cf b/vm-postfix-config-sender/master.cf deleted file mode 100644 index 3df29d8a9..000000000 --- a/vm-postfix-config-sender/master.cf +++ /dev/null @@ -1,113 +0,0 @@ -# -# Postfix master process configuration file. For details on the format -# of the file, see the master(5) manual page (command: "man 5 master"). -# -# Do not forget to execute "postfix reload" after editing this file. -# -# ========================================================================== -# service type private unpriv chroot wakeup maxproc command + args -# (yes) (yes) (yes) (never) (100) -# ========================================================================== -smtp inet n - - - - smtpd -#smtp inet n - - - 1 postscreen -#smtpd pass - - - - - smtpd -#dnsblog unix - - - - 0 dnsblog -#tlsproxy unix - - - - 0 tlsproxy -#submission inet n - - - - smtpd -# -o syslog_name=postfix/submission -# -o smtpd_tls_security_level=encrypt -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#smtps inet n - - - - smtpd -# -o syslog_name=postfix/smtps -# -o smtpd_tls_wrappermode=yes -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#628 inet n - - - - qmqpd -pickup fifo n - - 60 1 pickup -cleanup unix n - - - 0 cleanup -qmgr fifo n - n 300 1 qmgr -#qmgr fifo n - n 300 1 oqmgr -tlsmgr unix - - - 1000? 1 tlsmgr -rewrite unix - - - - - trivial-rewrite -bounce unix - - - - 0 bounce -defer unix - - - - 0 bounce -trace unix - - - - 0 bounce -verify unix - - - - 1 verify -flush unix n - - 1000? 0 flush -proxymap unix - - n - - proxymap -proxywrite unix - - n - 1 proxymap -smtp unix - - - - - smtp -relay unix - - - - - smtp -# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 -showq unix n - - - - showq -error unix - - - - - error -retry unix - - - - - error -discard unix - - - - - discard -local unix - n n - - local -virtual unix - n n - - virtual -lmtp unix - - - - - lmtp -anvil unix - - - - 1 anvil -scache unix - - - - 1 scache -# -# ==================================================================== -# Interfaces to non-Postfix software. Be sure to examine the manual -# pages of the non-Postfix software to find out what options it wants. -# -# Many of the following services use the Postfix pipe(8) delivery -# agent. See the pipe(8) man page for information about ${recipient} -# and other message envelope options. -# ==================================================================== -# -# maildrop. See the Postfix MAILDROP_README file for details. -# Also specify in main.cf: maildrop_destination_recipient_limit=1 -# -maildrop unix - n n - - pipe - flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} -# -# ==================================================================== -# -# Recent Cyrus versions can use the existing "lmtp" master.cf entry. -# -# Specify in cyrus.conf: -# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 -# -# Specify in main.cf one or more of the following: -# mailbox_transport = lmtp:inet:localhost -# virtual_transport = lmtp:inet:localhost -# -# ==================================================================== -# -# Cyrus 2.1.5 (Amos Gouaux) -# Also specify in main.cf: cyrus_destination_recipient_limit=1 -# -#cyrus unix - n n - - pipe -# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} -# -# ==================================================================== -# Old example of delivery via Cyrus. -# -#old-cyrus unix - n n - - pipe -# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} -# -# ==================================================================== -# -# See the Postfix UUCP_README file for configuration details. -# -uucp unix - n n - - pipe - flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) -# -# Other external delivery methods. -# -ifmail unix - n n - - pipe - flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) -bsmtp unix - n n - - pipe - flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient -scalemail-backend unix - n n - 2 pipe - flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} -mailman unix - n n - - pipe - flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py - ${nexthop} ${user} - diff --git a/vm-postfix-config-valid/dynamicmaps.cf b/vm-postfix-config-valid/dynamicmaps.cf deleted file mode 100644 index 1c48bdcde..000000000 --- a/vm-postfix-config-valid/dynamicmaps.cf +++ /dev/null @@ -1,6 +0,0 @@ -# Postfix dynamic maps configuration file. -# -#type location of .so file open function (mkmap func) -#==== ================================ ============= ============ -tcp /usr/lib/postfix/dict_tcp.so dict_tcp_open -sqlite /usr/lib/postfix/dict_sqlite.so dict_sqlite_open diff --git a/vm-postfix-config-valid/master.cf b/vm-postfix-config-valid/master.cf deleted file mode 100644 index 3df29d8a9..000000000 --- a/vm-postfix-config-valid/master.cf +++ /dev/null @@ -1,113 +0,0 @@ -# -# Postfix master process configuration file. For details on the format -# of the file, see the master(5) manual page (command: "man 5 master"). -# -# Do not forget to execute "postfix reload" after editing this file. -# -# ========================================================================== -# service type private unpriv chroot wakeup maxproc command + args -# (yes) (yes) (yes) (never) (100) -# ========================================================================== -smtp inet n - - - - smtpd -#smtp inet n - - - 1 postscreen -#smtpd pass - - - - - smtpd -#dnsblog unix - - - - 0 dnsblog -#tlsproxy unix - - - - 0 tlsproxy -#submission inet n - - - - smtpd -# -o syslog_name=postfix/submission -# -o smtpd_tls_security_level=encrypt -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#smtps inet n - - - - smtpd -# -o syslog_name=postfix/smtps -# -o smtpd_tls_wrappermode=yes -# -o smtpd_sasl_auth_enable=yes -# -o smtpd_client_restrictions=permit_sasl_authenticated,reject -# -o milter_macro_daemon_name=ORIGINATING -#628 inet n - - - - qmqpd -pickup fifo n - - 60 1 pickup -cleanup unix n - - - 0 cleanup -qmgr fifo n - n 300 1 qmgr -#qmgr fifo n - n 300 1 oqmgr -tlsmgr unix - - - 1000? 1 tlsmgr -rewrite unix - - - - - trivial-rewrite -bounce unix - - - - 0 bounce -defer unix - - - - 0 bounce -trace unix - - - - 0 bounce -verify unix - - - - 1 verify -flush unix n - - 1000? 0 flush -proxymap unix - - n - - proxymap -proxywrite unix - - n - 1 proxymap -smtp unix - - - - - smtp -relay unix - - - - - smtp -# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 -showq unix n - - - - showq -error unix - - - - - error -retry unix - - - - - error -discard unix - - - - - discard -local unix - n n - - local -virtual unix - n n - - virtual -lmtp unix - - - - - lmtp -anvil unix - - - - 1 anvil -scache unix - - - - 1 scache -# -# ==================================================================== -# Interfaces to non-Postfix software. Be sure to examine the manual -# pages of the non-Postfix software to find out what options it wants. -# -# Many of the following services use the Postfix pipe(8) delivery -# agent. See the pipe(8) man page for information about ${recipient} -# and other message envelope options. -# ==================================================================== -# -# maildrop. See the Postfix MAILDROP_README file for details. -# Also specify in main.cf: maildrop_destination_recipient_limit=1 -# -maildrop unix - n n - - pipe - flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} -# -# ==================================================================== -# -# Recent Cyrus versions can use the existing "lmtp" master.cf entry. -# -# Specify in cyrus.conf: -# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 -# -# Specify in main.cf one or more of the following: -# mailbox_transport = lmtp:inet:localhost -# virtual_transport = lmtp:inet:localhost -# -# ==================================================================== -# -# Cyrus 2.1.5 (Amos Gouaux) -# Also specify in main.cf: cyrus_destination_recipient_limit=1 -# -#cyrus unix - n n - - pipe -# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} -# -# ==================================================================== -# Old example of delivery via Cyrus. -# -#old-cyrus unix - n n - - pipe -# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} -# -# ==================================================================== -# -# See the Postfix UUCP_README file for configuration details. -# -uucp unix - n n - - pipe - flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) -# -# Other external delivery methods. -# -ifmail unix - n n - - pipe - flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) -bsmtp unix - n n - - pipe - flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient -scalemail-backend unix - n n - 2 pipe - flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} -mailman unix - n n - - pipe - flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py - ${nexthop} ${user} - diff --git a/vm-postfix-config-valid/tls_policy b/vm-postfix-config-valid/tls_policy deleted file mode 100644 index f8d6a4968..000000000 --- a/vm-postfix-config-valid/tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com encrypt protocols=TLSv1.1 From 46ce09d36d0b15113fd0ea1f61b4db842fb8e948 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 05:01:46 -0700 Subject: [PATCH 027/639] MTA config wrangling seems to work --- mta-config-generator.py | 91 ++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 33 deletions(-) mode change 100644 => 100755 mta-config-generator.py diff --git a/mta-config-generator.py b/mta-config-generator.py old mode 100644 new mode 100755 index 5d79423ef..1346d37dd --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -4,7 +4,7 @@ import string DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" -def parse_line(self, line_data): +def parse_line(line_data): "return the and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache @@ -12,7 +12,7 @@ def parse_line(self, line_data): left, sep, right = line.partition("=") if not sep: return None - return (left.strip(), right.strip()) + return (num, left.strip(), right.strip()) #def get_cf_values(lines, var): @@ -23,10 +23,11 @@ class MTAConfigGenerator: class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, stlse_config): + def __init__(self, stlse_config, fixup=False): + self.fixup = fixup MTAConfigGenerator.__init__(self, stlse_config) - this.postfix_cf_file = this.find_postfix_cf() - this.wrangle_existing_config() + self.postfix_cf_file = self.find_postfix_cf() + self.wrangle_existing_config() def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -35,53 +36,77 @@ class PostfixConfigGenerator(MTAConfigGenerator): acceptable = [ideal] + also_acceptable - l = [num,line for num,line in enumerate(cf) if line.startswith(var)] + l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] if not any(l): - this.additions.append("smtpd_use_tls = yes") + self.additions.append(var + " = " + ideal) else: - values = [right for left, right in map(parse_line, l)] + values = map(parse_line, l) if len(set(values)) > 1: - if this.fixup: - this.deletions.append( - raise ExistingConfigError, "Conflicting existing config values " + `l` - if values[0] != "yes": + if self.fixup: + #print "Scheduling deletions:" + `values` + conflicting_lines = [num for num,_var,val in values] + self.deletions.extend(conflicting_lines) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Conflicting existing config values " + `l` + val = values[0][2] + if val not in acceptable: + #print "Scheduling deletions:" + `values` + if self.fixup: + self.deletions.append(values[0][0]) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - def wrangle_existing_config(self, fixup=false): + def wrangle_existing_config(self): """ Try to ensure/mutate that the config file is in a sane state. Fixup means we'll delete existing lines if necessary to get there. """ - this.additions = [] - fn = find_postfix_cf() - raw_cf = open(fn).readlines() - this.cf = map(string.strip, raw_cf) - #this.cf = [line for line in cf if line and not line.startswith("#")] - this.fixup = fixup + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] # Check we're currently accepting inbound STARTTLS sensibly - this.ensure_cf_var("smtpd_use_tls", "yes", []) + self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction - this.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) # Maximum verbosity lets us collect failure information - this.ensure_cf_var("smtp_tls_loglevel", "1", []) + self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - this.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + self.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) - this.maybe_add_config_lines() + self.maybe_add_config_lines() def maybe_add_config_lines(self): - if not this.additions: + if not self.additions: return - this.additions[:0]=["","# New config lines added by STARTTLS Everywhere",""] - new_cf_lines = "\n".join(this.additions) - print "Adding to %s:" % fn + if self.fixup: + print "Deleting lines:", self.deletions + self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + new_cf_lines = "\n".join(self.additions) + print "Adding to %s:" % self.fn print new_cf_lines - if raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - new_cf = "".join(raw_cf) + sep + new_cf_lines - f = open(fn, "w").write(new_cf) - f.close() + if self.raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" + + self.new_cf = "" + for num, line in enumerate(self.raw_cf): + if self.fixup and num in self.deletions: + self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line + else: + self.new_cf += line + self.new_cf += sep + new_cf_lines + + print self.new_cf + f = open(self.fn, "w").write(self.new_cf) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return "/etc/postfix/main.cf" + +if __name__ == "__main__": + pcgen = PostfixConfigGenerator(None, fixup=True) From 0c4e33281184c489378758d70ccd0cec129c6293 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 11 Jun 2014 11:45:28 -0400 Subject: [PATCH 028/639] Set up test CA and valid signed cert by that CA. Also require valid cert for host 'valid'. --- vagrant-shared/postfix-config-sender-tls_policy | 2 +- vagrant-shared/postfix-config-sender.cf | 1 + vagrant-shared/postfix-config-valid-example-recipient.cf | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vagrant-shared/postfix-config-sender-tls_policy b/vagrant-shared/postfix-config-sender-tls_policy index af948c5e7..f4f1e80c5 100644 --- a/vagrant-shared/postfix-config-sender-tls_policy +++ b/vagrant-shared/postfix-config-sender-tls_policy @@ -1 +1 @@ -#valid-example-recipient.com encrypt protocols=TLSv1.1 +valid-example-recipient.com secure match=valid-example-recipient.com:.valid-example-recipient.com diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf index cf74d122f..6fc9435c6 100644 --- a/vagrant-shared/postfix-config-sender.cf +++ b/vagrant-shared/postfix-config-sender.cf @@ -43,3 +43,4 @@ smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy smtp_tls_loglevel = 1 smtp_tls_security_level = may +smtp_tls_CAfile = /etc/certificates/ca.crt diff --git a/vagrant-shared/postfix-config-valid-example-recipient.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf index a5f7ce575..1d08a20cd 100644 --- a/vagrant-shared/postfix-config-valid-example-recipient.cf +++ b/vagrant-shared/postfix-config-valid-example-recipient.cf @@ -18,8 +18,6 @@ append_dot_mydomain = no readme_directory = no # TLS parameters -smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key smtpd_use_tls=yes smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache @@ -44,3 +42,5 @@ smtpd_tls_received_header = yes #STARTTLS EVERYWHERE MAGIC STARTS HERE smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy +smtpd_tls_cert_file=/etc/certificates/valid.crt +smtpd_tls_key_file=/etc/certificates/valid.key From 6e1bcfdb2a7ebe34511c5bc92265ceebe373b47f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:17:50 -0700 Subject: [PATCH 029/639] WIP implementing domain-wise TLS policies --- mta-config-generator.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/mta-config-generator.py b/mta-config-generator.py index 1346d37dd..1b98e7e96 100755 --- a/mta-config-generator.py +++ b/mta-config-generator.py @@ -2,7 +2,9 @@ import string -DEFAULT_POLICY_FILE = "texthash:/etc/postfix/starttls_everywhere_policy" + +DEFAULT_POLICY_FILE = "/etc/postfix/starttls_everywhere_policy" +POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): "return the and right hand sides of stripped, non-comment postfix config line" @@ -17,17 +19,18 @@ def parse_line(line_data): #def get_cf_values(lines, var): class MTAConfigGenerator: - def __init__(self, stlse_config): - self.c = stlse_config + def __init__(self, policy_config): + self.policy_config = policy_config class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, stlse_config, fixup=False): + def __init__(self, policy_config, fixup=False): self.fixup = fixup - MTAConfigGenerator.__init__(self, stlse_config) + MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() + self.set_domainwise_tls_policies() def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -77,7 +80,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - self.ensure_cf_var("smtp_tls_policy_maps", DEFAULT_POLICY_FILE, []) + self.ensure_cf_var("smtp_tls_policy_maps", POLICY_CF_ENTRY, []) self.maybe_add_config_lines() @@ -87,7 +90,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): if self.fixup: print "Deleting lines:", self.deletions self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + new_cf_lines = "\n".join(self.additions) + "\n" print "Adding to %s:" % self.fn print new_cf_lines if self.raw_cf[-1][-1] == "\n": sep = "" @@ -108,5 +111,18 @@ class PostfixConfigGenerator(MTAConfigGenerator): "Search far and wide for the correct postfix configuration file" return "/etc/postfix/main.cf" + def set_domainwise_tls_policies(self): + self.policy_lines = [] + for domain, policy in self.policy_config.tls_policies: + entry = domain + " encrypt " + if "min-tls-version" in policy: + entry += " " + policy["min-tls-version"] + self.policy_lines.append(entry) + + f = open(DEFAULT_POLICY_FILE, "w") + f.write("\n".join(self.policy_lines)) + if __name__ == "__main__": - pcgen = PostfixConfigGenerator(None, fixup=True) + import config-parser + c = config-parser.Config() + pcgen = PostfixConfigGenerator(c, fixup=True) From eea1b0d8c5cf5367dcb91099ac7331443ad8a631 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:18:56 -0700 Subject: [PATCH 030/639] Switch naming conventions so that modules are importable :) It turns out that python won't import modules with hyphens in their names. It seems that CamelCase is most consistent with our Class naming. However feel free to do something different instead :) --- check-starttls.py => CheckSTARTTLS.py | 0 config-parser.py => ConfigParser.py | 0 mta-config-generator.py => MTAConfigGenerator.py | 0 ...-google-starttls-domains.py => ProcessGoogleSTARTTLSDomains.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename check-starttls.py => CheckSTARTTLS.py (100%) rename config-parser.py => ConfigParser.py (100%) rename mta-config-generator.py => MTAConfigGenerator.py (100%) rename process-google-starttls-domains.py => ProcessGoogleSTARTTLSDomains.py (100%) diff --git a/check-starttls.py b/CheckSTARTTLS.py similarity index 100% rename from check-starttls.py rename to CheckSTARTTLS.py diff --git a/config-parser.py b/ConfigParser.py similarity index 100% rename from config-parser.py rename to ConfigParser.py diff --git a/mta-config-generator.py b/MTAConfigGenerator.py similarity index 100% rename from mta-config-generator.py rename to MTAConfigGenerator.py diff --git a/process-google-starttls-domains.py b/ProcessGoogleSTARTTLSDomains.py similarity index 100% rename from process-google-starttls-domains.py rename to ProcessGoogleSTARTTLSDomains.py From 2540f1f1e8f924d50b629a682986d5973715a968 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:31:41 -0700 Subject: [PATCH 031/639] Writing to the domain-wise policy file actually works now. --- MTAConfigGenerator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 1b98e7e96..27a714361 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -113,16 +113,16 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for domain, policy in self.policy_config.tls_policies: - entry = domain + " encrypt " + for domain, policy in self.policy_config.tls_policies.items(): + entry = domain + " encrypt" if "min-tls-version" in policy: entry += " " + policy["min-tls-version"] self.policy_lines.append(entry) f = open(DEFAULT_POLICY_FILE, "w") - f.write("\n".join(self.policy_lines)) + f.write("\n".join(self.policy_lines) + "\n") if __name__ == "__main__": - import config-parser - c = config-parser.Config() + import ConfigParser + c = ConfigParser.Config() pcgen = PostfixConfigGenerator(c, fixup=True) From 182e9b29e4353a64a910a86edd6adb1d25976882 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:42:17 -0700 Subject: [PATCH 032/639] Trying to standardize JSON terms --- ConfigParser.py | 10 +++++----- MTAConfigGenerator.py | 7 +++++-- config.json | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index dbc244b25..aba4eb23f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -47,7 +47,7 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "tls-policies": + elif atr == "security-policies": self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: @@ -67,18 +67,18 @@ class Config: def check_tls_policy_domains(self, val): if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` + raise TypeError, "security-policies should be a dict" + `val` for domain, policies in val.items(): try: assert type(domain) == unicode d = str(domain) # convert from unicode except: - raise TypeError, "tls-policy domain not a string" + `domain` + raise TypeError, "security-policy domain not a string" + `domain` if not d.startswith("*."): - raise ValueError, "tls-policy domains must start with *.; try *."+d + raise ValueError, "security-policy domains must start with *.; try *."+d d = d.partition("*.")[2] if not looks_like_a_domain(d): - raise ValueError, "tls-policy for something that a domain? " + d + raise ValueError, "security-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 27a714361..bee518e05 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -105,7 +105,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += sep + new_cf_lines print self.new_cf - f = open(self.fn, "w").write(self.new_cf) + f = open(self.fn, "w") + f.write(self.new_cf) + f.close() def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -121,8 +123,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): f = open(DEFAULT_POLICY_FILE, "w") f.write("\n".join(self.policy_lines) + "\n") + f.close() if __name__ == "__main__": import ConfigParser - c = ConfigParser.Config() + c = ConfigParser.Config("starttls-everywhere.json") pcgen = PostfixConfigGenerator(c, fixup=True) diff --git a/config.json b/config.json index 1a9034545..8d38696ff 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", - "tls-policies": { + "security-policies": { "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } From 3712a4539907a05a98e289cfc48a6c62adccd686 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:48:43 -0700 Subject: [PATCH 033/639] Further (and different, and better) standardisation --- ConfigParser.py | 10 +++++----- README.md | 2 +- starttls-everywhere.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index aba4eb23f..dbc244b25 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -47,7 +47,7 @@ class Config: self.timestamp = parse_timestamp(val) elif atr == "expires": self.expires = parse_timestamp(val) - elif atr == "security-policies": + elif atr == "tls-policies": self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: @@ -67,18 +67,18 @@ class Config: def check_tls_policy_domains(self, val): if type(val) != dict: - raise TypeError, "security-policies should be a dict" + `val` + raise TypeError, "tls-policies should be a dict" + `val` for domain, policies in val.items(): try: assert type(domain) == unicode d = str(domain) # convert from unicode except: - raise TypeError, "security-policy domain not a string" + `domain` + raise TypeError, "tls-policy domain not a string" + `domain` if not d.startswith("*."): - raise ValueError, "security-policy domains must start with *.; try *."+d + raise ValueError, "tls-policy domains must start with *.; try *."+d d = d.partition("*.")[2] if not looks_like_a_domain(d): - raise ValueError, "security-policy for something that a domain? " + d + raise ValueError, "tls-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/README.md b/README.md index 79e0ed30d..2f0a410a1 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co // "timestamp": 1401414363, : also acceptable "author": "Electronic Frontier Foundation https://eff.org", "expires": "2014-06-06T14:30:16+00:00", - "security-policies": { + "tls-policies": { // These match on the MX domain. "*.yahoodns.net": { "require-valid-certificate": true, diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 4c91866d5..d0e656186 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,5 @@ { - "mx-domains": { + "tls-policies": { "*.mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true @@ -49,7 +49,7 @@ "require-tls": true } }, - "address-domains": { + "acceptable-mxs": { "wp.pl": { "accept-mx-domains": [ "*.wp.pl" From 34cba3accfe2da92846cdf6a427724f60999a96a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 09:51:56 -0700 Subject: [PATCH 034/639] Now successfully parsing the larger policy set --- ConfigParser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfigParser.py b/ConfigParser.py index dbc244b25..8bc7e708f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -55,7 +55,7 @@ class Config: self.tls_policies[domain] = {} # being here enforces TLS at all for policy, value in policies.items(): if policy == "min-tls-version": - reasonable = ["TLS", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) From a2ee328bc0dc3e384f9d1ecf6567aa16afbeb7ef Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 11 Jun 2014 10:32:52 -0700 Subject: [PATCH 035/639] Paramaterise "/etc/postfix" --- MTAConfigGenerator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index bee518e05..3b69ba37a 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,9 +1,10 @@ #!/usr/bin/env python import string +import os.path - -DEFAULT_POLICY_FILE = "/etc/postfix/starttls_everywhere_policy" +POSTFIX_DIR = "/etc/postfix" +DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): @@ -111,7 +112,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return "/etc/postfix/main.cf" + return os.path.join(POSTFIX_DIR,"main.cf") def set_domainwise_tls_policies(self): self.policy_lines = [] From e99abfacfd40c4f7c1f404c7225d5d3ad4792ef8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:22:30 -0700 Subject: [PATCH 036/639] Include the demo example with the real stuff, temporarily --- starttls-everywhere.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d0e656186..4b47ea0c3 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,9 @@ { "tls-policies": { + "*.valid-example-recipient.com": { + "min-tls-version": "TLSv1.1", + "force-tls" : true + }, "*.mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true @@ -50,6 +54,9 @@ } }, "acceptable-mxs": { + "valid-example-recipient.com": { + "accept-mx-domains": [ "*.valid-example-recipient.com" ] + }, "wp.pl": { "accept-mx-domains": [ "*.wp.pl" From 3d7b53daf19c4f47e34e1fb74b57367462968142 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:24:05 -0700 Subject: [PATCH 037/639] Tune verbosity; reload postfix conf if we can --- MTAConfigGenerator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3b69ba37a..d78fa55e5 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,14 +1,16 @@ #!/usr/bin/env python import string -import os.path +import os, os.path POSTFIX_DIR = "/etc/postfix" DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): - "return the and right hand sides of stripped, non-comment postfix config line" + """ + return the (line number, left hand side, right hand side) + of a stripped postfix config line""" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache num,line = line_data @@ -32,6 +34,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -105,7 +108,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += line self.new_cf += sep + new_cf_lines - print self.new_cf + #print self.new_cf f = open(self.fn, "w") f.write(self.new_cf) f.close() From 02abaf57bd0cd67cabb8e1cdaefe1dabbb269776 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 12 Jun 2014 05:24:51 -0700 Subject: [PATCH 038/639] Remove stray conf entry from README --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 2f0a410a1..7a08c3ecf 100644 --- a/README.md +++ b/README.md @@ -86,9 +86,6 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co "eff.org": { "accept-mx-domains": ["*.eff.org"] } - "*.yahoodns.net": { - "require-valid-certificate": true, - } } } From 9ce047980a991129de1bfc6b56a8bd133f798942 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:40:17 -0400 Subject: [PATCH 039/639] Add .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..94c6f7089 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.* +*.orig From 9cd71642fb13688565a2b245eac06089eaad8dcb Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:50:39 -0400 Subject: [PATCH 040/639] Fix italicization boundaries --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2f0a410a1..6717f8b72 100644 --- a/README.md +++ b/README.md @@ -109,19 +109,19 @@ Config-generator should attempt to fetch the configuration file daily and transf The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. -Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains _list must correspond to an exactly matching field in the _mx-domains_ config section. +Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains_ list must correspond to an exactly matching field in the _mx-domains_ config section. The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. **mx-domains** -The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain _can be a subdomain of any other _mx-domain _in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. +The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain_can be a subdomain of any other _mx-domain_in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ -_Require-valid-certificate _defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: +_Require-valid-certificate_defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: 1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. 2. The certificate is unexpired. From 499f6c2fad62fe4d793cfee727576f399012a1b2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 11:53:02 -0400 Subject: [PATCH 041/639] Add comment to ProcessgoogleSTARTTLSDomains.py --- ProcessGoogleSTARTTLSDomains.py | 11 ++++++++++- config.json | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ProcessGoogleSTARTTLSDomains.py b/ProcessGoogleSTARTTLSDomains.py index 0a0201875..abb2b3495 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/ProcessGoogleSTARTTLSDomains.py @@ -1,4 +1,13 @@ #!/usr/bin/python +""" +Process Google's TLS delivery data from +https://www.google.com/transparencyreport/saferemail/data/?hl=en +to look for outbound domains that can negotiate an encrypted +connection >99% of the time. + +Usage: + ./ProcessGoogleSTARTTLSDomains.py google-starttls-domains.csv +""" import csv import codecs import sys @@ -14,5 +23,5 @@ for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in pass for address_suffix, fraction_encrypted in d.iteritems(): - if min(fraction_encrypted) >= 0.50: + if min(fraction_encrypted) >= 0.99: print min(fraction_encrypted), address_suffix diff --git a/config.json b/config.json index 8d38696ff..1a9034545 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "timestamp": 1401093333, "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", - "security-policies": { + "tls-policies": { "*.valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } From 43d457aa771e663a7c15886fbc81db081947d9cf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 12 Jun 2014 13:18:20 -0400 Subject: [PATCH 042/639] Typo cleanup in MTAConfigGenerator --- ConfigParser.py | 1 - MTAConfigGenerator.py | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 8bc7e708f..206132293 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -36,7 +36,6 @@ class Config: f = open(cfg_file_name) self.cfg = json.loads(f.read()) for atr, val in self.cfg.items(): - #print atr,val # Verify each attribute of the structure if atr.startswith("comment"): continue diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3b69ba37a..d5ec334ec 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -3,12 +3,12 @@ import string import os.path -POSTFIX_DIR = "/etc/postfix" +POSTFIX_DIR = "postfix-copy" DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE def parse_line(line_data): - "return the and right hand sides of stripped, non-comment postfix config line" + "return the left and right hand sides of stripped, non-comment postfix config line" # lines are like: # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache num,line = line_data @@ -17,8 +17,6 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) -#def get_cf_values(lines, var): - class MTAConfigGenerator: def __init__(self, policy_config): self.policy_config = policy_config @@ -36,8 +34,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. """ - + values; if not, set it to the ideal value. + """ acceptable = [ideal] + also_acceptable l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] From 3cf61a54b7ce08389a85f5282fd86ca958b121b7 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 13 Jun 2014 13:57:05 -0400 Subject: [PATCH 043/639] Add alternatives section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6717f8b72..6e3d4212b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. +## Alternatives + +Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of a DANE solution easier. + ## Detailed design Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: From 9da4f93ae587156d4f9be5829a32a4201e5f1588 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 16 Jun 2014 02:46:46 -0700 Subject: [PATCH 044/639] Changes for live demo --- Vagrantfile | 1 + starttls-everywhere.json | 1 - vagrant-bootstrap.sh | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 3ce93c917..b7153a7b8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,6 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| valid.vm.hostname = "valid-example-recipient.com" end config.vm.synced_folder "vagrant-shared", "/vagrant" + config.vm.synced_folder "vagrant-shared/starttls-everywhere", "/vagrant/starttls-everywhere" config.vm.provision :shell, path: "vagrant-bootstrap.sh" config.vm.provider "virtualbox" do |vb| diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 4b47ea0c3..df3ff41ab 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,7 +1,6 @@ { "tls-policies": { "*.valid-example-recipient.com": { - "min-tls-version": "TLSv1.1", "force-tls" : true }, "*.mx.aol.com": { diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh index 13593a0cc..82622edea 100755 --- a/vagrant-bootstrap.sh +++ b/vagrant-bootstrap.sh @@ -23,9 +23,9 @@ if [ "`hostname`" = "sender" ]; then (while sleep 10; do echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com done) & - ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy + #ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy fi -ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf -ln -sf "/vagrant/certificates" /etc/certificates +#ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf +#ln -sf "/vagrant/certificates" /etc/certificates postfix reload From 51f90ffafb3e74406e24eb63e7f359ed0483b461 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 16 Jun 2014 18:26:56 +0000 Subject: [PATCH 045/639] Write policies based on address domain, not stripped mx-domain --- ConfigParser.py | 9 ++------- MTAConfigGenerator.py | 13 +++++++++---- starttls-everywhere.json | 5 ----- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 206132293..072c7e8ff 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -59,10 +59,10 @@ class Config: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) elif atr == "acceptable-mxs": + self.acceptable_mxs = val pass else: - sys.stderr.write("Uknown attribute: " + `atr` + "\n") - print self.tls_policies + sys.stderr.write("Unknown attribute: " + `atr` + "\n") def check_tls_policy_domains(self, val): if type(val) != dict: @@ -73,11 +73,6 @@ class Config: d = str(domain) # convert from unicode except: raise TypeError, "tls-policy domain not a string" + `domain` - if not d.startswith("*."): - raise ValueError, "tls-policy domains must start with *.; try *."+d - d = d.partition("*.")[2] - if not looks_like_a_domain(d): - raise ValueError, "tls-policy for something that a domain? " + d yield (d, policies) if __name__ == "__main__": diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index d5ec334ec..859095acc 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -114,10 +114,15 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for domain, policy in self.policy_config.tls_policies.items(): - entry = domain + " encrypt" - if "min-tls-version" in policy: - entry += " " + policy["min-tls-version"] + for address_domain, properties in self.policy_config.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + mx_policy = self.policy_config.tls_policies[mx_domain] + entry = address_domain + " encrypt" + if "min-tls-version" in mx_policy: + entry += " " + mx_policy["min-tls-version"] self.policy_lines.append(entry) f = open(DEFAULT_POLICY_FILE, "w") diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d0e656186..a98a2293f 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -50,11 +50,6 @@ } }, "acceptable-mxs": { - "wp.pl": { - "accept-mx-domains": [ - "*.wp.pl" - ] - }, "yahoo.co.uk": { "accept-mx-domains": [ "*.yahoodns.net" From 2eba47a716956bdcfc99c466b617ecc2e5b9c32a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 16 Jun 2014 14:23:33 -0700 Subject: [PATCH 046/639] Verify more of the policy language --- ConfigParser.py | 28 ++++++++++++++++++++++++---- MTAConfigGenerator.py | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 206132293..541ff3f30 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -35,6 +35,8 @@ class Config: def __init__(self, cfg_file_name = "config.json"): f = open(cfg_file_name) self.cfg = json.loads(f.read()) + self.tls_policies = {} + self.mx_map = {} for atr, val in self.cfg.items(): # Verify each attribute of the structure if atr.startswith("comment"): @@ -47,21 +49,39 @@ class Config: elif atr == "expires": self.expires = parse_timestamp(val) elif atr == "tls-policies": - self.tls_policies = {} for domain,policies in self.check_tls_policy_domains(val): if type(policies) != dict: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, value in policies.items(): - if policy == "min-tls-version": + for policy, v in policies.items(): + value = lower(str(v)) + if policy == "require-tls": + if value in ("true", "1", "yes"): + self.tls_policies[domain]["required"] = True + elif value in ("false", "0", "no"): + self.tls_policies[domain]["required"] = False + else: + raise ValueError, "Unknown require-tls value " + `value` + elif policy == "min-tls-version": reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) + elif policy == "enforce-mode": + if value == "enforce": + self.tls_policies[domain]["enforce"] = True + elif value == "log-only": + self.tls_policies[domain]["enforce"] = False + else: + raise ValueError, "Not a known enoforcement policy " + `value` elif atr == "acceptable-mxs": - pass + for domain, mxball in acceptable_mx: + pass else: sys.stderr.write("Uknown attribute: " + `atr` + "\n") + # XXX is it ever permissible to have a domain with an acceptable-mx + # that does not point to a TLS security policy? If not, check/warn/fail + # here print self.tls_policies def check_tls_policy_domains(self, val): diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 875624e57..1d322c8e9 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -78,7 +78,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Check we're currently accepting inbound STARTTLS sensibly self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt"]) + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map From 67ee3b048876d1e27cdc21d29139afc1bb812b68 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 18 Jun 2014 12:32:17 -0400 Subject: [PATCH 047/639] Config format change - don't use * as it's misleading. --- config.json | 4 +-- starttls-everywhere.json | 58 ++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config.json b/config.json index 1a9034545..05fc237bf 100644 --- a/config.json +++ b/config.json @@ -4,13 +4,13 @@ "author": "Electronic Frontier Foundation https://eff.org", "expires": 1404677353, "comment 2:": "epoch seconds", "tls-policies": { - "*.valid-example-recipient.com": { + ".valid-example-recipient.com": { "min-tls-version": "TLSv1.1" } }, "acceptable-mxs": { "valid-example-recipient.com": { - "accept-mx-domains": [ "*.valid-example-recipient.com" ] + "accept-mx-domains": [ ".valid-example-recipient.com" ] } } diff --git a/starttls-everywhere.json b/starttls-everywhere.json index a98a2293f..d00859d8c 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,50 +1,50 @@ { "tls-policies": { - "*.mx.aol.com": { + ".mx.aol.com": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.psmtp.com": { + ".psmtp.com": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.ukr.net": { + ".ukr.net": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.interia.pl": { + ".interia.pl": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.gmx.net": { + ".gmx.net": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.web.de": { + ".web.de": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.marktplaats.nl": { + ".marktplaats.nl": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.wp.pl": { + ".wp.pl": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.yahoodns.net": { + ".yahoodns.net": { "min-tls-version": "TLSv1", "require-tls": true }, - "*.t-online.de": { + ".t-online.de": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.rambler.ru": { + ".rambler.ru": { "min-tls-version": "TLSv1.1", "require-tls": true }, - "*.t.facebook.com": { + ".t.facebook.com": { "min-tls-version": "TLSv1", "require-tls": true } @@ -52,87 +52,87 @@ "acceptable-mxs": { "yahoo.co.uk": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "rocketmail.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "web.de": { "accept-mx-domains": [ - "*.web.de" + ".web.de" ] }, "sbcglobal.net": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "aol.com": { "accept-mx-domains": [ - "*.mx.aol.com" + ".mx.aol.com" ] }, "facebook.com": { "accept-mx-domains": [ - "*.t.facebook.com" + ".t.facebook.com" ] }, "sompo-japan.co.jp": { "accept-mx-domains": [ - "*.psmtp.com" + ".psmtp.com" ] }, "salesforce.com": { "accept-mx-domains": [ - "*.psmtp.com" + ".psmtp.com" ] }, "rambler.ru": { "accept-mx-domains": [ - "*.rambler.ru" + ".rambler.ru" ] }, "t-online.de": { "accept-mx-domains": [ - "*.t-online.de" + ".t-online.de" ] }, "gmx.net": { "accept-mx-domains": [ - "*.gmx.net" + ".gmx.net" ] }, "gmx.de": { "accept-mx-domains": [ - "*.gmx.net" + ".gmx.net" ] }, "ukr.net": { "accept-mx-domains": [ - "*.ukr.net" + ".ukr.net" ] }, "rogers.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "ymail.com": { "accept-mx-domains": [ - "*.yahoodns.net" + ".yahoodns.net" ] }, "marktplaats.nl": { "accept-mx-domains": [ - "*.marktplaats.nl" + ".marktplaats.nl" ] }, "interia.pl": { "accept-mx-domains": [ - "*.interia.pl" + ".interia.pl" ] } } From 3df343495e2ac81d5ceb501b43240a68cbbca54c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 18 Jun 2014 10:58:53 -0700 Subject: [PATCH 048/639] Various fixups --- ConfigParser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 85b704d93..4fb69b6b2 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -54,7 +54,7 @@ class Config: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all for policy, v in policies.items(): - value = lower(str(v)) + value = str(v).lower() if policy == "require-tls": if value in ("true", "1", "yes"): self.tls_policies[domain]["required"] = True @@ -64,6 +64,7 @@ class Config: raise ValueError, "Unknown require-tls value " + `value` elif policy == "min-tls-version": reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + reasonable = map(string.lower, reasonable) if not value in reasonable: raise ValueError, "Not a valid TLS version string: " + `value` self.tls_policies[domain]["min-tls-version"] = str(value) @@ -76,7 +77,7 @@ class Config: raise ValueError, "Not a known enoforcement policy " + `value` elif atr == "acceptable-mxs": self.acceptable_mxs = val - for domain, mxball in selg.acceptable_mxs: + for domain, mxball in self.acceptable_mxs.items(): pass else: sys.stderr.write("Unknown attribute: " + `atr` + "\n") From 51980e212f7ea46fc584ebc8e7938f337d2e666c Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 18 Jun 2014 17:50:41 -0400 Subject: [PATCH 049/639] First pass at logs analysis --- ConfigParser.py | 19 +++++++++++++++++-- PostfixLogSummary.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 PostfixLogSummary.py diff --git a/ConfigParser.py b/ConfigParser.py index 072c7e8ff..69779548f 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -4,7 +4,7 @@ import sys import json from datetime import datetime import string - +import collections def parse_timestamp(ts): try: @@ -48,7 +48,7 @@ class Config: self.expires = parse_timestamp(val) elif atr == "tls-policies": self.tls_policies = {} - for domain,policies in self.check_tls_policy_domains(val): + for domain, policies in self.check_tls_policy_domains(val): if type(policies) != dict: raise TypeError, domain + "'s policies should be a dict: " + `policies` self.tls_policies[domain] = {} # being here enforces TLS at all @@ -60,10 +60,25 @@ class Config: self.tls_policies[domain]["min-tls-version"] = str(value) elif atr == "acceptable-mxs": self.acceptable_mxs = val + self.mx_domain_to_address_domains = collections.defaultdict(set) + for address_domain, properties in self.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + self.mx_domain_to_address_domains[mx_domain].add(address_domain) pass else: sys.stderr.write("Unknown attribute: " + `atr` + "\n") + def get_address_domains(self, mx_hostname): + for mx_domain, address_domains in self.mx_domain_to_address_domains.items(): + # TODO: write this better + if (mx_hostname.find(mx_domain) > 0 and + mx_hostname.find(mx_domain) == len(mx_hostname) - len(mx_domain)): + return address_domains + return None + def check_tls_policy_domains(self, val): if type(val) != dict: raise TypeError, "tls-policies should be a dict" + `val` diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py new file mode 100755 index 000000000..e13c230aa --- /dev/null +++ b/PostfixLogSummary.py @@ -0,0 +1,33 @@ +#!/usr/bin/python2.7 +import re +import sys +import collections + +import ConfigParser + +def get_counts(input, config): + counts = collections.defaultdict(lambda: collections.defaultdict(int)) + r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + for line in sys.stdin: + result = r.search(line) + if result: + validation = result.group(1) + mx_hostname = result.group(2) + address_domains = config.get_address_domains(mx_hostname) + if address_domains: + for d in address_domains: + counts[d][validation] += 1 + counts[d]["all"] += 1 + return counts + +def print_summary(counts): + for mx_hostname, validations in counts.items(): + for validation, validation_count in validations.items(): + if validation == "all": + continue + print mx_hostname, validation, validation_count / validations["all"] + +if __name__ == "__main__": + config = ConfigParser.Config("starttls-everywhere.json") + counts = get_counts(sys.stdin, config) + print_summary(counts) From 3de8c2a65152a9985b04f11a64c6568c93a70dbf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 17:08:30 -0400 Subject: [PATCH 050/639] Do a better job finding address domain from mx doman. --- ConfigParser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ConfigParser.py b/ConfigParser.py index 69779548f..eff4d3557 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -72,11 +72,11 @@ class Config: sys.stderr.write("Unknown attribute: " + `atr` + "\n") def get_address_domains(self, mx_hostname): - for mx_domain, address_domains in self.mx_domain_to_address_domains.items(): - # TODO: write this better - if (mx_hostname.find(mx_domain) > 0 and - mx_hostname.find(mx_domain) == len(mx_hostname) - len(mx_domain)): - return address_domains + labels = mx_hostname.split(".") + for n in range(1, len(labels)): + parent = "." + ".".join(labels[n:]) + if parent in self.mx_domain_to_address_domains: + return self.mx_domain_to_address_domains[parent] return None def check_tls_policy_domains(self, val): From 1c3c69aaad9477988e48f1a3c5080f1d1b201e2d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 17:08:49 -0400 Subject: [PATCH 051/639] Allow parameters to MTAConfigGenerator. --- MTAConfigGenerator.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 859095acc..15bc017ab 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -1,16 +1,17 @@ #!/usr/bin/env python +import sys import string import os.path -POSTFIX_DIR = "postfix-copy" -DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy") -POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE - def parse_line(line_data): - "return the left and right hand sides of stripped, non-comment postfix config line" - # lines are like: - # smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + Return the left and right hand sides of stripped, non-comment postfix + config line. + + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ num,line = line_data left, sep, right = line.partition("=") if not sep: @@ -24,12 +25,15 @@ class MTAConfigGenerator: class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, fixup=False): + def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + print "Configuration complete. Now run `sudo service postfix reload'." def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -59,7 +63,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.additions.append(var + " = " + ideal) else: raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - + def wrangle_existing_config(self): """ Try to ensure/mutate that the config file is in a sane state. @@ -79,7 +83,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - self.ensure_cf_var("smtp_tls_policy_maps", POLICY_CF_ENTRY, []) + policy_cf_entry = "texthash:" + self.policy_file + + self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.maybe_add_config_lines() @@ -110,7 +116,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return os.path.join(POSTFIX_DIR,"main.cf") + return os.path.join(self.postfix_dir, "main.cf") def set_domainwise_tls_policies(self): self.policy_lines = [] @@ -122,14 +128,18 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.tls_policies[mx_domain] entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: - entry += " " + mx_policy["min-tls-version"] + entry += " protocols=" + mx_policy["min-tls-version"] self.policy_lines.append(entry) - f = open(DEFAULT_POLICY_FILE, "w") + f = open(self.policy_file, "w") f.write("\n".join(self.policy_lines) + "\n") f.close() if __name__ == "__main__": import ConfigParser - c = ConfigParser.Config("starttls-everywhere.json") - pcgen = PostfixConfigGenerator(c, fixup=True) + if len(sys.argv) != 3: + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" + sys.exit(1) + c = ConfigParser.Config(sys.argv[1]) + postfix_dir = sys.argv[2] + pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) From a85fad98c0be1c5494090486c476608c4951032f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:22:08 -0400 Subject: [PATCH 052/639] Restore default config --- vagrant-shared/postfix-config-sender.cf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf index 6fc9435c6..b9f265058 100644 --- a/vagrant-shared/postfix-config-sender.cf +++ b/vagrant-shared/postfix-config-sender.cf @@ -37,10 +37,3 @@ mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + inet_interfaces = all - -#STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy - -smtp_tls_loglevel = 1 -smtp_tls_security_level = may -smtp_tls_CAfile = /etc/certificates/ca.crt From 127d49e837d2fa8713bf1932ccecad11607b58e3 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:22:32 -0400 Subject: [PATCH 053/639] Manually add a couple known-good domains. These were skipped because in the Google data they are represented as, e.g. 'gmail.{..}'. --- golden-domains.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/golden-domains.txt b/golden-domains.txt index 9a0c95948..d340f23e4 100644 --- a/golden-domains.txt +++ b/golden-domains.txt @@ -20,3 +20,5 @@ facebook.com craigslist.org bigpond.com aol.com +gmail.com +yahoo.com From dd4f9d35ae8fd21a8593cd232177f6550b2a63e8 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 6 Aug 2014 18:23:13 -0400 Subject: [PATCH 054/639] Improve checker and starttls-everywhere.json. Now we alphabetize keys on output for more useful diffs. --- CheckSTARTTLS.py | 23 ++-- starttls-everywhere.json | 225 ++++++++++++++++++--------------------- 2 files changed, 122 insertions(+), 126 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index cd8dfea42..205a4a779 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -10,6 +10,9 @@ import json import dns.resolver from M2Crypto import X509 +from publicsuffix import PublicSuffixList + +public_suffix_list = PublicSuffixList() def mkdirp(path): try: @@ -71,6 +74,8 @@ def valid_cert(filename): if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: return False try: + # The file contains both the leaf cert and any intermediates, so we pass it + # as both the cert to validate and as the "untrusted" chain. output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ -untrusted "%s" \ "%s" @@ -87,10 +92,11 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) + new_names = map(lambda n: public_suffix_list.get_public_suffix(n), new_names) names.update(new_names) - names.add(filename.rstrip(".")) if len(names) >= 1: - return common_suffix(names) + # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. + return names.pop() else: return "" @@ -134,6 +140,7 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): + print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: @@ -143,7 +150,7 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" if len(sys.argv) == 1: - print("Please pass at least one mail domain as an argument") + print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = { "address-domains": { @@ -151,14 +158,16 @@ if __name__ == '__main__': "mx-domains": { } } - for domain in sys.argv[1:]: - collect(domain) + for domain in open(sys.argv[1]).readlines(): + domain = domain.strip() + if not os.path.exists(domain): + collect(domain) if len(os.listdir(domain)) == 0: continue suffix = check_certs(domain) min_version = min_tls_version(domain) if suffix != "": - suffix_match = "*." + suffix + suffix_match = "." + suffix config["address-domains"][domain] = { "accept-mx-domains": [suffix_match] } @@ -167,4 +176,4 @@ if __name__ == '__main__': "min-tls-version": min_version } - print json.dumps(config, indent=2) + print json.dumps(config, indent=2, sort_keys=True) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index d00859d8c..5dd487f9a 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,128 +1,18 @@ { - "tls-policies": { - ".mx.aol.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".psmtp.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".ukr.net": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".interia.pl": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".gmx.net": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".web.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".marktplaats.nl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".wp.pl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yahoodns.net": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".t-online.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".rambler.ru": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".t.facebook.com": { - "min-tls-version": "TLSv1", - "require-tls": true - } - }, - "acceptable-mxs": { - "yahoo.co.uk": { + "address-domains": { + "craigslist.org": { "accept-mx-domains": [ - ".yahoodns.net" + ".craigslist.org" ] }, - "rocketmail.com": { + "gmail.com": { "accept-mx-domains": [ - ".yahoodns.net" + ".google.com" ] }, - "web.de": { + "interia.pl": { "accept-mx-domains": [ - ".web.de" - ] - }, - "sbcglobal.net": { - "accept-mx-domains": [ - ".yahoodns.net" - ] - }, - "aol.com": { - "accept-mx-domains": [ - ".mx.aol.com" - ] - }, - "facebook.com": { - "accept-mx-domains": [ - ".t.facebook.com" - ] - }, - "sompo-japan.co.jp": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "salesforce.com": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "rambler.ru": { - "accept-mx-domains": [ - ".rambler.ru" - ] - }, - "t-online.de": { - "accept-mx-domains": [ - ".t-online.de" - ] - }, - "gmx.net": { - "accept-mx-domains": [ - ".gmx.net" - ] - }, - "gmx.de": { - "accept-mx-domains": [ - ".gmx.net" - ] - }, - "ukr.net": { - "accept-mx-domains": [ - ".ukr.net" - ] - }, - "rogers.com": { - "accept-mx-domains": [ - ".yahoodns.net" - ] - }, - "ymail.com": { - "accept-mx-domains": [ - ".yahoodns.net" + ".interia.pl" ] }, "marktplaats.nl": { @@ -130,10 +20,107 @@ ".marktplaats.nl" ] }, - "interia.pl": { + "rambler.ru": { "accept-mx-domains": [ - ".interia.pl" + ".rambler.ru" ] + }, + "rocketmail.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "rogers.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "salesforce.com": { + "accept-mx-domains": [ + ".psmtp.com" + ] + }, + "sbcglobal.net": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "sompo-japan.co.jp": { + "accept-mx-domains": [ + ".psmtp.com" + ] + }, + "t-online.de": { + "accept-mx-domains": [ + ".t-online.de" + ] + }, + "wp.pl": { + "accept-mx-domains": [ + ".wp.pl" + ] + }, + "yahoo.co.uk": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "yahoo.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + }, + "yandex.ru": { + "accept-mx-domains": [ + ".yandex.ru" + ] + }, + "ymail.com": { + "accept-mx-domains": [ + ".yahoo.com" + ] + } + }, + "mx-domains": { + ".craigslist.org": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".google.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".interia.pl": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".marktplaats.nl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".psmtp.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".rambler.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".t-online.de": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".wp.pl": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".yahoo.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".yandex.ru": { + "min-tls-version": "TLSv1.1", + "require-tls": true } } } From 6a40e1964b98ba19709055600278807718bac899 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 15:05:10 -0400 Subject: [PATCH 055/639] Notice if there are no "Trusted" entries. --- PostfixLogSummary.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index e13c230aa..daec93db0 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -6,18 +6,24 @@ import collections import ConfigParser def get_counts(input, config): + seen_trusted = False + counts = collections.defaultdict(lambda: collections.defaultdict(int)) r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) if result: validation = result.group(1) - mx_hostname = result.group(2) + mx_hostname = result.group(2).lower() + if validation == "Trusted" or validation == "Verified": + seen_trusted = True address_domains = config.get_address_domains(mx_hostname) if address_domains: for d in address_domains: counts[d][validation] += 1 counts[d]["all"] += 1 + if not seen_trusted: + print "Didn't see any trusted connections. Need to install some certs?" return counts def print_summary(counts): @@ -25,7 +31,7 @@ def print_summary(counts): for validation, validation_count in validations.items(): if validation == "all": continue - print mx_hostname, validation, validation_count / validations["all"] + print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": config = ConfigParser.Config("starttls-everywhere.json") From bdd4d01dc73f6c91af07531304584348dbf0d22d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 16:57:51 -0400 Subject: [PATCH 056/639] Update and sort golden-domains.txt, commit google-starttls-domains.csv --- golden-domains.txt | 50 +- google-starttls-domains.csv | 6734 +++++++++++++++++++++++++++++++++++ 2 files changed, 6763 insertions(+), 21 deletions(-) create mode 100644 google-starttls-domains.csv diff --git a/golden-domains.txt b/golden-domains.txt index d340f23e4..bca6d2f89 100644 --- a/golden-domains.txt +++ b/golden-domains.txt @@ -1,24 +1,32 @@ -ymail.com -yandex.ru -yahoo.co.uk -wp.pl -web.de -vtext.com -ukr.net -t-online.de -sompo-japan.co.jp -sbcglobal.net -salesforce.com -rogers.com -rocketmail.com -rambler.ru -marktplaats.nl -interia.pl -gmx.net -gmx.de -facebook.com -craigslist.org -bigpond.com +163.com aol.com +bigpond.com +comcast.net +craigslist.org +facebook.com gmail.com +gmx.de +hotmail.com +icloud.com +live.com +mac.com +me.com +msn.com +naver.com +outlook.com +qq.com +rocketmail.com +rogers.com +salesforce.com +sbcglobal.net +shaw.ca +sympatico.ca +t-online.de +ukr.net +vtext.com +web.de +wp.pl yahoo.com +yahoogroups.com +yandex.ru +ymail.com diff --git a/google-starttls-domains.csv b/google-starttls-domains.csv new file mode 100644 index 000000000..5d6a00f17 --- /dev/null +++ b/google-starttls-domains.csv @@ -0,0 +1,6734 @@ +Address Suffix,Hostname Suffix,Direction,UN M.49 Region Code,Region Name,Fraction Encrypted +0101.co.jp,0101.co.jp,inbound,001,World,0 +04auto.biz,01auto.biz,inbound,001,World,0 +0bz.biz,hmts.jp,inbound,001,World,0 +1-day.co.nz,1-day.co.nz,inbound,001,World,0 +104.com.tw,104.com.tw,inbound,001,World,0.091112 +1105info.com,1105info.com,inbound,001,World,0 +1111.com.tw,1111.com.tw,inbound,001,World,0 +123.com.tw,123.com.tw,inbound,001,World,0 +12manage.com,netarrest.com,inbound,001,World,0.99998 +160by2.us,160by2.us,inbound,001,World,0 +160by2inbox.com,160by2inbox.com,inbound,001,World,0 +160by2invite.com,160by2invite.com,inbound,001,World,0 +160by2mail.com,160by2mail.com,inbound,001,World,0 +163.com,163.com,inbound,001,World,0.711306 +163.com,netease.com,outbound,001,World,1 +17life.com.tw,17life.com.tw,inbound,001,World,0 +1800flowersinc.com,1800flowersinc.com,inbound,001,World,0 +1800petmeds.com,1800petmeds.com,inbound,001,World,0.00599 +1lejend.com,asumeru.com,inbound,001,World,0 +1lejend.com,asumeru001.com,inbound,001,World,0 +1sale.com,1sale.com,inbound,001,World,0 +1v1y.com,euromsg.net,inbound,001,World,0 +2touchbase.com,infimail.com,inbound,001,World,0 +33go.com.tw,33go.com.tw,inbound,001,World,0 +3suisses.be,3suisses.be,inbound,001,World,0 +3suisses.fr,3suisses.fr,inbound,001,World,0 +4shared.com,4shared.com,inbound,001,World,0.999946 +4wheelparts.com,4wheelparts.com,inbound,001,World,0 +518.com.tw,518.com.tw,inbound,001,World,0 +6pm.com,6pm.com,inbound,001,World,1 +6pm.com,zappos.com,inbound,001,World,0.782581 +7net.com.tw,7net.com.tw,inbound,001,World,0 +99acres.com,99acres.com,inbound,001,World,0 +9dot9digital.in,emce2.in,inbound,001,World,0 +a8.net,a8.net,inbound,001,World,0 +aaa.com,nextjump.com,inbound,001,World,0 +aaas-science.org,aaas-science.org,inbound,001,World,0 +aafes.com,aafes.com,inbound,001,World,0 +aanotifier.nl,aanotifier.nl,inbound,001,World,0 +aarp.org,aarp.org,inbound,001,World,0.006249 +ab0.jp,altovision.co.jp,inbound,001,World,0 +abercrombie-email.com,abercrombie-email.com,inbound,001,World,0 +abercrombiekids-email.com,abercrombie-email.com,inbound,001,World,0 +about.com,about.com,inbound,001,World,2.4e-05 +about.com,sailthru.com,inbound,001,World,0 +academy-enews.com,academy-enews.com,inbound,001,World,0 +accenture.com,outlook.com,inbound,001,World,1 +accountonline.com,accountonline.com,inbound,001,World,0.348991 +acehelpfulemails.com,teradatadmc.com,inbound,001,World,0 +acemserv.com,acemserv.com,inbound,001,World,0 +activesafelist.com,zoothost.com,inbound,001,World,0 +activetrail.com,atmailsvr.net,inbound,001,World,0 +activetrail.com,mymarketing.co.il,inbound,001,World,0 +actorsaccess.com,nonfatmedia.com,inbound,001,World,0 +adchiever.com,kinder-rash-marketing.com,inbound,001,World,0 +adidas.com,neolane.net,inbound,001,World,0 +adidasusnews.com,adidasusnews.com,inbound,001,World,0 +adityabirla.com,adityabirla.com,inbound,001,World,0.006468 +adjockeys.com,thomas-j-brown.com,inbound,001,World,0 +admail.hu,sanomaonline.hu,inbound,001,World,0 +admastersafelist.com,zoothost.com,inbound,001,World,0 +adminforfree.com,adminforfree.com,inbound,001,World,1 +adminforfree.net,adminforfree.com,inbound,001,World,1 +administrativejobinsider.com,administrativejobinsider.com,inbound,001,World,0 +adobe.com,obsmtp.com,inbound,001,World,0.999986 +adobesystems.com,adobesystems.com,inbound,001,World,0 +adorama.com,adorama.com,inbound,001,World,0 +adoreme.com,exacttarget.com,inbound,001,World,0 +adp.com,adp.com,inbound,001,World,1 +adpirate.net,thomas-j-brown.com,inbound,001,World,0 +adsender.us,adsender.us,inbound,001,World,0 +adsolutionline.com,adsolutionline.com,inbound,001,World,0 +adtpulse.com,adtpulse.com,inbound,001,World,0 +adultfriendfinder.com,friendfinder.com,inbound,001,World,0 +advanceauto.com,bigfootinteractive.com,inbound,001,World,0 +advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,001,World,0 +adverts.ie,adverts.ie,inbound,001,World,1 +advfn.com,advfn.com,inbound,001,World,0.635382 +ae.com,ae.com,inbound,001,World,0 +aexp.com,aexp.com,inbound,001,World,1 +af.mil,af.mil,inbound,001,World,0.996166 +affairalert.com,iverificationsystems.com,inbound,001,World,0 +agnitas.de,agnitas.de,inbound,001,World,0.999277 +agoda-emails.com,agoda-emails.com,inbound,001,World,0 +agora.co.il,1host.co.il,inbound,001,World,0 +agorafinancial.com,agorafinancial.com,inbound,001,World,0 +agrupemonos.cl,agrupemonos.cl,inbound,001,World,1 +airbnb.com,airbnb.com,inbound,001,World,0.867975 +airbrake.io,mailgun.net,inbound,001,World,1 +airfarewatchdog.com,smartertravelmedia.com,inbound,001,World,0.011961 +airliquide.com,airliquide.com,inbound,001,World,1 +airmiles.ca,bigfootinteractive.com,inbound,001,World,0 +airtel.com,airtel.in,inbound,001,World,0.064377 +akcijatau.lt,akcijatau.lt,inbound,001,World,0 +alarm.com,alarm.com,inbound,001,World,0 +alarmnet.com,alarmnet.com,inbound,001,World,0 +alaskaair.com,alaskaair.com,inbound,001,World,0 +albertsonsemail.com,email4-mywebgrocer.com,inbound,001,World,0 +alerteimmo.com,alerteimmo.com,inbound,001,World,0 +alertid.com,alertid.com,inbound,001,World,0 +alertsindia.in,alertsindia.in,inbound,001,World,1 +alibaba.com,alibaba.com,inbound,001,World,0 +alice.it,alice.it,inbound,001,World,0 +alice.it,aliceposta.it,outbound,001,World,0 +aliexpress.com,alibaba.com,inbound,001,World,0 +alinea.fr,bp06.net,inbound,001,World,0 +alipay.com,alipay.com,inbound,001,World,1 +allegro.pl,allegro.pl,inbound,001,World,0 +allegrogroup.ua,allegrogroup.ua,inbound,001,World,0 +allegroup.hu,allegroup.hu,inbound,001,World,0 +allheart.com,allheart.com,inbound,001,World,0 +alljob.co.il,alljob.co.il,inbound,001,World,0 +allmodern.com,allmodern.com,inbound,001,World,0 +allout.org,allout.org,inbound,001,World,1 +allrecipes.com,allrecipes.com,inbound,001,World,0 +allsaints.com,allsaints.com,inbound,001,World,0 +allstate.com,rsys1.com,inbound,001,World,0 +alm.com,sailthru.com,inbound,001,World,0 +alumniclass.com,alumniclass.com,inbound,001,World,0 +alumniconnections.com,alumniconnections.com,inbound,001,World,0 +alza.cz,alza.cz,inbound,001,World,0.039449 +alza.sk,alza.cz,inbound,001,World,0.027725 +ama-assn.org,elabs10.com,inbound,001,World,0 +amadeus.com,amadeus.net,inbound,001,World,0 +amazon.{...},amazon.{...},inbound,001,World,0.020886 +amazon.{...},amazonses.com,inbound,001,World,0.999971 +amazon.{...},postini.com,inbound,001,World,0.7325 +amazon.{...},yahoo.{...},inbound,001,World,0.995995 +amazonses.com,amazonses.com,inbound,001,World,0.997919 +amazonses.com,postini.com,inbound,001,World,0.843221 +amctheatres.com,amctheatres.com,inbound,001,World,0 +americanas.com,americanas.com,inbound,001,World,0 +americanbar.org,abanet.org,inbound,001,World,0.00574 +americanexpress.com,americanexpress.com,inbound,001,World,0.000688 +americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,001,World,0 +amubm.com,amubm.com,inbound,001,World,1 +amwayemail.com,mailrouter.net,inbound,001,World,1 +ana.co.jp,ana.co.jp,inbound,001,World,0.534416 +ancestry.com,ancestry.com,inbound,001,World,0 +andrewchristian.com,emv8.com,inbound,001,World,0 +angelbroking.in,infimail.com,inbound,001,World,0 +anghami.com,mailgun.net,inbound,001,World,1 +angieslist.com,angieslist.com,inbound,001,World,0.014064 +anntaylor.com,anntaylor.com,inbound,001,World,0 +anpasia.com,anpasia.com,inbound,001,World,0 +anpdm.com,anpdm.com,inbound,001,World,6e-06 +anthropologie.com,freepeople.com,inbound,001,World,0 +aol.com,aol.com,inbound,001,World,0.999529 +aol.com,aol.com,outbound,001,World,0.999992 +aol.com,sailthru.com,inbound,001,World,0 +aol.net,aol.com,inbound,001,World,1 +apache.org,apache.org,inbound,001,World,0 +apnacomplex.com,apnacomplex.com,inbound,001,World,0.999981 +apple.com,apple.com,inbound,001,World,0.974422 +apply-4-jobs.com,apply-4-jobs.com,inbound,001,World,0 +aprovaconcursos.com.br,eadunicid.com.br,inbound,001,World,0 +aptmail.in,mailurja.com,inbound,001,World,0 +ara.cat,ara.cat,inbound,001,World,0.997709 +arcamax.com,arcamax.com,inbound,001,World,1.4e-05 +argos.co.uk,argos.co.uk,inbound,001,World,1 +argos.co.uk,exacttarget.com,inbound,001,World,1 +aritzia.com,aritzia.com,inbound,001,World,0 +armaniexchange.com,bronto.com,inbound,001,World,0 +artists-hub.com,artists-hub.com,inbound,001,World,0 +artscow.com,dyxnet.com,inbound,001,World,0.00062 +aruba.it,aruba.it,inbound,001,World,0.055666 +asadventure.com,asadventure.com,inbound,001,World,0 +asana.com,asana.com,inbound,001,World,1 +asda.com,ec-cluster.com,inbound,001,World,0 +ashampoo.com,ashampoo.com,inbound,001,World,1 +ashleymadison.com,ashleymadison.com,inbound,001,World,1 +ask.fm,ask.fm,inbound,001,World,1e-05 +askmen.com,askmen.com,inbound,001,World,0 +asos.com,asos.com,inbound,001,World,0 +assembla.com,assembla.com,inbound,001,World,1 +astrocenter.com,center.com,inbound,001,World,0 +astrology.com,astrology.com,inbound,001,World,0 +astrology.com,hsnlmailsvc.com,inbound,001,World,0 +astrology.com,webstakes.com,inbound,001,World,0 +astrology.com,wsafmailsvc.com,inbound,001,World,0 +asus.com,asus.com,inbound,001,World,0 +aswatson.com,emarsys.net,inbound,001,World,0 +athleta.com,athleta.com,inbound,001,World,0 +atlassian.net,uc-inf.net,inbound,001,World,1 +atrapalo.cl,atrapalo.com,inbound,001,World,0 +atrapalo.com,atrapalo.com,inbound,001,World,0 +att-mail.com,att-mail.com,inbound,001,World,2.2e-05 +att-mail.com,att.com,inbound,001,World,0.999966 +att.net,att.net,outbound,001,World,0.204629 +att.net,mycingular.net,inbound,001,World,0.00021 +att.net,yahoo.{...},inbound,001,World,0.99997 +auctionzip-email.com,email-auctionholdings.com,inbound,001,World,0 +auinmeio.com.br,fnac.com.br,inbound,001,World,0 +australiagsm.net,australiagsm.net,inbound,001,World,0 +authorize.net,authorize.net,inbound,001,World,0 +authorize.net,visa.com,inbound,001,World,0.993558 +autoloop.us,loop28.com,inbound,001,World,0 +autoreply.com,autoreply.com,inbound,001,World,0 +avaaz.org,avaaz.org,inbound,001,World,0 +avalanchesafelist.com,zoothost.com,inbound,001,World,0 +avast.com,avast.com,inbound,001,World,0.007867 +aveda.com,esteelauder.com,inbound,001,World,0 +avenue.com,avenue.com,inbound,001,World,0 +avg.com,avg.com,inbound,001,World,0.015703 +avira.com,avira.com,inbound,001,World,0.042528 +avito.ru,avito.ru,inbound,001,World,0.002677 +avomail.com,avomail.com,inbound,001,World,0 +avon.com,email-avonglobal.com,inbound,001,World,0 +avon.com,postdirect.com,inbound,001,World,0 +aweber.com,aweber.com,inbound,001,World,3e-06 +ayi.com,ayi.com,inbound,001,World,0 +b2b-mail.net,b2b-mail.net,inbound,001,World,0 +b2b-mail.net,contact-list.net,inbound,001,World,0 +babycenter.com,rsys3.com,inbound,001,World,0 +babyoye.com,babyoye.com,inbound,001,World,0.011564 +backcountry.com,backcountry.com,inbound,001,World,0 +backlog.jp,backlog.jp,inbound,001,World,0 +badoo.com,monopost.com,inbound,001,World,1 +bagitgetitmailer.in,emce2.in,inbound,001,World,0 +baligam.co.il,baligam.co.il,inbound,001,World,1 +balsamik.fr,balsamik.fr,inbound,001,World,0 +banamex.com,citi.com,inbound,001,World,0.999958 +banamex.com,ibrands.es,inbound,001,World,0 +bananarepublic.com,bananarepublic.com,inbound,001,World,3.48869418874254e-07 +bancoahorrofamsa.com,avantel.net.mx,inbound,001,World,1 +bancochile.cl,bancochile.cl,inbound,001,World,0.999504 +bancofalabella.com,bancofalabella.com,inbound,001,World,0 +bancomer.com,postini.com,inbound,001,World,5.9e-05 +bancomercorreo.com,bancomercorreo.com,inbound,001,World,0 +bandsintown.com,bandsintown.com,inbound,001,World,1 +banesco.com,banesco.com,inbound,001,World,0 +bankofamerica.com,bankofamerica.com,inbound,001,World,0.97133 +banorte.com,gfnorte.com.mx,inbound,001,World,0.994999 +barclaycard.co.uk,barclays.co.uk,inbound,001,World,0 +barclaycardus.com,bigfootinteractive.com,inbound,001,World,0 +barenecessities.com,barenecessities.com,inbound,001,World,0 +barleyment.ca,barleyment.ca,inbound,001,World,0 +barneys.com,barneys.com,inbound,001,World,0 +baseballsavings.com,baseballsavings.com,inbound,001,World,0 +basecamp.com,basecamp.com,inbound,001,World,1 +basecamphq.com,basecamphq.com,inbound,001,World,1 +baskinrobbins.com,baskinrobbins.com,inbound,001,World,0 +basspronews.com,basspronews.com,inbound,001,World,0 +bathandbodyworks.com,bathandbodyworks.com,inbound,001,World,0 +baublebar.com,baublebar.com,inbound,001,World,0.011219 +baycrews.co.jp,webcas.net,inbound,001,World,0 +bayt.com,bayt.com,inbound,001,World,2e-06 +bazarchic-invitations.com,bazarchic-emstech.com,inbound,001,World,0 +bbvacompass.com,postini.com,inbound,001,World,0.996402 +bcbg.com,bcbg.com,inbound,001,World,0 +bci.cl,bci.cl,inbound,001,World,0.999963 +bcp.com.pe,bcp.com.pe,inbound,001,World,1 +be2.com,nmp1.net,inbound,001,World,0 +beamtele.com,beamtele.com,inbound,001,World,0 +beanfun.com,beanfun.com,inbound,001,World,1 +beatport-email.com,beatport-email.com,inbound,001,World,0 +beautylish.com,beautylish.com,inbound,001,World,1 +bebe.com,ed10.com,inbound,001,World,0 +befrugal.com,befrugal.com,inbound,001,World,0.050462 +belkemail.com,belkemail.com,inbound,001,World,0 +bellsouth.net,att.net,outbound,001,World,0 +bellsouth.net,yahoo.{...},inbound,001,World,0.999967 +belluna.net,belluna.net,inbound,001,World,0 +benihana-news.com,benihana-news.com,inbound,001,World,0 +bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,001,World,0 +bespokeoffers.co.uk,chtah.net,inbound,001,World,0 +bestbuy.ca,bestbuy.ca,inbound,001,World,0 +bestbuy.com,bestbuy.com,inbound,001,World,0.003289 +bestdealsforyou.in,elabs5.com,inbound,001,World,0 +beta.lt,mailersend3.com,inbound,001,World,0 +betrend.com,betrend.com,inbound,001,World,0 +bevmo.com,bevmo.com,inbound,001,World,0.000967 +beyondtherack.com,beyondtherack.com,inbound,001,World,0 +bharatmatrimony.com,bharatmatrimony.com,inbound,001,World,1 +bhcosmetics.com,bronto.com,inbound,001,World,0 +bhg.com,meredith.com,inbound,001,World,0 +bigfishgames.com,bigfishgames.com,inbound,001,World,0 +biglion.ru,biglion.ru,inbound,001,World,0.999775 +biglist.com,biglist.com,inbound,001,World,0 +biglots.com,biglots.com,inbound,001,World,0.00029 +bigmailsender.com,bigmailsender.com,inbound,001,World,0 +bigpond.com,bigpond.com,inbound,001,World,0 +bigpond.com,bigpond.com,outbound,001,World,1 +bigtent.com,carezen.net,inbound,001,World,0 +bioagri.com.br,postini.com,inbound,001,World,0.991765 +biomedcentral.com,emv5.com,inbound,001,World,0 +bionexo.com,bionexo.com.br,inbound,001,World,0.999594 +birthdayalarm.com,monkeyinferno.net,inbound,001,World,0 +bitbucket.org,bitbucket.org,inbound,001,World,0 +bitlysupport.com,mailgun.info,inbound,001,World,1 +bitlysupport.com,mailgun.us,inbound,001,World,1 +bitslane.email,bitslane.email,inbound,001,World,0 +bitstatement.org,bitstatement.org,inbound,001,World,1 +bizjournals.com,bizjournals.com,inbound,001,World,0 +bizmailtoday.com,bizmailtoday.com,inbound,001,World,0 +bjs.com,bjs.com,inbound,001,World,0 +bjsrestaurants.com,bjsrestaurants.com,inbound,001,World,0 +bk.ru,mail.ru,inbound,001,World,0.992498 +blablacar.com,blablacar.com,inbound,001,World,1 +blackberry.com,blackberry.com,inbound,001,World,0 +blackboard.com,blackboard.com,inbound,001,World,0.998206 +blackboard.com,notification.com,inbound,001,World,0 +blackpeoplemeet.com,blackpeoplemeet.com,inbound,001,World,0 +blayn.jp,bserver.jp,inbound,001,World,0 +blinkboxmusic.com,mediagraft.com,inbound,001,World,1 +blissworld.com,lstrk.net,inbound,001,World,1 +blizzard.com,battle.net,inbound,001,World,0.11977 +bloglovin.com,bloglovin.com,inbound,001,World,0.000154 +blogtrottr.com,blogtrottr.com,inbound,001,World,0 +bloomberg.com,bloomberg.com,inbound,001,World,0.00501 +bloomberg.net,bloomberg.net,inbound,001,World,1 +bloomingdales.com,bloomingdales.com,inbound,001,World,0 +bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,001,World,0 +blue-compass.com,blue-compass.com,inbound,001,World,0 +bluediamondhost3.com,web-hosting.com,inbound,001,World,1 +bluehornet.com,bluehornet.com,inbound,001,World,0 +bluehost.com,bluehost.com,inbound,001,World,0.000943 +bluehost.com,hostmonster.com,inbound,001,World,0 +bluehost.com,unifiedlayer.com,inbound,001,World,1.6e-05 +bluenile.com,bluenile.com,inbound,001,World,0 +blueshellgames.com,blueshellgames.com,inbound,001,World,0 +bluestatedigital.com,bluestatedigital.com,inbound,001,World,0 +bluestonemx.com,bluestonemx.com,inbound,001,World,1 +bm05.net,bm05.net,inbound,001,World,0 +bm23.com,bronto.com,inbound,001,World,0 +bm324.com,bronto.com,inbound,001,World,0 +bmdeda99.com,bmdeda99.com,inbound,001,World,0 +bme.jp,bserver.jp,inbound,001,World,0 +bmnt.jp,bmnt.jp,inbound,001,World,0 +bmsend.com,bmsend.com,inbound,001,World,0 +bn.com,bn.com,inbound,001,World,0 +bncollegemail.com,bncollegemail.com,inbound,001,World,0 +bnetmail.com,bnetmail.com,inbound,001,World,0 +bol.com.br,bol.com.br,inbound,001,World,0 +bol.com.br,bol.com.br,outbound,001,World,0 +boletinrenuevo.com,boletinrenuevo.com,inbound,001,World,0 +bolsfr.fr,colt.net,inbound,001,World,0 +bomnegocio.com,bomnegocio.com,inbound,001,World,0.687113 +bonobos.com,bronto.com,inbound,001,World,0 +bonuszbrigad.hu,bonuszbrigad.hu,inbound,001,World,0 +boohooemail.com,smartfocusdigital.net,inbound,001,World,0 +bookbub.com,bookbub.com,inbound,001,World,1 +booking.com,booking.com,inbound,001,World,1 +bookingbuddy.com,smartertravelmedia.com,inbound,001,World,0.000486 +bookmyshow.com,eccluster.com,inbound,001,World,0 +bookoffonline.co.jp,bookoffonline.co.jp,inbound,001,World,0 +boomtownroi.com,boomtownroi.com,inbound,001,World,0 +boots.com,boots.com,inbound,001,World,0 +boscovs.com,boscovs.com,inbound,001,World,0 +bostonproper.com,bostonproper.com,inbound,001,World,0 +bouncemanager.it,musvc.com,inbound,001,World,0.362026 +boutiquesecret.com,chtah.net,inbound,001,World,0 +box.com,box.com,inbound,001,World,0.955607 +br.com,cmailsys.com,inbound,001,World,0 +bradfordexchange.com,bradfordexchange.com,inbound,001,World,0 +bradsdeals.com,bradsdeals.com,inbound,001,World,0 +brandalley.com,brandalley.com,inbound,001,World,0 +brands4friends.de,emv5.com,inbound,001,World,0 +brands4friends.jp,webcas.net,inbound,001,World,0 +brandsfever.com,mailgun.net,inbound,001,World,1 +brandsvillage.net,brandsvillage.net,inbound,001,World,0 +brassring.com,brassring.com,inbound,001,World,0.999989 +briantracyintl.com,briantracyintl.com,inbound,001,World,0 +brierleycrm.com,brierleycrm.com,inbound,001,World,0 +brijj.com,brijj.com,inbound,001,World,0 +brincltd.com,brincltd.com,inbound,001,World,0 +bronto.com,bronto.com,inbound,001,World,0 +brooksbrothers.com,brooksbrothers.com,inbound,001,World,0 +bsf01.com,bsftransmit33.com,inbound,001,World,0 +bt.com,bt.com,inbound,001,World,0.409404 +btinternet.com,cpcloud.co.uk,inbound,001,World,0 +btinternet.com,cpcloud.co.uk,outbound,001,World,0 +btinternet.com,yahoo.{...},inbound,001,World,0.99998 +budgettravel.com,email-budgettravel.com,inbound,001,World,0 +buffalo.edu,buffalo.edu,inbound,001,World,0.001059 +bumeran.com,bumeran.com,inbound,001,World,0 +burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,001,World,0 +burton.co.uk,burton.co.uk,inbound,001,World,0 +buscojobs.com,amazonaws.com,inbound,001,World,0 +buy123.com.tw,buy123.com.tw,inbound,001,World,1 +buyinvite.com.au,buyinvite.com.au,inbound,001,World,0 +buyma.com,buyma.com,inbound,001,World,0 +bv.com.br,bv.com.br,inbound,001,World,0 +bweeble.com,adlabsinc.com,inbound,001,World,0 +byway.it,byway.it,inbound,001,World,0 +bzm.mobi,nmsrv.com,inbound,001,World,1 +c21stores.com,c21stores.com,inbound,001,World,0 +ca.gov,ca.gov,inbound,001,World,0.694242 +cabelas.com,cabelas.com,inbound,001,World,0 +cabestan.com,cab07.net,inbound,001,World,0 +cadremploi.fr,cadremploi.fr,inbound,001,World,0 +cafepress.com,cafepress.com,inbound,001,World,0.000778 +caixa.gov.br,caixa.gov.br,inbound,001,World,0 +californiajobdepartment.com,californiajobdepartment.com,inbound,001,World,0 +californiapsychicsemail.com,californiapsychicsemail.com,inbound,001,World,0 +callcommand.com,callcommand.com,inbound,001,World,0 +calottery.com,calottery.com,inbound,001,World,0.999517 +cam2life.com,hinet.net,inbound,001,World,0 +camel.com,rjrsignup.com,inbound,001,World,0 +camsonline.com,camsonline.com,inbound,001,World,0.136261 +canadiantire.ca,canadiantire.ca,inbound,001,World,0 +canadianvisaexpert.net,canadianvisaexpert.net,inbound,001,World,0 +canalplus.es,canalplus.es,inbound,001,World,0 +cancer.org,delivery.net,inbound,001,World,0 +capillary.co.in,capillary.co.in,inbound,001,World,1 +capitalone.com,bigfootinteractive.com,inbound,001,World,0 +capitalone360.com,ingdirect.com,inbound,001,World,0 +capitaloneemail.com,capitaloneemail.com,inbound,001,World,0 +cardsys.at,cardsys.at,inbound,001,World,1 +care.com,care.com,inbound,001,World,0 +care2.com,care2.com,inbound,001,World,0 +career-hub.net,career-hub.net,inbound,001,World,1 +careerage.com,careerage.com,inbound,001,World,0 +careerbuilder-email.com,careerbuilder-email.com,inbound,001,World,0 +careerbuilder.com,careerbuilder.com,inbound,001,World,0.000449 +careerflash.net,careerflash.net,inbound,001,World,1 +careers24.com,careers24.com,inbound,001,World,0 +careesma.in,careesma.in,inbound,001,World,0 +carmamail.com,carmamail.com,inbound,001,World,0 +carnivalfunmail.com,carnivalfunmail.com,inbound,001,World,0 +carolsdaughter.com,carolsdaughter.com,inbound,001,World,0 +carrefour.fr,carrefour.fr,inbound,001,World,0 +carters.com,carters.com,inbound,001,World,0.003931 +cartrade.com,cartrade.com,inbound,001,World,0.006389 +carwale.com,carwale.com,inbound,001,World,0 +casasbahia.com.br,casasbahia.com.br,inbound,001,World,0 +case.edu,cwru.edu,inbound,001,World,0.999942 +caseyresearch.com,caseyresearch.com,inbound,001,World,1 +castingnetworks.com,castingnetworks.com,inbound,001,World,0.000102 +catchoftheday.com.au,inxserver.de,inbound,001,World,1 +catchyfreebies.net,mmsend53.com,inbound,001,World,0 +catererglobal.com,madgexjb.com,inbound,001,World,0 +caterermail.com,totaljobsmail.co.uk,inbound,001,World,0 +cathkidston.com,cathkidston.co.uk,inbound,001,World,0 +causes.com,causes.com,inbound,001,World,1 +cb2.com,cb2.com,inbound,001,World,0 +cbsig.net,cbsig.net,inbound,001,World,1 +ccavenue.com,avenues.info,inbound,001,World,1 +ccbchurch.com,ccbchurch.com,inbound,001,World,1 +cccampaigns.com,emv5.com,inbound,001,World,0 +cccampaigns.com,emv8.com,inbound,001,World,0 +cccampaigns.net,01net.com,inbound,001,World,0 +cccampaigns.net,cccampaigns.net,inbound,001,World,0 +cccampaigns.net,emv4.net,inbound,001,World,0 +cccampaigns.net,emv9.net,inbound,001,World,0 +ccialerts.com,ccialerts.com,inbound,001,World,0 +ccmbg.com,benchmark.fr,inbound,001,World,0 +ccs.com,footlocker.com,inbound,001,World,2.5e-05 +cdongroup.com,cdongroup.com,inbound,001,World,0.000147 +cecentertainment.com,cecentertainment.com,inbound,001,World,0 +celebritycruises.com,celebritycruises.com,inbound,001,World,0 +cenlat.com,cenlat.com,inbound,001,World,0.064459 +centaur.co.uk,centaur.co.uk,inbound,001,World,0 +centauro.com.br,centauro.com.br,inbound,001,World,0 +centerparcs.co.uk,ec-cluster.com,inbound,001,World,0 +cerberusapp.com,cerberusapp.com,inbound,001,World,1 +cfmailer.com,elabs11.com,inbound,001,World,0 +cfmvmail.com,cfmvmail.com,inbound,001,World,0 +chabad.org,chabad.org,inbound,001,World,0 +champssports.com,footlocker.com,inbound,001,World,0.000131 +chance.com,data-hotel.net,inbound,001,World,0 +change.org,change.org,inbound,001,World,1 +channel4.com,channel4.com,inbound,001,World,0.000613 +charlestyrwhitt.com,charlestyrwhitt.com,inbound,001,World,0 +charter.net,charter.net,inbound,001,World,0 +charter.net,charter.net,outbound,001,World,0 +chase.com,bigfootinteractive.com,inbound,001,World,0 +chase.com,jpmchase.com,inbound,001,World,0.999999 +chatcitynotifications.com,chatcitynotifications.com,inbound,001,World,0 +chaturbate.com,chaturbate.com,inbound,001,World,1 +cheapairmailer.com,cheapairmailer.com,inbound,001,World,0 +cheaperthandirt.com,cheaperthandirt.com,inbound,001,World,2.2e-05 +cheapflights.co.uk,cheapflights.co.uk,inbound,001,World,0 +cheapflights.com,cheapflights.com,inbound,001,World,0 +check.me,check.me,inbound,001,World,0 +cheekylovers.com,ropot.net,inbound,001,World,0 +chefscatalog.com,chefscatalog.com,inbound,001,World,0 +chelseafc.com,chelseafc.com,inbound,001,World,0.003566 +chemistdirect.co.uk,ec-cluster.com,inbound,001,World,0 +chemistry.com,chemistry.com,inbound,001,World,0 +chess.com,chess.com,inbound,001,World,1 +chiangcn.com,chiangcn.com,inbound,001,World,0 +chicagotribune.com,latimes.com,inbound,001,World,0 +chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,001,World,0 +chicos.com,chicos.com,inbound,001,World,0.000156 +childrensplace.com,childrensplace.com,inbound,001,World,0 +chinatrust.com.tw,chinatrust.com.tw,inbound,001,World,0.001272 +chopra.com,chopra.com,inbound,001,World,1 +christianbook.com,christianbook.com,inbound,001,World,1 +christianmingle.com,christianmingle.com,inbound,001,World,0 +christianmingle.com,postdirect.com,inbound,001,World,0 +chtah.com,chtah.net,inbound,001,World,0 +chtah.net,chtah.net,inbound,001,World,0 +cincghq.com,searchhomesingta.com,inbound,001,World,1 +cinesa.es,cccampaigns.com,inbound,001,World,0 +cipherzone.com,infimail.com,inbound,001,World,0 +cir.ca,cir.ca,inbound,001,World,1 +circleofmomsmail.com,circleofmomsmail.com,inbound,001,World,0 +citi.com,citi.com,inbound,001,World,0.999941 +citibank.com,bigfootinteractive.com,inbound,001,World,0 +citibank.com,citi.com,inbound,001,World,0.999997 +citicorp.com,citi.com,inbound,001,World,0.999999 +citruslane.com,citruslane.com,inbound,001,World,5e-06 +citybrands.hu,webinform.hu,inbound,001,World,1 +cityheaven.net,cityheaven.net,inbound,001,World,0 +ck.com,ck.com,inbound,001,World,0 +clarisonic.com,clarisonic.com,inbound,001,World,0 +clarks.com,clarks.com,inbound,001,World,0 +classmates.com,classmates.com,inbound,001,World,0 +clickdimensions.com,clickdimensions.com,inbound,001,World,0 +clickexperts.net,clickexperts.net,inbound,001,World,0 +clickmailer.jp,clickmailer.jp,inbound,001,World,9e-05 +clickon.com.ar,clickon.com.ar,inbound,001,World,0 +clickon.com.br,clickon.com.br,inbound,001,World,0 +clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,001,World,0 +clicplan.com,dmdelivery.com,inbound,001,World,0 +climber.com,climber.com,inbound,001,World,0 +clinique.com,esteelauder.com,inbound,001,World,0 +clubcupon.com.ar,clubcupon.com.ar,inbound,001,World,0 +cmail1.com,createsend.com,inbound,001,World,0 +cmail2.com,createsend.com,inbound,001,World,0 +cmm01.com,coremotivesmarketing.com,inbound,001,World,0 +cmrfalabella.com,cmrfalabella.com,inbound,001,World,0 +coach.com,delivery.net,inbound,001,World,0 +cobone.com,emarsys.net,inbound,001,World,0 +cocacola.co.jp,cocacola.co.jp,inbound,001,World,0 +codebreak.info,codebreak.info,inbound,001,World,1 +codeproject.com,codeproject.com,inbound,001,World,0 +coldwatercreek.com,coldwatercreek.com,inbound,001,World,0 +collectionsetc.com,collectionsetc.com,inbound,001,World,0 +columbia.edu,columbia.edu,inbound,001,World,0.762355 +combzmail.jp,combzmail.jp,inbound,001,World,0 +comcast.net,comcast.net,inbound,001,World,0.888399 +comcast.net,comcast.net,outbound,001,World,0.999999 +comenity.net,alldata.net,inbound,001,World,1 +comenity.net,bigfootinteractive.com,inbound,001,World,0 +commonfloor.com,commonfloor.com,inbound,001,World,1 +communicatoremail.com,communicatoremail.com,inbound,001,World,0 +communitymatrimony.com,communitymatrimony.com,inbound,001,World,1 +compute.internal,amazonaws.com,inbound,001,World,0.858276 +computerworld.com,computerworld.com,inbound,001,World,0 +comunicacaodemkt.com,locaweb.com.br,inbound,001,World,0 +confirmedoptin.com,confirmedoptin.com,inbound,001,World,0 +confirmsignup.com,mmsend53.com,inbound,001,World,0 +conrepmail.com,conrepmail.com,inbound,001,World,0 +constantcontact.com,confirmedcc.com,inbound,001,World,0 +constantcontact.com,constantcontact.com,inbound,001,World,5.3e-05 +constantcontact.com,postini.com,inbound,001,World,0.078144 +constantcontact.com,yahoo.{...},inbound,001,World,0.999724 +contact-darty.com,mm-send.com,inbound,001,World,0 +contactlab.it,contactlab.it,inbound,001,World,0 +containerstore.com,containerstore.com,inbound,001,World,0 +continente.pt,1-hostingservice.com,inbound,001,World,0 +converse.com,converse.com,inbound,001,World,1 +convio.net,convio.net,inbound,001,World,0 +cookingchanneltv.com,cookingchanneltv.com,inbound,001,World,0 +cookpad.com,cookpad.com,inbound,001,World,0 +copernica.nl,picsrv.net,inbound,001,World,0.011507 +copernica.nl,vicinity.nl,inbound,001,World,0.011753 +coppel.com,coppel.com,inbound,001,World,0 +coremotivesmarketing.com,coremotivesmarketing.com,inbound,001,World,0 +cornell.edu,cornell.edu,inbound,001,World,0.195104 +corporateperks.com,nextjump.com,inbound,001,World,0 +correosocc.com,correosocc.com,inbound,001,World,1 +costco.co.uk,costco.com,inbound,001,World,0 +costco.com,costco.com,inbound,001,World,7e-06 +costcophotocenter.com,wc09.net,inbound,001,World,0 +costcoservices.com,costco.com,inbound,001,World,0 +cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,001,World,0 +couchsurfing.org,couchsurfing.com,inbound,001,World,0 +countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,001,World,0 +couponamama.com,couponamama.com,inbound,001,World,1 +coupondunia.in,coupondunia.in,inbound,001,World,1 +cox.com,cox.com,inbound,001,World,0.001665 +cox.net,cox.net,inbound,001,World,0.009187 +cox.net,cox.net,outbound,001,World,0 +coyotelogistics.com,postini.com,inbound,001,World,0 +cp20.com,cp20.com,inbound,001,World,0 +cpbnc.com,cpbnc.com,inbound,001,World,0 +cpbnc.com,fye.com,inbound,001,World,0 +cpc.gov.in,cpc.gov.in,inbound,001,World,0 +cpm.co.ma,cpm.co.ma,inbound,001,World,0 +crabtree-evelyn.com,crabtree-evelyn.com,inbound,001,World,0.000566 +crackle.com,crackle.com,inbound,001,World,0 +craigslist.org,craigslist.org,inbound,001,World,0 +craigslist.org,craigslist.org,outbound,001,World,1 +crainnewsalerts.com,crainnewsalerts.com,inbound,001,World,0 +crashlytics.com,crashlytics.com,inbound,001,World,1 +crashlytics.com,sendgrid.net,inbound,001,World,1 +crateandbarrel.com,crateandbarrel.com,inbound,001,World,0 +cratusservices.in,ramcorp.in,inbound,001,World,0 +creationsrewards.net,creationsrewards.net,inbound,001,World,0 +creditkarma.com,creditkarma.com,inbound,001,World,1 +credoaction.com,credoaction.com,inbound,001,World,1 +cricinfo.com,cricinfo.com,inbound,001,World,0 +cricut.com,elabs12.com,inbound,001,World,0 +criticalimpactinc.com,criticalimpactinc.com,inbound,001,World,0 +critsend.com,critsend.com,inbound,001,World,0 +crmstyle.com,crmstyle.com,inbound,001,World,0 +crocos.jp,crocos.jp,inbound,001,World,0 +crocs-email.com,crocs-email.com,inbound,001,World,0 +crosswalkmail.com,crosswalkmail.com,inbound,001,World,0 +crowdcut.com,crowdcut.com,inbound,001,World,1 +crsend.com,crsend.com,inbound,001,World,0.008688 +crunchyroll.com,crunchyroll.com,inbound,001,World,0 +csas.cz,csas.cz,inbound,001,World,0.999971 +ctrip.com,ctrip.com,inbound,001,World,0.01917 +cudo.com.au,exacttarget.com,inbound,001,World,0 +cuenote.jp,cuenote.jp,inbound,001,World,0 +cumulusdist.net,cumulusdist.net,inbound,001,World,0 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,001,World,1 +cuponatic.com.pe,cuponatic.com.pe,inbound,001,World,1 +cuponicamail.com,fnbox.com,inbound,001,World,0 +cuppon.pl,cuppon.pl,inbound,001,World,0 +curbednetwork.com,curbednetwork.com,inbound,001,World,1 +curriculum.com.br,curriculum.com.br,inbound,001,World,0 +currys.co.uk,currys.co.uk,inbound,001,World,0 +cuspemail.com,neimanmarcusemail.com,inbound,001,World,0 +custom-emailing.com,elabs12.com,inbound,001,World,0 +custombriefings.com,custombriefings.com,inbound,001,World,0 +customercenter.net,customercenter.net,inbound,001,World,0.996453 +customeriomail.com,customeriomail.com,inbound,001,World,1 +cv-library.co.uk,cv-library.co.uk,inbound,001,World,0 +cvbankas.lt,efadm.eu,inbound,001,World,0 +cvent-planner.com,cvent-planner.com,inbound,001,World,0 +cw.com.tw,cw.com.tw,inbound,001,World,0.002535 +cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 +cxomedia.com,cxomedia.com,inbound,001,World,0 +cybercoders.com,cybercoders.com,inbound,001,World,0 +cyberdiet.com.br,allinmedia.com.br,inbound,001,World,0 +cyberlinkmember.com,cyberlinkmember.com,inbound,001,World,0 +d-reizen.nl,dmdelivery.com,inbound,001,World,0 +dabmail.com,iaires.com,inbound,001,World,0 +dabmail.com,mailurja.com,inbound,001,World,0 +dafiti.cl,dafiti.cl,inbound,001,World,0 +dafiti.com.br,fagms.de,inbound,001,World,0 +dailyhoroscope.com,tarot.com,inbound,001,World,0 +dailyom.com,dailyom.com,inbound,001,World,1 +dairyqueen.com,dairyqueen.com,inbound,001,World,0 +datadrivenemail.com,datadrivenemail.com,inbound,001,World,0 +datehookup.com,datehookup.com,inbound,001,World,0 +datingfactory.com,caerussolutions.net,inbound,001,World,0 +datingvipnotifications.com,datingvipnotifications.com,inbound,001,World,0 +daveramsey.com,daveramsey.com,inbound,001,World,0.025924 +daviacalendar.com,daviacalendar.com,inbound,001,World,1 +davidsbridal.com,davidsbridal.com,inbound,001,World,0 +davidstea.com,bronto.com,inbound,001,World,0 +daz3d.com,bronto.com,inbound,001,World,0 +dbgi.co.uk,emc1.co.uk,inbound,001,World,0 +ddc-emails.com,ddc-emails.com,inbound,001,World,0 +deal.com.sg,emarsys.net,inbound,001,World,0 +dealchicken.com,dealchicken.com,inbound,001,World,0 +dealchicken.com,exacttarget.com,inbound,001,World,0 +dealersocket.com,dealersocket.com,inbound,001,World,4e-06 +dealfind.com,dealfind.com,inbound,001,World,0 +dealnews.com,dealnews.com,inbound,001,World,0 +dealsaver.com,secondstreetmedia.com,inbound,001,World,0.999823 +dealsdirect.com.au,dealsdirect.com.au,inbound,001,World,0 +dealspl.us,dealspl.us,inbound,001,World,0 +debian.org,debian.org,inbound,001,World,1 +debshops.com,lstrk.net,inbound,001,World,1 +deezer.com,dms30.com,inbound,001,World,0 +deliasshopemail.com,deliasshopemail.com,inbound,001,World,0 +delivery.net,delivery.net,inbound,001,World,0 +delivery.net,m0.net,inbound,001,World,0 +dell.com,bfi0.com,inbound,001,World,0 +dell.com,dell.com,inbound,001,World,0.969277 +delta.com,delta.com,inbound,001,World,0.092496 +dena.ne.jp,dena.ne.jp,inbound,001,World,0.000249 +dentalsenders.com,dentalsenders.com,inbound,001,World,0 +dermstore.com,exacttarget.com,inbound,001,World,0 +descontos.pt,descontos.pt,inbound,001,World,0 +designerapparel.com,myperfectsale.com,inbound,001,World,1 +despegar.com,despegar.com,inbound,001,World,0 +dhgate.com,chtah.net,inbound,001,World,0 +dhl.com,dhl.com,inbound,001,World,0.994107 +dice.com,dice.com,inbound,001,World,0 +dietaesaude.com.br,dietaesaude.com.br,inbound,001,World,0 +dietnavi.com,data-hotel.net,inbound,001,World,0 +digitalmailer.com,digitalmailer.com,inbound,001,World,0 +digitalmedia-comunicacion.es,chtah.net,inbound,001,World,0 +digitalromanceinc.com,digitalromanceinc.com,inbound,001,World,1 +dinda.com.br,dinda.com.br,inbound,001,World,0.017061 +dip-net.co.jp,dip-net.co.jp,inbound,001,World,0 +directcrm.ru,directcrm.ru,inbound,001,World,0 +directresponsemanager.com,wide.ne.jp,inbound,001,World,0 +directv.com,directv.com,inbound,001,World,0.050604 +directvla.com,directvla.com,inbound,001,World,0 +disc.co.jp,disc.co.jp,inbound,001,World,0.001051 +discover.com,discover.com,inbound,001,World,0 +discover.com,discoverfinancial.com,inbound,001,World,1 +dishtv.co.in,dishtv.co.in,inbound,001,World,0.047664 +disney.co.uk,emv9.com,inbound,001,World,0 +disneydestinations.com,disneyparks.com,inbound,001,World,0 +disneydestinations.com,disneyworld.com,inbound,001,World,0 +disparadordeemails.com,locaweb.com.br,inbound,001,World,0 +disqus.net,disqus.net,inbound,001,World,0 +diynetwork.com,diynetwork.com,inbound,001,World,0 +dks.com.tw,dks.com.tw,inbound,001,World,0.018134 +dmm.com,dmm.com,inbound,001,World,0 +dn.net,naukri.com,inbound,001,World,0 +docomo.ne.jp,docomo.ne.jp,inbound,001,World,0 +docomo.ne.jp,docomo.ne.jp,outbound,001,World,0 +doctoroz.com,email-sharecare2.com,inbound,001,World,0 +docusign.net,docusign.net,inbound,001,World,0.985435 +dollartree.com,email-dollartree.com,inbound,001,World,0 +dominos.com,dominos.com,inbound,001,World,0.011684 +dominos.com.au,dominos.com.au,inbound,001,World,0 +dominosemail.co.uk,dominosemail.co.uk,inbound,001,World,0 +donationnet.net,donationnet.net,inbound,001,World,0 +donuts.ne.jp,dnuts.jp,inbound,001,World,0 +doodle.com,doodle.com,inbound,001,World,1 +dorothyperkins.com,dorothyperkins.com,inbound,001,World,0 +dotmailer-email.com,dotmailer.com,inbound,001,World,0 +dotmailer.co.uk,dotmailer.com,inbound,001,World,0 +dotz.com.br,dotz.com.br,inbound,001,World,0 +doubletakeoffers.com,doubletakeoffers.com,inbound,001,World,0 +dowjones.info,dowjones.info,inbound,001,World,0 +downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,001,World,0 +dpapp.nl,prikbordmailer.nl,inbound,001,World,0 +dpapp.nl,sslsecuref.nl,inbound,001,World,0 +dptagent.biz,dptagent.biz,inbound,001,World,0 +dptagent.net,dptagent.net,inbound,001,World,0 +draftkings.com,draftkings.com,inbound,001,World,0 +dreamhost.com,dreamhost.com,inbound,001,World,0 +dreammail.ne.jp,dreammail.jp,inbound,001,World,0 +dreamwidth.org,dreamwidth.org,inbound,001,World,0 +dreivip.com,dreivip.com,inbound,001,World,0.000301 +dress-for-less.de,privalia.com,inbound,001,World,0 +drhinternet.net,drhinternet.net,inbound,001,World,0 +driftem.com,emce2.in,inbound,001,World,0 +driftem.com,mailurja.com,inbound,001,World,0 +drjays-mail.com,drjays-mail.com,inbound,001,World,0 +dromadaire-news.com,ecmcluster.com,inbound,001,World,0 +dropbox.com,dropbox.com,inbound,001,World,1 +dropboxmail.com,dropbox.com,inbound,001,World,1 +drushim.co.il,drushim.co.il,inbound,001,World,0 +drweb.com,drweb.com,inbound,001,World,0.969315 +dstyleweb.com,dstyleweb.com,inbound,001,World,0.001099 +dsw.com,dsw.com,inbound,001,World,0 +ducks.org,uptilt.com,inbound,001,World,0 +duke.edu,duke.edu,inbound,001,World,0.308101 +dukecareers.com,dukecareers.com,inbound,001,World,1 +duluthtradingemail.com,email-duluthtrading.com,inbound,001,World,0 +dvor.com,dvor.com,inbound,001,World,0 +dynamite-safelist.com,thomas-j-brown.com,inbound,001,World,0 +dynect-mailer.net,dynect.net,inbound,001,World,0 +dynect-mailer.net,sendlabs.com,inbound,001,World,0 +e-activist.com,e-activist.com,inbound,001,World,0 +e-beallsonline.com,e-stagestores.com,inbound,001,World,0 +e-bodyc.com,email-bodycentral.com,inbound,001,World,0 +e-boks.dk,e-boks.dk,inbound,001,World,1 +e-costco.mx,costco.com,inbound,001,World,0 +e-ebuyer.com,e-ebuyer.com,inbound,001,World,0 +e-goodysonline.com,e-stagestores.com,inbound,001,World,0 +e-jobs-ville.com,e-jobs-ville.com,inbound,001,World,1 +e-leclerc.com,e-leclerc.com,inbound,001,World,0.000248 +e-mark.nl,e-mark.nl,inbound,001,World,0 +e-ngine.nl,e-ngine.nl,inbound,001,World,0 +e-peebles.com,e-stagestores.com,inbound,001,World,0 +e-rewards.net,e-rewards.net,inbound,001,World,1 +e-stagestores.com,e-stagestores.com,inbound,001,World,0 +e-travelclub.es,e-travelclub.es,inbound,001,World,0 +e2ma.net,e2ma.net,inbound,001,World,1 +ea.com,ea.com,inbound,001,World,0.001314 +eaccess.net,postini.com,inbound,001,World,0 +earn-e-miles.com,earn-e-miles.com,inbound,001,World,0 +earnerslist.com,traxweb.net,inbound,001,World,9e-06 +earthfare-email.com,edclient2.com,inbound,001,World,0 +earthlink.net,earthlink.net,inbound,001,World,0.031678 +earthlink.net,earthlink.net,outbound,001,World,0 +eastbay.com,footlocker.com,inbound,001,World,0 +easycanvasprints.com,easycanvasprints.com,inbound,001,World,0 +easyhealthoptions.com,easyhealthoptions.com,inbound,001,World,1 +easyhits4u.com,easyhits4u.com,inbound,001,World,1 +easyhits4u.com,relmax.net,inbound,001,World,0 +easyroommate.com,easyroommate.com,inbound,001,World,0.108483 +ebags.com,ebags.com,inbound,001,World,0 +ebates.com,bfi0.com,inbound,001,World,0 +ebay-kleinanzeigen.de,mobile.de,inbound,001,World,1 +ebay.{...},ebay.{...},inbound,001,World,0.99953 +ebay.{...},emarsys.net,inbound,001,World,0 +ebay.{...},postdirect.com,inbound,001,World,0 +ebizac2.com,ebizac2.com,inbound,001,World,0 +ebizac3.com,ebizac3.com,inbound,001,World,0 +eblastengine.com,secondstreetmedia.com,inbound,001,World,0.999827 +ebuildabear.com,ebuildabear.com,inbound,001,World,8e-06 +ec2.internal,amazonaws.com,inbound,001,World,0.769117 +ec21.com,ec21.com,inbound,001,World,0.002261 +ecasend.com,ecasend.com,inbound,001,World,0 +ecnavi.jp,ecnavi.jp,inbound,001,World,0 +ecommzone.com,ecommzone.com,inbound,001,World,0 +ed.gov,leepfrog.com,inbound,001,World,0.106381 +ed10.net,ed10.com,inbound,001,World,0 +ed10.net,postini.com,inbound,001,World,0.083658 +edarling.fr,fagms.de,inbound,001,World,0 +eddiebauer.com,eddiebauer.com,inbound,001,World,0 +edima.hu,edima.hu,inbound,001,World,0 +edirect1.com,ivytech.edu,inbound,001,World,0 +edmodo.com,edmodo.com,inbound,001,World,1.1e-05 +educationzone.co.in,iaires.com,inbound,001,World,0 +eduk.com,eduk.com,inbound,001,World,0 +efamilydollar.com,efamilydollar.com,inbound,001,World,0 +effectivesafelist.com,zoothost.com,inbound,001,World,0 +efox-shop.com,dmdelivery.com,inbound,001,World,0 +eharmony.com,eharmony.com,inbound,001,World,1e-06 +eigbox.net,eigbox.net,inbound,001,World,0 +ejobs.ro,ejobs.ro,inbound,001,World,8.4e-05 +elabs10.com,elabs10.com,inbound,001,World,0 +elabs12.com,elabs12.com,inbound,001,World,0 +elabs3.com,elabs3.com,inbound,001,World,0 +elabs3.com,meritline.com,inbound,001,World,0 +elabs5.com,elabs5.com,inbound,001,World,0 +elabs6.com,elabs6.com,inbound,001,World,0 +elaine-asp.de,artegic.net,inbound,001,World,0.999997 +elanceonline.com,elanceonline.com,inbound,001,World,0 +elcorteingles.es,elcorteingles.es,inbound,001,World,0 +eleadtrack.net,eleadtrack.net,inbound,001,World,0 +elektronskaposta.si,eprvak.si,inbound,001,World,0 +elettershop.de,servicemail24.de,inbound,001,World,1 +elistas.net,elistas.net,inbound,001,World,0 +elitesafelist.com,elitesafelist.com,inbound,001,World,0 +elkjop.no,ec-cluster.com,inbound,001,World,0 +elkjop.no,eccluster.com,inbound,001,World,0 +elo7.com.br,elo7.com.br,inbound,001,World,0 +emag.ro,emag.ro,inbound,001,World,0.005013 +email-1800contacts.com,email-1800contacts.com,inbound,001,World,0 +email-aaa.com,email-aaa.com,inbound,001,World,0 +email-aeriagames.com,email-aeriagames.com,inbound,001,World,0 +email-comparethemarket.com,smartfocusdigital.net,inbound,001,World,0 +email-cooking.com,email-cooking.com,inbound,001,World,0 +email-dressbarn.com,email-dressbarn.com,inbound,001,World,0 +email-firestone.com,reminder-firestone.com,inbound,001,World,0 +email-galls.com,email-galls.com,inbound,001,World,1 +email-honest.com,email-honest.com,inbound,001,World,0 +email-od.com,email-od.com,inbound,001,World,0.999652 +email-od.com,smtprelayserver.com,inbound,001,World,0.999779 +email-petsmart.com,email-petsmart.com,inbound,001,World,0 +email-sportchalet.com,email-sportchalet.com,inbound,001,World,0 +email-telekom.de,ecm-cluster.com,inbound,001,World,0 +email-ticketdada.com,email-ticketdada.com,inbound,001,World,1 +email-totalwine.com,email-totalwine.com,inbound,001,World,0 +email-wildstar-online.com,email-carbine.com,inbound,001,World,0 +email2-beyond.com,messagebus.com,inbound,001,World,0 +email360api.com,email360api.com,inbound,001,World,0 +email365inc.com,email365inc.com,inbound,001,World,0 +email3m.com,email3m.com,inbound,001,World,0 +email4-beyond.com,email4-beyond.com,inbound,001,World,0 +emailcounts.com,secureserver.net,inbound,001,World,0 +emaildir2.com,emaildirect.net,inbound,001,World,0 +emaildir2.com,espsnd.com,inbound,001,World,0 +emailnotify.net,emailnotify.net,inbound,001,World,0.961424 +emailrestaurant.com,emailrestaurant.com,inbound,001,World,0 +emailsbancoestado.cl,emailsbancoestado.cl,inbound,001,World,0 +emailsripley.cl,etarget.cl,inbound,001,World,0 +emailtoryburch.com,emailtoryburch.com,inbound,001,World,0 +emarsys.net,emarsys.net,inbound,001,World,0.092041 +embarqmail.com,centurylink.net,inbound,001,World,0.999918 +embluejet.com,embluejet.com,inbound,001,World,0 +embluejet.com,emblueuser.com,inbound,001,World,0 +emcsend.com,emcsend.com,inbound,001,World,0 +emergencyemail.org,emergencyemail.org,inbound,001,World,0 +eminentinc.com,eminentinc.com,inbound,001,World,0 +emktsender.net,locaweb.com.br,inbound,001,World,0 +emktviajarbarato.com.br,splio.com.br,inbound,001,World,0.875902 +emma.cl,emma.cl,inbound,001,World,0.996895 +emobile.ad.jp,postini.com,inbound,001,World,0 +employboard.com,employboard.com,inbound,001,World,1 +empoweredcomms.com.au,empoweredcomms.com.au,inbound,001,World,0 +emsecure.net,emsecure.net,inbound,001,World,0 +emsmtp.com,emsmtp.com,inbound,001,World,0.175797 +en-japan.com,en-japan.com,inbound,001,World,0.000204 +en25.com,kqed.org,inbound,001,World,0 +enewscartes.net,bp06.net,inbound,001,World,0 +enewsletter.pl,enewsletter.pl,inbound,001,World,0.005395 +enewsletter.pl,mydeal.pl,inbound,001,World,0 +enewsletter.pl,sare25.com,inbound,001,World,0 +enplenitud.com,enplenitud.com,inbound,001,World,0 +entregadeemails.com,locaweb.com.br,inbound,001,World,0 +entregadordecampanhas.net,locaweb.com.br,inbound,001,World,0 +entrepreneur.com,entrepreneur.com,inbound,001,World,0 +enviodecampanhas.net,locaweb.com.br,inbound,001,World,0 +enviodemkt.com.br,locaweb.com.br,inbound,001,World,0 +eonet.ne.jp,eonet.ne.jp,inbound,001,World,0.99955 +epaper.com.tw,epaper.com.tw,inbound,001,World,0 +eplus.jp,eplus.jp,inbound,001,World,0 +epriority.com,epriority.com,inbound,001,World,0 +equifax.com,equifax.com,inbound,001,World,0.936013 +equussafelist.com,equussafelist.com,inbound,001,World,0.000265 +eslitebooks.com,eslitebooks.com,inbound,001,World,0 +espmp-agfr.net,bp06.net,inbound,001,World,0 +esprit-friends.com,esprit-friends.com,inbound,001,World,0 +esri.com,esri.com,inbound,001,World,0.983104 +esteelauder.com,esteelauder.com,inbound,001,World,0 +ethingsremembered.com,ethingsremembered.com,inbound,001,World,0 +etrade.com,etrade.com,inbound,001,World,0.021422 +etransmail.com,etransmail.com,inbound,001,World,0 +etransmail.com,ptransmail.com,inbound,001,World,0 +etrmailbox.com,etrmailbox.com,inbound,001,World,0 +etsy.com,etsy.com,inbound,001,World,0.020844 +euromsg.net,euromsg.net,inbound,001,World,0 +evaair.com,evaair.com,inbound,001,World,0.007363 +evanguard.com,evanguard.com,inbound,001,World,0 +evanscycles.com,msgfocus.com,inbound,001,World,0 +eventbrite.com,eventbrite.com,inbound,001,World,0 +evernote.com,evernote.com,inbound,001,World,1 +eversavelocal.com,eversavelocal.com,inbound,001,World,0 +everydayfamily.com,everydayfamily.com,inbound,001,World,0 +everydayhealthinc.com,waterfrontmedia.net,inbound,001,World,0 +everyjobforme.com,everyjobforme.com,inbound,001,World,0 +everytown.org,everytown.org,inbound,001,World,1 +exacttarget.com,bazaarvoice.com,inbound,001,World,0 +exacttarget.com,booksamillion.com,inbound,001,World,0 +exacttarget.com,exacttarget.com,inbound,001,World,0.000325 +exacttarget.com,msg.com,inbound,001,World,0 +exacttarget.com,redboxinstant.com,inbound,001,World,0 +exacttarget.com,skylinetechnologies.com,inbound,001,World,0 +exchangesolutions.com,exchangesolutions.com,inbound,001,World,0.000143 +exec-u-net-mail.com,exec-u-net-mail.com,inbound,001,World,0 +expediamail.com,airasiago.com,inbound,001,World,1 +expediamail.com,exacttarget.com,inbound,001,World,1 +expediamail.com,expediamail.com,inbound,001,World,0.839662 +expediamail.com,quotitmail.com,inbound,001,World,0 +experteer.com,experteer.com,inbound,001,World,0 +express.com,expressfashion.com,inbound,001,World,0 +exprpt.com,exprpt.com,inbound,001,World,0 +expvtinboxhub.net,expvtinboxhub.net,inbound,001,World,0 +extra.com.br,emv8.com,inbound,001,World,0 +eyepin.com,eyepin.com,inbound,001,World,0 +ezweb.ne.jp,ezweb.ne.jp,inbound,001,World,0.282076 +ezweb.ne.jp,ezweb.ne.jp,outbound,001,World,0 +fabfurnish.com,fagms.de,inbound,001,World,0 +fabletics.com,bronto.com,inbound,001,World,0 +facebook.com,facebook.com,inbound,001,World,0.553197 +facebook.com,facebook.com,outbound,001,World,1 +facebookappmail.com,facebook.com,inbound,001,World,1 +facebookmail.com,facebook.com,inbound,001,World,1 +facebookmail.com,postini.com,inbound,001,World,0.726315 +facebookmail.com,yahoo.{...},inbound,001,World,0.999904 +facilisimo.com,facilisimo.com,inbound,001,World,1 +fagms.net,fagms.de,inbound,001,World,0 +falabella.com,falabella.com,inbound,001,World,0 +familychristianmail.com,familychristianmail.com,inbound,001,World,0 +famousfootwear.com,famousfootwear.com,inbound,001,World,0 +fanatics.com,fanatics.com,inbound,001,World,0 +fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,001,World,0 +fanbridge.com,fanbridge.com,inbound,001,World,0.000198 +fanfiction.com,fictionpress.com,inbound,001,World,1 +fanofannas.com,fanofannas.com,inbound,001,World,0 +fansedge.com,fansedge.com,inbound,001,World,0 +farmers.com,farmers.com,inbound,001,World,0.999984 +farmersonly.com,mailgun.net,inbound,001,World,1 +farmersonly.com,mailgun.us,inbound,001,World,1 +fashion2hub.in,mgenie.in,inbound,001,World,0 +fastcompany.com,fastcompany.com,inbound,001,World,0 +fastgb.com,fastgb.com,inbound,001,World,0 +fastlistmailer.com,zoothost.com,inbound,001,World,0 +fastweb.com,fastweb.com,inbound,001,World,0 +fbi.gov,fbi.gov,inbound,001,World,0 +fbmta.com,fbmta.com,inbound,001,World,0 +fc2.com,fc2.com,inbound,001,World,0.000798 +fedex.com,fedex.com,inbound,001,World,0.921011 +fedoraproject.org,fedoraproject.org,inbound,001,World,5e-06 +feedblitz.com,feedblitz.com,inbound,001,World,0 +feld-ent.com,postdirect.com,inbound,001,World,0 +felissimo.jp,felissimo.jp,inbound,001,World,0 +fellowshiponemail.com,fellowshiponemail.com,inbound,001,World,0 +fetlifemail.com,fetlifemail.com,inbound,001,World,0 +fibertel.com.ar,fibertel.com.ar,inbound,001,World,0.003897 +fidelity.com,fidelity.com,inbound,001,World,1 +fidelizador.org,fidelizador.org,inbound,001,World,0 +financialfreedommail.com,financialfreedommail.com,inbound,001,World,0 +finansbank.com.tr,finansbank.com.tr,inbound,001,World,0.927 +findexpvtinbox.com,findexpvtinbox.com,inbound,001,World,0 +fingerhut.com,fingerhut.com,inbound,001,World,0.020664 +finishline.com,finishline.com,inbound,001,World,0 +finn.no,schibsted-it.no,inbound,001,World,0.002893 +firemountaingems.com,firemountaingems.com,inbound,001,World,0 +fiscosoft.com.br,fiscosoft.com.br,inbound,001,World,0 +fisher-price.com,fisher-price.com,inbound,001,World,0 +fitbit.com,fitbit.com,inbound,001,World,1 +fitnessmagazine.com,meredith.com,inbound,001,World,0 +fiverr.com,fiverr.com,inbound,001,World,0 +fixeads.com,fixeads.com,inbound,001,World,0 +flets.com,flets.com,inbound,001,World,0 +flexmls.com,flexmls.com,inbound,001,World,0.999992 +flightaware.com,flightaware.com,inbound,001,World,0.020698 +flipkart.com,flipkart.com,inbound,001,World,1 +flirchi.com,flirchi.com,inbound,001,World,1.0 +flirt.com,ropot.net,inbound,001,World,0 +flirthookup.com,flirthookup.com,inbound,001,World,1 +flirtlocal.com,flirtlocal.com,inbound,001,World,1 +flmsecure.com,fling.com,inbound,001,World,0 +flmsecure.com,flmsecure.com,inbound,001,World,0 +floridajobdepartment.com,floridajobdepartment.com,inbound,001,World,0 +flyceb.com,flyceb.com,inbound,001,World,0 +flyfrontier.com,flyfrontier.com,inbound,001,World,0 +flymonarchemail.com,flymonarchemail.com,inbound,001,World,0 +fmworld.net,fmworld.net,inbound,001,World,0 +fnac.com,fnac.com,inbound,001,World,0.021985 +fnb.co.za,fnb.co.za,inbound,001,World,0.104983 +fofa.jp,mpme.jp,inbound,001,World,0 +follow-up.se,follow-up.se,inbound,001,World,1 +foodnetwork.com,foodnetwork.com,inbound,001,World,0 +foolsubs.com,foolcs.com,inbound,001,World,0 +foolsubs.com,foolsubs.com,inbound,001,World,0 +footaction.com,footlocker.com,inbound,001,World,0 +footlocker.com,footlocker.com,inbound,001,World,0.000605 +forcemail.in,iaires.com,inbound,001,World,0 +foreseegame.com,iaires.com,inbound,001,World,0 +forever21.com,forever21.com,inbound,001,World,0 +fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,001,World,0 +fotffamily.com,fotffamily.com,inbound,001,World,0 +fotocasa.es,fotocasa.es,inbound,001,World,0 +fotolivro.com.br,fotolivro.com.br,inbound,001,World,0 +fotostrana.ru,fotocdn.net,inbound,001,World,3.4e-05 +foursquare.com,foursquare.com,inbound,001,World,1 +foxnews.com,foxnews.com,inbound,001,World,0.0139 +fpmailerbr.com,fpmailerbr.com,inbound,001,World,0 +fragrancenet.com,fragrancenet.com,inbound,001,World,0.000427 +francescas.com,bronto.com,inbound,001,World,0 +free-lance.ru,free-lance.ru,inbound,001,World,0 +free.fr,free.fr,inbound,001,World,0.984012 +free.fr,free.fr,outbound,001,World,6.9e-05 +freeadsmailer.com,zoothost.com,inbound,001,World,0 +freebeesafelist.com,zoothost.com,inbound,001,World,0 +freebizmag.com,delivery.net,inbound,001,World,0 +freebsd.org,freebsd.org,inbound,001,World,0.999835 +freecycle.org,freecycle.org,inbound,001,World,0.999927 +freedesktop.org,freedesktop.org,inbound,001,World,0 +freeflys.com,freeflys.com,inbound,001,World,0 +freelancer.com,freelancer.com,inbound,001,World,0 +freelancer.com,freelancernotify.com,inbound,001,World,0 +freelancer.com,getafreelancer.com,inbound,001,World,0 +freelists.org,iquest.net,inbound,001,World,0 +freelotto.com,plasmanetinc.com,inbound,001,World,0 +freemail.hu,freemail.hu,outbound,001,World,0 +freeml.com,gmo-media.jp,inbound,001,World,0 +freepeople.com,freepeople.com,inbound,001,World,0 +freesafelistking.com,zoothost.com,inbound,001,World,0 +freesafelistmailer.com,waters-advertising.com,inbound,001,World,0 +freshdesk.com,freshdesk.com,inbound,001,World,1 +freshers2015.com,secureserver.net,inbound,001,World,0 +freshlatesave.com,freshlatesave.com,inbound,001,World,1 +freshmail.pl,freshmail.pl,inbound,001,World,0 +fridays.com,fridays.com,inbound,001,World,0 +friskone.com,mailurja.com,inbound,001,World,0 +frk.com,frk.com,inbound,001,World,0.999995 +frontdoor.com,frontdoor.com,inbound,001,World,0 +frontgate-email.com,frontgate-email.com,inbound,001,World,0 +frontsight.com,frontsight.com,inbound,001,World,0 +frys.com,frys.com,inbound,001,World,0.00388 +frysmail.com,frysmail.com,inbound,001,World,0 +fspeletters.com,agorapub.co.uk,inbound,001,World,0 +ftchinese.com,ftchinese.com,inbound,001,World,0 +fubonshop.com,fubonshop.com,inbound,001,World,0 +fuckbooknet.net,infinitypersonals.com,inbound,001,World,0 +fuelrewards.com,britecast.com,inbound,001,World,0 +fundplaza.co.in,arrowsignindia.com,inbound,001,World,0 +fundplaza.in,fundplaza.in,inbound,001,World,0 +funonthenet.in,funonthenet.in,inbound,001,World,1 +futureshop.com,futureshop.com,inbound,001,World,0 +futurmailer.pt,futurmailer.pt,inbound,001,World,0 +gabbar.info,gabbar.info,inbound,001,World,1 +gaiaonline.com,gaiaonline.com,inbound,001,World,0 +gamecity.ne.jp,gamecity.ne.jp,inbound,001,World,0 +gamefly.com,gamefly.com,inbound,001,World,0.013381 +gamehouse.com,gamehouse.com,inbound,001,World,0 +gamingmails.com,gamingmails.com,inbound,001,World,0 +gap.com,gap.com,inbound,001,World,0 +gap.eu,gap.eu,inbound,001,World,0 +gapcanada.ca,gapcanada.ca,inbound,001,World,0 +garanti.com.tr,euromsg.net,inbound,001,World,0 +garanti.com.tr,garanti.com.tr,inbound,001,World,0.421936 +gardeningclubmail.co.uk,msgfocus.com,inbound,001,World,0 +garnethill-email.com,garnethill-email.com,inbound,001,World,0 +gaylordalert.com,gaylordalert.com,inbound,001,World,0 +gbyguess.com,guess.com,inbound,001,World,0.001787 +gcast.com.au,systemsserver.net,inbound,001,World,0 +gdtsuccess.com,groupdealtools.com,inbound,001,World,1 +geico.com,geico.com,inbound,001,World,0.096879 +gemoney.com,rsys1.com,inbound,001,World,0 +gene.com,roche.com,inbound,001,World,1 +generalmills.com,boxtops4education.com,inbound,001,World,0 +generalmills.com,pillsbury.com,inbound,001,World,0 +gentoo.org,gentoo.org,inbound,001,World,1 +geocaching.com,groundspeak.com,inbound,001,World,1 +geojit.com,geojit.com,inbound,001,World,0.203748 +get-me-jobs.com,get-me-jobs.com,inbound,001,World,0 +gethired.com,gethired.com,inbound,001,World,1 +getinbox.net,getinbox.net,inbound,001,World,0 +getitfree.us,getitfree.us,inbound,001,World,0 +getkeepsafe.com,getkeepsafe.com,inbound,001,World,1 +getmein.com,getmein.com,inbound,001,World,0 +getpaidsolutions.com,getpaidsolutions.com,inbound,001,World,1 +getpocket.com,bronto.com,inbound,001,World,0 +getresponse.com,getresponse.com,inbound,001,World,0 +gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,001,World,0 +ghin.com,ghinconnect.com,inbound,001,World,0 +ghup.in,mgenie.in,inbound,001,World,0 +giffgaff.com,giffgaff.com,inbound,001,World,0 +gillyhicks-email.com,abercrombie-email.com,inbound,001,World,0 +gilt.com,gilt.com,inbound,001,World,1e-06 +gilt.jp,gilt.jp,inbound,001,World,0 +github.com,github.com,inbound,001,World,1 +github.com,github.net,inbound,001,World,1 +github.com,postini.com,inbound,001,World,0.872186 +glassdoor.com,glassdoor.com,inbound,001,World,0.662834 +glasses.com,glasses.com,inbound,001,World,0 +gliq.com,gliq.com,inbound,001,World,0.99852 +globalmembersupport.com,globalmembersupport.com,inbound,001,World,0 +globalsafelist.com,globalsafelist.com,inbound,001,World,0 +globalsources.com,globalsources.com,inbound,001,World,0.007546 +globalspec.com,globalspec.com,inbound,001,World,0 +globaltestmarket.com,globaltestmarket.com,inbound,001,World,0 +globasemail.com,globasemail.com,inbound,001,World,0.951088 +globetel.com.ph,globetel.com.ph,inbound,001,World,1 +gmail.com,02.net,inbound,001,World,0.998007 +gmail.com,amazonaws.com,inbound,001,World,0.988441 +gmail.com,anteldata.net.uy,inbound,001,World,0.998653 +gmail.com,as13285.net,inbound,001,World,0.999943 +gmail.com,asianet.co.th,inbound,001,World,0.998473 +gmail.com,au-net.ne.jp,inbound,001,World,1 +gmail.com,bbox.fr,inbound,001,World,0.919868 +gmail.com,bbtec.net,inbound,001,World,1 +gmail.com,belgacom.be,inbound,001,World,0.822281 +gmail.com,bell.ca,inbound,001,World,0.992482 +gmail.com,bellsouth.net,inbound,001,World,0.9996 +gmail.com,bezeqint.net,inbound,001,World,0.974444 +gmail.com,bigpond.net.au,inbound,001,World,0.999907 +gmail.com,blackberry.com,inbound,001,World,0.996337 +gmail.com,bluewin.ch,inbound,001,World,0.9337 +gmail.com,brasiltelecom.net.br,inbound,001,World,0.99995 +gmail.com,btcentralplus.com,inbound,001,World,0.999969 +gmail.com,centurytel.net,inbound,001,World,0.999415 +gmail.com,cgocable.net,inbound,001,World,0.998291 +gmail.com,charter.com,inbound,001,World,0.999111 +gmail.com,chello.nl,inbound,001,World,0.999951 +gmail.com,claro.net.br,inbound,001,World,1 +gmail.com,comcast.net,inbound,001,World,0.999616 +gmail.com,comcastbusiness.net,inbound,001,World,0.985464 +gmail.com,cox.net,inbound,001,World,0.962636 +gmail.com,data-hotel.net,inbound,001,World,0.000605 +gmail.com,emailsrvr.com,inbound,001,World,1 +gmail.com,embarqhsd.net,inbound,001,World,0.999554 +gmail.com,fastwebnet.it,inbound,001,World,0.984708 +gmail.com,franchiseindia.com,inbound,001,World,1 +gmail.com,frontiernet.net,inbound,001,World,0.995233 +gmail.com,gvt.net.br,inbound,001,World,0.999513 +gmail.com,hinet.net,inbound,001,World,0.97312 +gmail.com,iinet.net.au,inbound,001,World,0.966984 +gmail.com,jazztel.es,inbound,001,World,0.999701 +gmail.com,lorexddns.net,inbound,001,World,0 +gmail.com,majesticmoneymailer.com,inbound,001,World,1 +gmail.com,mchsi.com,inbound,001,World,0.999951 +gmail.com,movistar.cl,inbound,001,World,0.999361 +gmail.com,mtnl.net.in,inbound,001,World,0.999527 +gmail.com,mycingular.net,inbound,001,World,0.999918 +gmail.com,myvzw.com,inbound,001,World,0.999889 +gmail.com,naukri.com,inbound,001,World,0.000998 +gmail.com,net24.it,inbound,001,World,0.999964 +gmail.com,netcabo.pt,inbound,001,World,0.998164 +gmail.com,netvigator.com,inbound,001,World,0.818637 +gmail.com,numericable.fr,inbound,001,World,0.999726 +gmail.com,ocn.ne.jp,inbound,001,World,0.99466 +gmail.com,ono.com,inbound,001,World,0.995991 +gmail.com,optonline.net,inbound,001,World,0.999776 +gmail.com,optusnet.com.au,inbound,001,World,0.992856 +gmail.com,orange.es,inbound,001,World,0.998743 +gmail.com,orange.fr,inbound,001,World,0 +gmail.com,otenet.gr,inbound,001,World,0.957964 +gmail.com,panda-world.ne.jp,inbound,001,World,1 +gmail.com,postini.com,inbound,001,World,0.776029 +gmail.com,proxad.net,inbound,001,World,0.998094 +gmail.com,qwest.net,inbound,001,World,0.997575 +gmail.com,rcn.com,inbound,001,World,0.986768 +gmail.com,rima-tde.net,inbound,001,World,0.99915 +gmail.com,rogers.com,inbound,001,World,0.999917 +gmail.com,rr.com,inbound,001,World,0.967896 +gmail.com,sbcglobal.net,inbound,001,World,0.998817 +gmail.com,secureserver.net,inbound,001,World,0.272513 +gmail.com,seed.net.tw,inbound,001,World,0.992252 +gmail.com,sfr.net,inbound,001,World,0.999878 +gmail.com,shawcable.net,inbound,001,World,0.999998 +gmail.com,singnet.com.sg,inbound,001,World,0.955461 +gmail.com,skybroadband.com,inbound,001,World,0.999854 +gmail.com,spcsdns.net,inbound,001,World,0.999998 +gmail.com,suddenlink.net,inbound,001,World,0.961594 +gmail.com,t-ipconnect.de,inbound,001,World,0.999841 +gmail.com,tdc.net,inbound,001,World,0.999591 +gmail.com,telecom.net.ar,inbound,001,World,0.999664 +gmail.com,telecomitalia.it,inbound,001,World,0.996998 +gmail.com,telekom.hu,inbound,001,World,0.999977 +gmail.com,telenet.be,inbound,001,World,1 +gmail.com,telepac.pt,inbound,001,World,0.999584 +gmail.com,telesp.net.br,inbound,001,World,0.999743 +gmail.com,telia.com,inbound,001,World,1 +gmail.com,telkomadsl.co.za,inbound,001,World,0.999989 +gmail.com,telus.com,inbound,001,World,1 +gmail.com,telus.net,inbound,001,World,0.974677 +gmail.com,threembb.co.uk,inbound,001,World,1 +gmail.com,tmodns.net,inbound,001,World,0.999979 +gmail.com,totbb.net,inbound,001,World,0.999876 +gmail.com,tpgi.com.au,inbound,001,World,0.999649 +gmail.com,tpnet.pl,inbound,001,World,0.999761 +gmail.com,veloxzone.com.br,inbound,001,World,0.999969 +gmail.com,verizon.net,inbound,001,World,0.990214 +gmail.com,videotron.ca,inbound,001,World,0.967213 +gmail.com,virginm.net,inbound,001,World,0.996611 +gmail.com,vodacom.co.za,inbound,001,World,1 +gmail.com,vodafone-ip.de,inbound,001,World,1 +gmail.com,vodafone.pt,inbound,001,World,0.999563 +gmail.com,vodafonedsl.it,inbound,001,World,0.999006 +gmail.com,vtr.net,inbound,001,World,0.999072 +gmail.com,wanadoo.fr,inbound,001,World,0.999753 +gmail.com,websitewelcome.com,inbound,001,World,1 +gmail.com,wideopenwest.com,inbound,001,World,0.999729 +gmail.com,windstream.net,inbound,001,World,0.951847 +gmail.com,yahoo.{...},inbound,001,World,0.999147 +gmail.com,ziggo.nl,inbound,001,World,1 +gmail.com,zoothost.com,inbound,001,World,0.033858 +gmo.jp,gmo-media.jp,inbound,001,World,0 +gmoes.jp,gmoes.jp,inbound,001,World,0 +gmsend.com,gmsend.com,inbound,001,World,0 +gmt.ne.jp,gmt.ne.jp,inbound,001,World,0 +gmx.de,gmx.net,inbound,001,World,1 +gmx.de,gmx.net,outbound,001,World,1 +gmx.net,gmx.net,inbound,001,World,1 +gnavi.co.jp,gnavi.co.jp,inbound,001,World,0.002441 +go.com,starwave.com,inbound,001,World,0.006276 +go4worldbusiness.com,go4worldbusiness.com,inbound,001,World,1 +goalunited.org,ccmdcampaigns.net,inbound,001,World,0 +gob.ar,gob.ar,inbound,001,World,0.159673 +gob.ec,gob.ec,inbound,001,World,0.618006 +godaddy.com,secureserver.net,inbound,001,World,0 +godtubemail.com,godtubemail.com,inbound,001,World,0 +godvinemail.com,godvinemail.com,inbound,001,World,0 +gog.com,gog.com,inbound,001,World,0 +gogecapital.com,rsys1.com,inbound,001,World,0 +gogroopie.com,gogroopie.com,inbound,001,World,0.000283 +gohappy.com.tw,gohappy.com.tw,inbound,001,World,0.000104 +goldenbrands.gr,goldenbrands.gr,inbound,001,World,1 +goldenline.pl,goldenline.pl,inbound,001,World,1 +goldenopsafelist.com,zoothost.com,inbound,001,World,0 +goldstar.com,goldstar.com,inbound,001,World,1 +golfmnb.com,golfmnb.com,inbound,001,World,0 +golfnow.com,email-golfnow.com,inbound,001,World,0 +gomaji.com,gomaji.com,inbound,001,World,0 +goodgame.com,emsmtp.com,inbound,001,World,0 +goodlife.pt,emv8.com,inbound,001,World,0 +google.com,postini.com,inbound,001,World,0.703706 +googlegroups.com,postini.com,inbound,001,World,0.674075 +googlemail.com,t-ipconnect.de,inbound,001,World,0.999957 +gop.com,gop.com,inbound,001,World,0 +gopusamedia.com,gopusamedia.com,inbound,001,World,0 +govdelivery.com,govdelivery.com,inbound,001,World,0 +govdelivery.com,postini.com,inbound,001,World,0.089736 +governmentjobs.com,governmentjobs.com,inbound,001,World,0 +gpmailer.com.br,parperfeito.com,inbound,001,World,0 +grabone-mail-ie.com,grabone-mail-ie.com,inbound,001,World,0 +grabone-mail.com,grabone-mail.com,inbound,001,World,0 +grassrootsaction.com,grassfire.net,inbound,001,World,0 +gratka.pl,gratka.pl,inbound,001,World,0 +greatergood.com,greatergood.com,inbound,001,World,0 +gree.jp,gree.jp,inbound,001,World,0 +grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,001,World,0 +groopdealz.com,groopdealz.com,inbound,001,World,1 +groupalia.es,groupalia.es,inbound,001,World,0 +groupalia.it,groupalia.it,inbound,001,World,0 +groupon.jp,data-hotel.net,inbound,001,World,1e-06 +groupon.{...},chtah.net,inbound,001,World,0 +groupon.{...},groupon.{...},inbound,001,World,0.989844 +groupon.{...},postini.com,inbound,001,World,0.887392 +grouponmail.{...},grouponmail.{...},inbound,001,World,0 +grubhubmail.com,grubhubmail.com,inbound,001,World,0 +grupanya.com,euromsg.net,inbound,001,World,0 +grupos.com.br,grupos.com.br,inbound,001,World,0 +gtbank.com,gtbank.com,inbound,001,World,0.056121 +guess.ca,guess.com,inbound,001,World,0.000807 +guess.com,guess.com,inbound,001,World,0.003805 +guessfactory.com,guess.com,inbound,001,World,0.00164 +gumtree.com,marktplaats.nl,inbound,001,World,0 +gumtree.com.au,kijiji.com,inbound,001,World,0 +gunosy.com,gunosy.com,inbound,001,World,0.998682 +guruin.info,guru.net.in,inbound,001,World,1 +gurunavi.jp,gurunavi.jp,inbound,001,World,0 +gustazos.com,cityoferta.com,inbound,001,World,1 +gymglish.com,gymglish.com,inbound,001,World,0.000788 +habitaclia.com,splio.es,inbound,001,World,0.951406 +hallmark.com,hallmark.com,inbound,001,World,0 +hannaandersson.com,hannaandersson.com,inbound,001,World,0.013533 +harborfreightemail.com,harborfreightemail.com,inbound,001,World,0 +harristeetermail.com,harristeetermail.com,inbound,001,World,0.99673 +harvard.edu,harvard.edu,inbound,001,World,0.1751 +haskell.org,haskell.org,inbound,001,World,0.000703 +hautelook.com,hautelook.com,inbound,001,World,0.000459 +hayneedle.com,hayneedle.com,inbound,001,World,0 +hazteoir.org,hazteoir.org,inbound,001,World,0.31478 +hdfcbank.com,powerelay.com,inbound,001,World,1 +hdfcbank.net,powerelay.com,inbound,001,World,1 +hdfcbank.net,quickvmail.com,inbound,001,World,0 +helpareporter.net,helpareporter.com,inbound,001,World,0 +hepsiburada.com,euromsg.net,inbound,001,World,0 +herbalifemail.com,herbalifemail.com,inbound,001,World,0 +herculist.com,herculist.com,inbound,001,World,0 +heteml.jp,heteml.jp,inbound,001,World,0.950766 +hgtv.com,hgtv.com,inbound,001,World,0 +hh.ru,hh.ru,inbound,001,World,0.996236 +hhgreggemail.com,hhgreggemail.com,inbound,001,World,0 +hilton.com,hiltonemail.com,inbound,001,World,0 +hinet.net,hinet.net,inbound,001,World,0.007093 +hinet.net,hinet.net,outbound,001,World,0.00565 +hipchat.com,hipchat.com,inbound,001,World,1 +hipmunk.com,hipmunk.com,inbound,001,World,0 +hispavista.com,hispavista.com,inbound,001,World,0 +hln.be,persgroep-ops.net,inbound,001,World,0 +hm-f.jp,hm-f.jp,inbound,001,World,0 +hobsonsmail.com,hobsonsmail.com,inbound,001,World,0 +hollister-email.com,abercrombie-email.com,inbound,001,World,0 +home.ne.jp,zaq.ne.jp,inbound,001,World,9e-06 +homeaway.com,haspf.com,inbound,001,World,0 +homebaselife.com,ec-cluster.com,inbound,001,World,0 +homechoice.co.za,homechoice.co.za,inbound,001,World,0 +homedecorators.com,homedecorators.com,inbound,001,World,0 +homedepot.com,homedepot.com,inbound,001,World,1 +homedepotemail.com,homedepotemail.com,inbound,001,World,0 +honto.jp,honto.jp,inbound,001,World,0 +hootsuite.com,hootsuite.com,inbound,001,World,1 +horchowemail.com,horchowemail.com,inbound,001,World,0 +horoscope.com,center.com,inbound,001,World,2e-06 +hostelworld.com,bronto.com,inbound,001,World,0 +hostgator.com,hostgator.com,inbound,001,World,0.976131 +hostgator.com,websitewelcome.com,inbound,001,World,0.999822 +hotel.de,emp-mail.de,inbound,001,World,0 +hotels.com,hotels.com,inbound,001,World,0 +hotelurbano.com.br,allin.com.br,inbound,001,World,0 +hotmail.{...},hotmail.{...},inbound,001,World,0.999968 +hotmail.{...},hotmail.{...},outbound,001,World,1 +hotmail.{...},postini.com,inbound,001,World,0.837967 +hotornot.com,monopost.com,inbound,001,World,0.99472 +hotschedules.com,hotschedules.com,inbound,001,World,0 +hotspotmailer.com,hotspotmailer.com,inbound,001,World,1 +hotukdeals.com,hotukdeals.com,inbound,001,World,1 +hotwire.com,hotwire.com,inbound,001,World,0 +house.gov,house.gov,inbound,001,World,0.999966 +houseoffraser.co.uk,houseoffraser.co.uk,inbound,001,World,0 +houzz.com,houzz.com,inbound,001,World,1 +hp.com,hp.com,inbound,001,World,0.202841 +hpnotifier.nl,hpnotifier.nl,inbound,001,World,0 +hsbc.co.in,hsbc.com.hk,inbound,001,World,1 +hsbc.com.hk,hsbc.com.hk,inbound,001,World,1 +hsn.com,hsn.com,inbound,001,World,0 +htcampusmailer.com,eccluster.com,inbound,001,World,0 +hubspot.com,hubspot.com,inbound,001,World,1 +huinforma.com.br,huinforma.com.br,inbound,001,World,0 +hulumail.com,hulumail.com,inbound,001,World,0 +hungry-girl.com,hungry-girl.com,inbound,001,World,0 +hungryhouse.co.uk,mxmfb.com,inbound,001,World,0 +huntington.com,huntington.com,inbound,001,World,0.987351 +i-part.com.tw,i-part.com.tw,inbound,001,World,0.001865 +i-say.com,ipsos-interactive.com,inbound,001,World,1 +iamlgnd2.com,iamlgnd2.com,inbound,001,World,1 +ibm.com,ibm.com,inbound,001,World,0.94413 +ibps.in,sify.net,inbound,001,World,0 +ibpsorg.org,sify.net,inbound,001,World,0 +ibsys.com,ibsys.com,inbound,001,World,9e-05 +icbc.com.ar,clickexperts.net,inbound,001,World,0 +icbc.com.ar,standardbank.com.ar,inbound,001,World,0 +icelandmail.co.uk,emsg-live.co.uk,inbound,001,World,0 +icicibank.com,icicibank.com,inbound,001,World,0.009835 +icicisecurities.com,icicibank.com,inbound,001,World,0.000803 +icims.com,icims.com,inbound,001,World,0.999939 +icloud.com,apple.com,inbound,001,World,1 +icloud.com,icloud.com,outbound,001,World,1 +icloud.com,mac.com,inbound,001,World,1 +icloud.com,me.com,inbound,001,World,0.999995 +icors.org,lsoft.us,inbound,001,World,0 +icpbounce.com,icpbounce.com,inbound,001,World,0 +idc.email,nmsrv.com,inbound,001,World,1 +idealista.com,idealista.com,inbound,001,World,0.001627 +ideascost.com,ramcorp.in,inbound,001,World,0 +idgconnect-resources.com,idgconnect-resources.com,inbound,001,World,0 +ieee.org,ieee.org,inbound,001,World,0.999912 +ifttt.com,ifttt.com,inbound,001,World,1 +ig.com.br,ig.com.br,inbound,001,World,0 +ig.com.br,ig.com.br,outbound,001,World,0 +ign.com,ign.com,inbound,001,World,0 +ignitionsender.com,ignitionsender.com,inbound,001,World,0 +igot-mails.com,zoothost.com,inbound,001,World,0 +iheart.com,iheart.com,inbound,001,World,0 +iimjobs.com,iimjobs.com,inbound,001,World,1 +ikmultimedianews.com,ikmultimedianews.com,inbound,001,World,0.993319 +illinois.edu,illinois.edu,inbound,001,World,0.866481 +imageshost.ca,imageshost.ca,inbound,001,World,0 +imakenews.net,imakenews.com,inbound,001,World,0 +imi.ne.jp,lifemedia.jp,inbound,001,World,0 +immobilienscout24.de,immobilienscout24.de,inbound,001,World,1 +imo.im,imo.im,inbound,001,World,1 +imodules.com,imodules.com,inbound,001,World,0 +imvu.com,imvu.com,inbound,001,World,7e-06 +in-boxpays.com,in-boxpays.com,inbound,001,World,0 +inboxair.com,inboxair.com,inbound,001,World,0 +inboxdollars.com,inboxdollars.com,inbound,001,World,0 +inboxfirst.com,inboxfirst.com,inbound,001,World,0 +inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,001,World,0.999877 +inboxpays.com,inboxpays.com,inbound,001,World,0 +inboxpounds.co.uk,inboxpounds.co.uk,inbound,001,World,0 +indeed.com,indeed.com,inbound,001,World,0.000121 +indeedemail.com,indeedemail.com,inbound,001,World,0 +independentlivingbullion.com,independentlivingbullion.com,inbound,001,World,0 +indiamart.com,indiamart.com,inbound,001,World,1 +indiaproperty.com,indiaproperty.com,inbound,001,World,0 +indiatimes.com,speakingtree.in,inbound,001,World,0 +indiatimeshop.com,sendpal.in,inbound,001,World,0 +indieroyale.com,desura.com,inbound,001,World,1 +infibeam.com,eccluster.com,inbound,001,World,0 +infobradesco.com.br,infobradesco.com.br,inbound,001,World,0 +infoempleo.com,infoempleo.com,inbound,001,World,0 +infojobs.com.br,anuntis.com,inbound,001,World,0 +infojobs.it,infojobs.it,inbound,001,World,0 +infojobs.net,infojobs.net,inbound,001,World,0 +infomoney.com.br,infomoney.com.br,inbound,001,World,1 +infopanel.jp,mailds.jp,inbound,001,World,0 +infopraca.pl,careesma.com,inbound,001,World,0 +informz.net,informz.net,inbound,001,World,0 +infos-micromania.com,infos-micromania.com,inbound,001,World,0 +infosephora.com,splio.com,inbound,001,World,0.962167 +infoworld.com,infoworld.com,inbound,001,World,0 +infradead.org,infradead.org,inbound,001,World,1 +infusionmail.com,infusionmail.com,inbound,001,World,0 +ingdirect.es,ingdirect.es,inbound,001,World,1 +inman.com,inman.com,inbound,001,World,0 +inmotionhosting.com,inmotionhosting.com,inbound,001,World,0.990051 +innovyx.net,innovyx.net,inbound,001,World,0 +ino.com,ino.com,inbound,001,World,0.999981 +insidehook.com,sailthru.com,inbound,001,World,0 +instagram.com,facebook.com,inbound,001,World,1 +instantprofitlist.com,screenshotads.com,inbound,001,World,0 +intage.co.jp,intage.co.jp,inbound,001,World,0.00557 +inter-chat.com,inter-chat.com,inbound,001,World,0 +interac.ca,certapay.com,inbound,001,World,0 +interactivebrokers.com,interactivebrokers.com,inbound,001,World,1 +interactiverealtyservices.com,interactiverealtyservices.com,inbound,001,World,0 +intercom.io,mailgun.info,inbound,001,World,1 +interdatesa.com,fagms.net,inbound,001,World,0 +interealty.net,interealty.net,inbound,001,World,1 +internations.org,internations.org,inbound,001,World,1 +interweave.com,interweave.com,inbound,001,World,0 +interwell.gr,interwell.gr,inbound,001,World,0 +intliv2.net,internationalliving.com,inbound,001,World,0 +intuit.com,intuit.com,inbound,001,World,0.89114 +invalidemail.com,taleo.net,inbound,001,World,1 +investopedia.com,vclk.net,inbound,001,World,0.999999 +investorplace.com,investorplace.com,inbound,001,World,0.995093 +inx1and1.de,1and1.com,inbound,001,World,1 +inxserver.com,inxserver.de,inbound,001,World,0.952863 +inxserver.de,inxserver.de,inbound,001,World,0.980838 +ipcmedia.co.uk,ipcmedia.co.uk,inbound,001,World,0 +iqelite.com,iqelite.com,inbound,001,World,0 +irctcshopping.com,chtah.net,inbound,001,World,0 +iridium.com,iridium.com,inbound,001,World,0.997882 +isbank.com.tr,isbank.com.tr,inbound,001,World,0.009655 +isendservice.com.br,isendservice.com.br,inbound,001,World,0 +itau-unibanco.com.br,itau.com.br,inbound,001,World,0 +itms.in.ua,itms.in.ua,inbound,001,World,0 +itsmyascent.com,itsmyascent.com,inbound,001,World,0 +ittoolbox.com,ittoolbox.com,inbound,001,World,0 +ittoolbox.com,toolbox.com,inbound,001,World,0 +itunes.com,apple.com,inbound,001,World,0.082921 +itwhitepapers.com,itwhitepapers.com,inbound,001,World,0 +iwantoneofthose.com,thehut.com,inbound,001,World,0 +ixs1.net,ixs1.net,inbound,001,World,0.005131 +jackwills.com,jackwills.com,inbound,001,World,0 +jalag.de,jalag.de,inbound,001,World,1 +jane.com,jane.com,inbound,001,World,1 +jango.com,jango.com,inbound,001,World,0 +jared.com,jared.com,inbound,001,World,0 +jcity.com,jcity.com,inbound,001,World,0 +jcpenney.com,jcpenney.com,inbound,001,World,9e-06 +jdate.com,postdirect.com,inbound,001,World,0 +jeevansathi.com,jeevansathi.com,inbound,001,World,0 +jetprivilege.com,jetprivilege.com,inbound,001,World,0 +jetsetter.com,smartertravelmedia.com,inbound,001,World,0.041362 +jeuxvideo.com,jeuxvideo.com,inbound,001,World,0.148921 +jeweloscoemail.com,email-mywebgrocer2.com,inbound,001,World,0 +jibjab.com,storybots.com,inbound,001,World,0 +jira.com,uc-inf.net,inbound,001,World,1 +jiscmail.ac.uk,lsoft.se,inbound,001,World,0 +joann-mail.com,joann-mail.com,inbound,001,World,0 +jobinsider.com,jobinsider.com,inbound,001,World,0 +jobinthailand.com,jobinthailand.com,inbound,001,World,1 +jobisjob.com,jobisjob.com,inbound,001,World,0 +jobmaster.co.il,jobmaster1.co.il,inbound,001,World,0 +jobomas.com,jobomas.com,inbound,001,World,1 +jobrapidoalert.com,jobrapidoalert.com,inbound,001,World,0 +jobs2web.com,ondemand.com,inbound,001,World,1 +jobscentral.com.sg,mailgun.net,inbound,001,World,1 +jobsdbalert.co.id,jobsdbalert.co.id,inbound,001,World,0 +jobsdbalert.com,jobsdbalert.com,inbound,001,World,0 +jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,001,World,0 +jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,001,World,0 +jobserve.com,jobserve.com,inbound,001,World,0 +jobsindubai.com,jobsindubai.ca,inbound,001,World,0 +jobsite.co.uk,jobsite.co.uk,inbound,001,World,0.000509 +jobson.com,jobsonmail.com,inbound,001,World,0 +jobsradar.com,jobsradar.com,inbound,001,World,0 +jobstreet.com,jobstreet.com,inbound,001,World,0 +jockeycomfort.com,jockeycomfort.com,inbound,001,World,0 +johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,001,World,0 +jomashop.com,lstrk.net,inbound,001,World,1 +joobmailer.com,joobmailer.com,inbound,001,World,0 +josbank.com,josbank.com,inbound,001,World,0 +joshin.co.jp,joshin.co.jp,inbound,001,World,8e-06 +jossandmain.com,jossandmain.com,inbound,001,World,0 +jpcycles.com,jpcycles.com,inbound,001,World,0.001716 +jtv.com,jtv.com,inbound,001,World,0 +jumia.com.ng,fagms.de,inbound,001,World,0 +jungleerummy.com,jungleerummy.com,inbound,001,World,1 +juno.com,untd.com,inbound,001,World,0 +juno.com,untd.com,outbound,001,World,0 +jusbrasil.com.br,jusbrasil.com.br,inbound,001,World,0 +just-eat.co.uk,ec-cluster.com,inbound,001,World,0 +justclick.ru,justclick.ru,inbound,001,World,0 +justdial.com,iaires.com,inbound,001,World,0 +justdial.com,mailurja.com,inbound,001,World,0 +justfab.com,bronto.com,inbound,001,World,0 +justfab.fr,bronto.com,inbound,001,World,0 +k1speed.com,k1speed.com,inbound,001,World,1 +kagoya.net,kagoya.net,inbound,001,World,0.013097 +kalunga.com.br,kalunga.com.br,inbound,001,World,0 +karvy.com,karvy.com,inbound,001,World,0.045186 +kasikornbank.com,kasikornbank.com,inbound,001,World,0 +kaskusnetworks.com,kaskus.com,inbound,001,World,0 +kay.com,kay.com,inbound,001,World,0 +keek.com,keek.com,inbound,001,World,1 +kernel.org,kernel.org,inbound,001,World,0 +kgbdeals.co.uk,email1-kgbdeals.com,inbound,001,World,0 +kgstores.com,kgstores.com,inbound,001,World,0 +kiabi.com,dms-02.net,inbound,001,World,0 +kickstarter.com,kickstarter.com,inbound,001,World,1 +kidsfootlocker.com,footlocker.com,inbound,001,World,0 +kijiji.ca,kijiji.com,inbound,001,World,0 +kik.com,kik.com,inbound,001,World,1 +kimblegroup.com,kimblegroup.com,inbound,001,World,1 +kintera.com,kintera.com,inbound,001,World,0 +kismia.com,kismia.com,inbound,001,World,1 +kiwari.com,kiwari.com,inbound,001,World,9e-06 +klaviyomail.com,klaviyomail.com,inbound,001,World,1 +kliksa.net,euromsg.net,inbound,001,World,0 +kliktoday.com,kliktoday.com,inbound,001,World,0 +klm-mail.com,klm-mail.com,inbound,001,World,0 +klove.com,emfbroadcasting.com,inbound,001,World,0 +kohls.com,kohls.com,inbound,001,World,0 +komando.com,komando.com,inbound,001,World,0 +kongregate.com,kongregate.com,inbound,001,World,0 +kotak.com,kotak.com,inbound,001,World,0.115651 +kp.org,kp.org,inbound,001,World,0.999975 +krogermail.com,bigfootinteractive.com,inbound,001,World,0 +krs.bz,tricorn.net,inbound,001,World,0 +kubra.com,kubra.com,inbound,001,World,1.8e-05 +kundenserver.de,kundenserver.de,inbound,001,World,1 +kvbmail.com,kvbmail.com,inbound,001,World,0 +la-meteo-mail.fr,splio.com,inbound,001,World,1 +laaptuemail.com,laaptuemail.com,inbound,001,World,0 +lakewoodchurch.com,lakewoodchurch.com,inbound,001,World,0 +lancers.jp,lancers.jp,inbound,001,World,0 +landmarketingmailer.com,zoothost.com,inbound,001,World,0 +landofnod.com,landofnod.com,inbound,001,World,0.002525 +landsend.com,email-landsend.com,inbound,001,World,0 +landsend.com,postdirect.com,inbound,001,World,0 +languagepod101.com,eclient10.com,inbound,001,World,0 +languagepod101.com,eddlvr.com,inbound,001,World,0 +languagepod101.com,ednwsltr3.com,inbound,001,World,0 +languagepod101.com,ednwsltr8.com,inbound,001,World,0 +languagepod101.com,emaildirect.net,inbound,001,World,0 +laposte.net,laposte.net,inbound,001,World,0.271722 +laposte.net,laposte.net,outbound,001,World,0 +laptuinvite.com,laptuinvite.com,inbound,001,World,0 +laredoute.fr,laredoute.fr,inbound,001,World,0 +lasenza.com,lasenza.com,inbound,001,World,0 +lastcallemail.com,lastcallemail.com,inbound,001,World,0 +lastminute.com,lastminute.com,inbound,001,World,0.00831 +laterooms.com,laterooms.com,inbound,001,World,0.014746 +latimes.com,latimes.com,inbound,001,World,0 +lauraashley.com,lauraashley.com,inbound,001,World,0 +lazerhits.com,lazerhits.com,inbound,001,World,1 +leadercontato.com.br,leadercontato.com.br,inbound,001,World,0 +leboncoin.fr,leboncoin.fr,inbound,001,World,0 +lefigaro.fr,splio.com,inbound,001,World,1 +leftlanesports.com,auspient.com,inbound,001,World,0.00705 +leftlanesports.com,leftlanesports.com,inbound,001,World,0 +legalshieldassociate.com,legalshield.com,inbound,001,World,0 +lelong.my,lelong.com.my,inbound,001,World,1 +lelong.my,lelong.net.my,inbound,001,World,1 +lemonde.fr,lemonde.fr,inbound,001,World,0.000111 +leparisien.fr,leparisien.fr,inbound,001,World,0 +lexico.com,lexico.com,inbound,001,World,0 +lexpress.fr,bp06.net,inbound,001,World,0 +libero.it,libero.it,inbound,001,World,0.00024 +libero.it,libero.it,outbound,001,World,0 +life360.com,life360.com,inbound,001,World,1 +lifecare-news.com,email-lifecare.com,inbound,001,World,0 +lifecooler.com,1-hostingservice.com,inbound,001,World,0 +lifemiles.com,bigfootinteractive.com,inbound,001,World,0 +lifescript.com,ilinkmd.com,inbound,001,World,0 +lindenlab.com,lindenlab.com,inbound,001,World,0.999444 +line.me,naver.com,inbound,001,World,1 +line6.com,line6.com,inbound,001,World,2.7e-05 +linkedin.com,linkedin.com,inbound,001,World,0.998882 +linkedin.com,postini.com,inbound,001,World,0.835872 +linkedin.com,yahoo.{...},inbound,001,World,0.999927 +linkshare.com,linksynergy.com,inbound,001,World,0 +liquidation.com,liquidation.com,inbound,001,World,6e-06 +listadventure.com,adlabsinc.com,inbound,001,World,0 +listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,001,World,0.00023 +listeneremail.net,listeneremail.net,inbound,001,World,0 +listia.com,listia.com,inbound,001,World,1 +listjoe.com,adlabsinc.com,inbound,001,World,0 +listnerds.com,listnerds.com,inbound,001,World,0 +listreturn.com,zoothost.com,inbound,001,World,0 +listserve.com,listserve.com,inbound,001,World,0 +listvolta.com,listvolta.com,inbound,001,World,0 +listwire.com,listwire.com,inbound,001,World,0 +litres.ru,litres.ru,inbound,001,World,1 +live.{...},hotmail.{...},inbound,001,World,0.999954 +live.{...},hotmail.{...},outbound,001,World,1 +livedoor.com,livedoor.com,inbound,001,World,0 +livefyre.com,andbit.net,inbound,001,World,1 +livejournal.com,livejournal.com,inbound,001,World,0 +livemailservice.com,livemailservice.com,inbound,001,World,0 +livenation.com,exacttarget.com,inbound,001,World,0 +livescribe.com,bronto.com,inbound,001,World,0 +livingsocial.com,livingsocial.com,inbound,001,World,0 +livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,001,World,0 +lmlmgv.com.br,gvarev.com.br,inbound,001,World,0 +localhires.com,localhires.com,inbound,001,World,1 +loccitane.com,neolane.net,inbound,001,World,0 +loft.com,anntaylor.com,inbound,001,World,0 +logentries.com,logentries.com,inbound,001,World,0 +logitech.com,dvsops.com,inbound,001,World,0 +logmein.com,logmein.com,inbound,001,World,0.031467 +lojasmarisa.com.br,lojasmarisa.com.br,inbound,001,World,0 +lolsolos.com,ultimateadsites.net,inbound,001,World,0.999996 +lombardipublishing.com,lombardipublishing.com,inbound,001,World,0 +lonelywifehookup.com,iverificationsystems.com,inbound,001,World,0 +lookout.com,lookout.com,inbound,001,World,1 +lordandtaylor.com,lordandtaylor.com,inbound,001,World,0 +loveaholics.com,ropot.net,inbound,001,World,0 +lovelywholesale.com,lovelywholesale.com,inbound,001,World,1 +loveplanet.ru,pochta.ru,inbound,001,World,0.640035 +lowcostholidays.co.uk,communicatoremail.com,inbound,001,World,0 +lrsmail.com,lrsmail.com,inbound,001,World,0 +lsi.com,postini.com,inbound,001,World,0.981121 +lt02.net,listrak.com,inbound,001,World,1 +lt02.net,lstrk.net,inbound,001,World,1 +ltdcommodities.com,ltdcomm.net,inbound,001,World,0 +lua.org,pepperfish.net,inbound,001,World,1 +luckymag.com,mkt4500.com,inbound,001,World,0 +ludokados.com,ludokado.com,inbound,001,World,0 +lulu.com,bronto.com,inbound,001,World,0 +lulus.com,lstrk.net,inbound,001,World,1 +lumosity.com,lumosity.com,inbound,001,World,1 +luxa.jp,luxa.jp,inbound,001,World,0 +lynxmail.in,iaires.com,inbound,001,World,0 +lyris.net,lyris.net,inbound,001,World,0 +lyst.com,lyst.com,inbound,001,World,1 +m1e.net,m1e.net,inbound,001,World,0.000342 +m3.com,m3.com,inbound,001,World,0 +mac.com,icloud.com,outbound,001,World,1 +mac.com,mac.com,inbound,001,World,1 +maccosmetics.com,esteelauder.com,inbound,001,World,0 +macromill.com,macromill.com,inbound,001,World,1e-06 +macupdate.com,mailgun.info,inbound,001,World,1 +macys.com,macys.com,inbound,001,World,0 +madmels.info,ultimateadsites.net,inbound,001,World,1 +madmimi.com,madmimi.com,inbound,001,World,0 +mag2.com,tandem-m.com,inbound,001,World,0 +magicbricks.com,tbsl.in,inbound,001,World,0 +magicjack.com,magicjack.com,inbound,001,World,1 +magix.net,magix.net,inbound,001,World,0.673176 +magnetdev.com,magnetmail.net,inbound,001,World,0 +mail-backcountry.com,email-bcmarketing.com,inbound,001,World,0 +mail-boss.com,mail-boss.com,inbound,001,World,0 +mail-cdiscount.com,mail-cdiscount.com,inbound,001,World,0 +mail-mbank.pl,mail-mbank.pl,inbound,001,World,0 +mail-route.com,mail-route.com,inbound,001,World,0 +mail-thestreet.com,mail-thestreet.com,inbound,001,World,0 +mail.mil,mail.mil,inbound,001,World,0 +mail.ru,mail.ru,inbound,001,World,0.987506 +mail.ru,mail.ru,outbound,001,World,0.00674 +mailaccurate.com,mgenie.in,inbound,001,World,0 +mailchimp.com,mailchimp.com,inbound,001,World,0.967415 +maileclipse.com,emce2.in,inbound,001,World,0 +mailengine1.com,mailengine1.com,inbound,001,World,0 +mailer-service.de,mailer-service.de,inbound,001,World,4.9e-05 +mailer4u.in,elabs10.com,inbound,001,World,0 +mailersend.com,mailersend.com,inbound,001,World,0 +mailfacil.com.br,md02.com,inbound,001,World,0 +mailfeast.com,mgenie.in,inbound,001,World,0 +mailgun.org,mailgun.info,inbound,001,World,1 +mailgun.org,mailgun.net,inbound,001,World,1 +mailgun.org,mailgun.us,inbound,001,World,1 +mailing-list.it,mailing-list.it,inbound,001,World,0 +mailingathome.net,mailingathome.net,inbound,001,World,0.999995 +mailjayde.com,mailjayde.com,inbound,001,World,0 +mailjet.com,mailjet.com,inbound,001,World,0.436025 +mailmachine1050.com,mailmachine1050.com,inbound,001,World,0 +mailmailmail.net,mailmailmail.net,inbound,001,World,0 +mailoct.in,tcmailer14.in,inbound,001,World,0 +mailoct1.in,mailoct1.in,inbound,001,World,0 +mailoct1.in,myntramail2.in,inbound,001,World,0 +mailorama.fr,mailorama.fr,inbound,001,World,0 +mailplus.nl,brightbase.net,inbound,001,World,1 +mailpost.in,iaires.com,inbound,001,World,0 +mailpv.net,pvmailer.net,inbound,001,World,1 +mailquant.com,iaires.com,inbound,001,World,0 +mailsend1.com,mailsend6.com,inbound,001,World,0 +mailsender.com.br,mailsender.com.br,inbound,001,World,0 +maisonsdumonde.com,bp06.net,inbound,001,World,0 +makro.nl,srv2.de,inbound,001,World,0.88256 +manager.com.br,manager.com.br,inbound,001,World,0 +mandrillapp.com,backpage.com,inbound,001,World,1 +mandrillapp.com,mandrillapp.com,inbound,001,World,1 +mandrillapp.com,mcsignup.com,inbound,001,World,1 +mandrillapp.com,myjobhelperalerts.com,inbound,001,World,1 +mango.com,emstechnology2.net,inbound,001,World,0 +manipal.edu,iaires.com,inbound,001,World,0 +manta.com,exacttarget.com,inbound,001,World,0 +mapfre.com,emv5.com,inbound,001,World,0 +mar0.net,mar0.net,inbound,001,World,0.976409 +marcustheatres.com,movio.co,inbound,001,World,0 +markandgraham.com,markandgraham.com,inbound,001,World,0 +markavip.com,markavip.com,inbound,001,World,0 +marketer-safelist.com,jsalfianmarketing.com,inbound,001,World,1 +marketinghq.net,elabs8.com,inbound,001,World,0 +marketingprofs.com,marketingprofs.com,inbound,001,World,0.004412 +marketingstudio.com,marketingstudio.com,inbound,001,World,0 +marksandspencer.com,marksandspencer.com,inbound,001,World,0 +marktplaats.nl,marktplaats.nl,inbound,001,World,0 +marlboro.com,marlboro.com,inbound,001,World,0 +maropost.com,biotrustnews.com,inbound,001,World,0 +maropost.com,mailing-truthaboutabs.com,inbound,001,World,0 +maropost.com,maropost.com,inbound,001,World,0 +maropost.com,mp2200.com,inbound,001,World,0 +maropost.com,mp2201.com,inbound,001,World,0 +maropost.com,survivallife.com,inbound,001,World,0 +marykay.com,marykay.com,inbound,001,World,0 +masivapp.com,masivapp.com,inbound,001,World,1 +massageenvyclinics.com,massageenvyclinics.com,inbound,001,World,0 +masterbase.com,masterbase.com,inbound,001,World,0 +mastercard-email.com,mastercard-email.com,inbound,001,World,0 +match.com,match.com,inbound,001,World,0 +matchwereld.nl,matchwereld.nl,inbound,001,World,0 +mate1.net,mate1.net,inbound,001,World,0 +matrixemailer.com,matrixemailer.com,inbound,001,World,0 +maxpark.com,gidepark.ru,inbound,001,World,1 +mbga.jp,mbga.jp,inbound,001,World,0 +mbna.co.uk,ec-cluster.com,inbound,001,World,0 +mbounces.com,emdbms.com,inbound,001,World,0 +mbstrm.com,mobilestorm.com,inbound,001,World,0 +mcafee.com,mcafee.com,inbound,001,World,0.963535 +mcarthurglen.com,mcarthurglen.com,inbound,001,World,0 +mcdlv.net,mcdlv.net,inbound,001,World,0 +mcdlv.net,postini.com,inbound,001,World,0.082163 +mckinsey.com,bigfootinteractive.com,inbound,001,World,0 +mcsv.net,mcsv.net,inbound,001,World,0 +mcsv.net,postini.com,inbound,001,World,0.101883 +mdirector.com,mdrctr.com,inbound,001,World,0 +mdlinx.com,mdlinx.com,inbound,001,World,0 +me.com,icloud.com,outbound,001,World,1 +me.com,mac.com,inbound,001,World,1 +mec.gov.br,mec.gov.br,inbound,001,World,0 +mecumauction.com,mecumauction.com,inbound,001,World,0 +medallia.com,medallia.com,inbound,001,World,0.999682 +mediabistro.com,iworld.com,inbound,001,World,0.006428 +mediapost.com,mediapost.com,inbound,001,World,0 +medium.com,messagebus.com,inbound,001,World,0 +medpagetoday.com,wc09.net,inbound,001,World,0 +medscape.com,medscape.com,inbound,001,World,0 +meetic.com,meetic.com,inbound,001,World,0 +meetmemail.com,meetmemail.com,inbound,001,World,0 +meetup.com,meetup.com,inbound,001,World,5e-06 +megasenders.com,megasenders.com,inbound,001,World,0.07662 +melaleuca.com,melaleuca.com,inbound,001,World,0.003493 +memberdealsusa.com,memberdealsusa.com,inbound,001,World,0 +menswearhouse.com,menswearhouse.com,inbound,001,World,0 +mequedouno.com,mequedouno.com,inbound,001,World,0 +mercadojobs.com,sendgrid.net,inbound,001,World,1 +mercadolibre.com,mercadolibre.com,inbound,001,World,0 +mercadolivre.com,mercadolibre.com,inbound,001,World,0 +merceworld.com,merceworld.com,inbound,001,World,0.996738 +mercola.com,mercola.com,inbound,001,World,0.000369 +merodea.me,sendgrid.net,inbound,001,World,1 +messagegears.net,messagegears.net,inbound,001,World,0 +met-art.com,hydentra.com,inbound,001,World,1 +metro.co.in,srv2.de,inbound,001,World,0.966947 +metrodeal.com,fagms.de,inbound,001,World,0 +mgmresorts.com,mgmresorts.com,inbound,001,World,0 +mgo.com,bronto.com,inbound,001,World,0 +michaels.com,chtah.net,inbound,001,World,0 +michaels.com,michaels.com,inbound,001,World,0 +microcentermedia.com,bfi0.com,inbound,001,World,0 +microsoft.com,hotmail.{...},inbound,001,World,1 +microsoft.com,msn.com,inbound,001,World,1 +microsoftemail.com,microsoftemail.com,inbound,001,World,0 +microsoftemail.com,microsoftstoreemail.com,inbound,001,World,0 +midnightsunsafelist.com,zoothost.com,inbound,001,World,0 +mightydeals.co.uk,mightydeals.co.uk,inbound,001,World,0 +mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,001,World,0 +milfaholic.com,iverificationsystems.com,inbound,001,World,0 +miltnews.com,miltnews.com,inbound,001,World,0 +mindbodyonline.com,mindbodyonline.com,inbound,001,World,1 +mindfieldonline.com,mindfieldonline.com,inbound,001,World,0 +mindmoviesmail.com,mindmoviesmail.com,inbound,001,World,0.004239 +mindvalleymail3.com,mindvalleymail3.com,inbound,001,World,0 +minhavida.com.br,minhavida.com.br,inbound,001,World,0 +mint.com,mint.com,inbound,001,World,0 +minted.com,messagelabs.com,inbound,001,World,0.999669 +mirtesen.ru,mtml.ru,inbound,001,World,0 +missselfridge.com,wallis-fashion.com,inbound,001,World,0 +mistersafelist.com,zoothost.com,inbound,001,World,0 +mit.edu,mit.edu,inbound,001,World,0.868705 +mitula.net,mitula.org,inbound,001,World,0 +mitula.org,mitula.org,inbound,001,World,0 +mixcloudmail.com,mixcloudmail.com,inbound,001,World,0.995482 +mixi.jp,mixi.jp,inbound,001,World,0 +mjinn.com,mailurja.com,inbound,001,World,0 +mkt015.com,mkt015.com,inbound,001,World,0 +mkt022.com,mkt022.com,inbound,001,World,0 +mkt063.com,mkt063.com,inbound,001,World,0 +mkt1136.com,mkt1136.com,inbound,001,World,0 +mkt1985.com,fmlinks.net,inbound,001,World,0 +mkt2010.com,mkt2010.com,inbound,001,World,0 +mkt2106.com,mkt2106.com,inbound,001,World,0 +mkt2170.com,mkt2170.com,inbound,001,World,0 +mkt2181.com,mkt2181.com,inbound,001,World,0 +mkt2615.com,mkt2615.com,inbound,001,World,0 +mkt2813.com,mkt2813.com,inbound,001,World,0 +mkt2944.com,mkt2944.com,inbound,001,World,0 +mkt3134.com,mkt3134.com,inbound,001,World,0 +mkt3142.com,mkt3142.com,inbound,001,World,0 +mkt3156.com,mkt3156.com,inbound,001,World,0 +mkt3203.com,mkt3203.com,inbound,001,World,0 +mkt346.com,mkt346.com,inbound,001,World,0 +mkt3544.com,mkt3544.com,inbound,001,World,0 +mkt3622.com,mkt3622.com,inbound,001,World,0 +mkt3682.com,mkt3682.com,inbound,001,World,0 +mkt3690.com,mkt3690.com,inbound,001,World,0 +mkt3695.com,mkt3695.com,inbound,001,World,0 +mkt3804.com,mkt3804.com,inbound,001,World,0 +mkt3815.com,mkt3815.com,inbound,001,World,0 +mkt3952.com,xoom.com,inbound,001,World,0 +mkt4355.com,mkt4355.com,inbound,001,World,0 +mkt4364.com,mkt4364.com,inbound,001,World,0 +mkt459.com,mkt459.com,inbound,001,World,0 +mkt4701.com,mkt4701.com,inbound,001,World,0 +mkt4728.com,mkt4728.com,inbound,001,World,0 +mkt4731.com,mkt4731.com,inbound,001,World,0 +mkt4738.com,mkt4738.com,inbound,001,World,0 +mkt5071.com,mkt5071.com,inbound,001,World,0 +mkt5098.com,mkt5098.com,inbound,001,World,0 +mkt5131.com,mkt5131.com,inbound,001,World,0 +mkt5144.com,mkt5144.com,inbound,001,World,0 +mkt5144.com,mkt5980.com,inbound,001,World,0 +mkt5144.com,mkt5981.com,inbound,001,World,0 +mkt5181.com,mkt5181.com,inbound,001,World,0 +mkt5269.com,mkt5269.com,inbound,001,World,0 +mkt529.com,mkt529.com,inbound,001,World,0 +mkt5297.com,mkt5297.com,inbound,001,World,0 +mkt5297.com,mkt5309.com,inbound,001,World,0 +mkt5371.com,mkt5371.com,inbound,001,World,0 +mkt5806.com,mkt5806.com,inbound,001,World,0 +mkt5934.com,mkt5934.com,inbound,001,World,0 +mkt5937.com,mkt5937.com,inbound,001,World,0 +mkt5970.com,mkt5970.com,inbound,001,World,0 +mkt6100.com,mkt6098.com,inbound,001,World,0 +mkt6276.com,mkt6276.com,inbound,001,World,0 +mkt6323.com,mkt6323.com,inbound,001,World,0 +mkt746.com,mkt746.com,inbound,001,World,0 +mkt824.com,mkt869.com,inbound,001,World,0 +mktdillards.com,mktdillards.com,inbound,001,World,0 +mktid10.com,1-hostingservice.com,inbound,001,World,0 +mktomail.com,mktdns.com,inbound,001,World,0 +mktomail.com,mktomail.com,inbound,001,World,0 +mktomail.com,mktroute.com,inbound,001,World,0 +ml.com,bankofamerica.com,inbound,001,World,1 +mlgns.com,mlgns.com,inbound,001,World,0 +mlgnserv.com,mlgnserv.com,inbound,001,World,0 +mlsend.com,mlsend.com,inbound,001,World,0 +mlsend2.com,mlsend2.com,inbound,001,World,0 +mlssoccer.com,mlssoccer.com,inbound,001,World,0 +mmaco.net,mmaco.net,inbound,001,World,0.999998 +mmagic.jp,mmagic.jp,inbound,001,World,0.000531 +mmks.it,mail-maker.it,inbound,001,World,0 +mmorpg.com,mmorpg.com,inbound,001,World,0.002979 +mmsecure.nl,donenad.nl,inbound,001,World,0 +mo1send.com,mo1send.com,inbound,001,World,0 +mobile01.com,mobile01.com,inbound,001,World,0 +mobly.com.br,mobly.com.br,inbound,001,World,0 +mocospace.com,mocospace.com,inbound,001,World,0 +modellsemail.com,n-email.net,inbound,001,World,0 +modnakasta.ua,emv5.com,inbound,001,World,0 +monex.co.jp,monex.co.jp,inbound,001,World,0.019424 +moneycontrol.com,active18.com,inbound,001,World,0 +moneyforward.com,moneyforward.com,inbound,001,World,0 +moneymorning.com,moneymappress.com,inbound,001,World,0 +moneysupermarketmail.com,moneysupermarketmail.com,inbound,001,World,0 +monipla.jp,aainc.co.jp,inbound,001,World,0 +monografias.com,elistas.net,inbound,001,World,0 +monster.co.in,monster.co.in,inbound,001,World,0 +monster.com,monster.com,inbound,001,World,0.000231 +monster.com,tmpw.net,inbound,001,World,0.000503 +monsterindia.com,monster.co.in,inbound,001,World,0 +moon-ray.com,moon-ray.com,inbound,001,World,0.001892 +mooply.co,mailendo.com,inbound,001,World,0 +mooresclothing.com,mooresclothing.com,inbound,001,World,0 +morhipo.com,euromsg.net,inbound,001,World,0 +morningstar.net,morningstar.net,inbound,001,World,0 +mothercaregroup.com,neolane.net,inbound,001,World,0 +motosnap.com,motosnap.com,inbound,001,World,0.994795 +moveon.org,moveon.org,inbound,001,World,0.99577 +moviestarplanet.com,moviestarplanet.com,inbound,001,World,0 +mozilla.org,mozilla.com,inbound,001,World,0.633237 +mpme.jp,mpme.jp,inbound,001,World,0 +mpse.jp,emsaqua.jp,inbound,001,World,0 +mpse.jp,emsbeige.jp,inbound,001,World,0 +mpse.jp,emsbrown.jp,inbound,001,World,0 +mpse.jp,emscyan.jp,inbound,001,World,0 +mpse.jp,emsgold.jp,inbound,001,World,0 +mpse.jp,emslime.jp,inbound,001,World,0 +mpse.jp,emsnavy.jp,inbound,001,World,0 +mpse.jp,emspink.jp,inbound,001,World,0 +mpse.jp,emssnow.jp,inbound,001,World,0 +mpse.jp,mpme.jp,inbound,001,World,0 +mpse.jp,yahoo.co.jp,inbound,001,World,0 +mpsnd.ch,agenceweb.net,inbound,001,World,0 +mrc.org,msgfocus.com,inbound,001,World,0 +mrmlsmatrix.com,mrmlsmatrix.com,inbound,001,World,0 +ms.com,ms.com,inbound,001,World,1 +ms00.net,ms00.net,inbound,001,World,0 +msdp1.com,msdp1.com,inbound,001,World,0 +msgfocus.com,msgfocus.com,inbound,001,World,0.011658 +msn.com,hotmail.{...},inbound,001,World,0.999964 +msn.com,hotmail.{...},outbound,001,World,1 +mta.info,ealert.com,inbound,001,World,0 +mtasv.net,mtasv.net,inbound,001,World,0.999999 +musiciansfriend.com,musiciansfriend.com,inbound,001,World,0 +musicnotes-alerts.com,mybuys.com,inbound,001,World,0 +mustanglist.com,mustanglist.com,inbound,001,World,0 +mxmfb.com,mxmfb.com,inbound,001,World,0 +mycheapoair.com,mycheapoair.com,inbound,001,World,0.957931 +mycolorscreen.com,mta4.net,inbound,001,World,0 +mydailymoment.biz,mydailymoment.biz,inbound,001,World,0 +mydailymoment.info,mydailymoment.info,inbound,001,World,0 +mydailymoment.net,mydailymoment.net,inbound,001,World,0 +mydailymoment.us,mydailymoment.us,inbound,001,World,0 +myfedloan.org,aessuccess.org,inbound,001,World,0.328917 +myfitnesspal.com,messagebus.com,inbound,001,World,0 +myfxbook.com,myfxbook.com,inbound,001,World,0 +mygreatlakes.org,glhec.org,inbound,001,World,0.000247 +mygroupon.co.th,grouponmail.{...},inbound,001,World,0 +myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,001,World,0 +myheritage.com,myheritage.com,inbound,001,World,0 +myideeli.com,myideeli.com,inbound,001,World,0 +mymeijer.com,mymeijer.com,inbound,001,World,0 +mymms.com,fagms.de,inbound,001,World,0 +mynavi.jp,mynavi.jp,inbound,001,World,3.5e-05 +myngp.com,ngpweb.com,inbound,001,World,0 +myntramail.com,iaires.com,inbound,001,World,0 +myntramail.com,myntramail.com,inbound,001,World,0 +myntramails.in,icubes.in,inbound,001,World,0 +myoutlets.in,trustmailer.com,inbound,001,World,0 +myperfectsale.com,myperfectsale.com,inbound,001,World,0.999988 +mypoints.com,mypoints.com,inbound,001,World,0 +myprotein.com,thehut.com,inbound,001,World,0 +mysafelistmailer.com,mysafelistmailer.com,inbound,001,World,0.00033 +mysale.my,mysale.my,inbound,001,World,0 +mysale.ph,mysale.ph,inbound,001,World,0 +mysmartprice.com,itzwow.com,inbound,001,World,1 +mysupermarket.co.uk,mysupermarket.co.uk,inbound,001,World,0 +mysurvey.com,mysurvey.com,inbound,001,World,0 +mysurvey.eu,mysurvey.com,inbound,001,World,0 +myvegas.com,myvegas.com,inbound,001,World,1 +myzamanamail.com,myzamanamail.com,inbound,001,World,0 +n-email.net,n-email.net,inbound,001,World,0 +n-email1.net,n-email1.net,inbound,001,World,0 +n-email4.net,n-email4.net,inbound,001,World,0 +naaptoldeals.com,eccluster.com,inbound,001,World,0 +namorico.me,namorico.me,inbound,001,World,1 +nanomail.com.br,araie.com.br,inbound,001,World,1 +napitipp.hu,napitipp.hu,inbound,001,World,0 +nasa.gov,nasa.gov,inbound,001,World,0.082443 +nascar.com,nascar.com,inbound,001,World,0 +nastygal.com,bronto.com,inbound,001,World,0 +nasza-klasa.pl,nasza-klasa.pl,inbound,001,World,0 +nationalexpress.com,nationalexpress.com,inbound,001,World,0 +nationbuilder.com,nationbuilder.com,inbound,001,World,1 +nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,001,World,0 +nature.com,nature.com,inbound,001,World,0 +naukri.com,naukri.com,inbound,001,World,0.002488 +nauta.cu,etecsa.net,inbound,001,World,0 +naver.com,naver.com,inbound,001,World,1 +naver.com,naver.com,outbound,001,World,1 +navy.mil,navy.mil,inbound,001,World,0.152107 +nba.com,nba.com,inbound,001,World,0.005828 +nbaa.org,nbaa.org,inbound,001,World,0 +ncl.com,ncl.com,inbound,001,World,0 +neimanmarcusemail.com,neimanmarcusemail.com,inbound,001,World,0 +nend.net,postini.com,inbound,001,World,0 +neolane.net,neolane.net,inbound,001,World,0.01682 +nesinemail.com,euromsg.net,inbound,001,World,0 +net-a-porter.com,net-a-porter.com,inbound,001,World,0 +net-empregos.com,net-empregos.com,inbound,001,World,0 +net-survey.jp,net-survey.jp,inbound,001,World,0 +netatlantic.com,netatlantic.com,inbound,001,World,0.001085 +netbk.co.jp,netbk.co.jp,inbound,001,World,0.010994 +netcommunity1.com,blackbaud.com,inbound,001,World,0 +netflix.com,amazonses.com,inbound,001,World,0.999999 +netflix.com,netflix.com,inbound,001,World,1 +netlogmail.com,netlogmail.com,inbound,001,World,0 +netopia.pt,netopia.pt,inbound,001,World,0.017294 +netprosoftmail.com,netprosoftmail.com,inbound,001,World,0 +netshoes.com.br,netshoes.com.br,inbound,001,World,0.078768 +netsuite.com,netsuite.com,inbound,001,World,0.57723 +networkworld.com,networkworld.com,inbound,001,World,0 +newegg.com,newegg.com,inbound,001,World,1e-06 +newgrounds.com,newgrounds.com,inbound,001,World,0 +newmarkethealth.com,newmarkethealth.com,inbound,001,World,0 +newrelic.com,sendlabs.com,inbound,001,World,0 +news-h5g.com,news-h5g.com,inbound,001,World,0 +newsletter-verychic.com,splio.es,inbound,001,World,0.9804 +newsmax.com,newsmax.com,inbound,001,World,0.004105 +newspaperdirect.com,newspaperdirect.com,inbound,001,World,0.000177 +newyorktimesinfo.com,newyorktimesinfo.com,inbound,001,World,0 +nexcess.net,nexcess.net,inbound,001,World,0.214803 +next-engine.org,next-engine.org,inbound,001,World,1 +nextdoor.com,mailgun.info,inbound,001,World,1 +nextdoor.com,mailgun.net,inbound,001,World,1 +nextdoor.com,nextdoor.com,inbound,001,World,1 +nfl.com,bfi0.com,inbound,001,World,0 +nflshop.com,nflshop.com,inbound,001,World,0 +nhs.jobs,nhscareersjobs.co.uk,inbound,001,World,0 +nic.in,relayout.nic.in,inbound,001,World,0 +nicovideo.jp,nicovideo.jp,inbound,001,World,0 +nieuwsblad.be,vummail.be,inbound,001,World,0 +nifty.com,nifty.com,inbound,001,World,0.762068 +nih.gov,nih.gov,inbound,001,World,8.8e-05 +nike.com,nike.com,inbound,001,World,0 +nikkei.com,nikkei.co.jp,inbound,001,World,0 +nikkeibp.co.jp,nikkeibp.co.jp,inbound,001,World,4.9e-05 +ninewestmail.com,ninewestmail.com,inbound,001,World,0 +ning.com,ning.com,inbound,001,World,0 +nissen.jp,nissen.jp,inbound,001,World,0 +nixle.com,nixle.com,inbound,001,World,0 +nl00.net,netline.com,inbound,001,World,5e-06 +nl00.net,nl00.net,inbound,001,World,0 +nmp1.com,nmp1.net,inbound,001,World,0 +nokia.com,nokia.com,inbound,001,World,0.001256 +nongnu.org,gnu.org,inbound,001,World,1 +nordstrom.com,taleo.net,inbound,001,World,1 +nortonfromsymantec.com,rsys1.com,inbound,001,World,0 +nos.pt,netcabo.pt,inbound,001,World,6.3e-05 +noticiasaominuto.com,ccmdcampaigns.net,inbound,001,World,0 +noticiasaominuto.com,noticiasaominuto.com,inbound,001,World,0 +novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,001,World,0 +npr.org,npr.org,inbound,001,World,0 +nrholding.net,nrholding.net,inbound,001,World,0 +ns.nl,tripolis.com,inbound,001,World,0 +nsandi.com,mxmfb.com,inbound,001,World,0 +numbersusa.com,numbersusa.com,inbound,001,World,0 +nyandcompany.com,nyandcompany.com,inbound,001,World,0 +nytimes.com,nytimes.com,inbound,001,World,0.00472 +nzsale.co.nz,nzsale.co.nz,inbound,001,World,0 +oakley.com,oakley.com,inbound,001,World,0.000904 +ocadomail.com,ocadomail.com,inbound,001,World,0 +ocmail1.in,tcmail.in,inbound,001,World,0 +ocmail14.in,tcmailer5.in,inbound,001,World,0 +ocmail22.in,tcmailer15.in,inbound,001,World,0 +ocmail22.in,tcmailer4.in,inbound,001,World,0 +ocmail40.in,tcmailer15.in,inbound,001,World,0 +ocmail40.in,tcmailer4.in,inbound,001,World,0 +ocn.ad.jp,ocn.ad.jp,inbound,001,World,0 +ocn.ne.jp,ocn.ad.jp,inbound,001,World,0 +ocnmail.in,ocmail6.in,inbound,001,World,0 +ocnmail.in,tcmail3.in,inbound,001,World,0 +odisseias.com,emv4.net,inbound,001,World,0 +odnoklassniki.ru,odnoklassniki.ru,inbound,001,World,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,001,World,0 +ofertasefacil.com.br,ofertasefacil.com.br,inbound,001,World,0 +ofertix.com,ofertix.com,inbound,001,World,0 +ofertop.pe,icommarketing.com,inbound,001,World,0 +offers.com,offers.com,inbound,001,World,1 +offerum.com,cccampaigns.com,inbound,001,World,0 +offerum.com,ccemails.com,inbound,001,World,0 +officedepot.com,officedepot.com,inbound,001,World,0.012483 +officemax.com,officemax.com,inbound,001,World,0 +officemax.com,officemaxworkplace.com,inbound,001,World,0 +ofsys.com,bulletin-metro.ca,inbound,001,World,0 +oknotify2.com,oknotify2.com,inbound,001,World,0 +oldnavy.ca,oldnavy.ca,inbound,001,World,0 +oldnavy.com,oldnavy.com,inbound,001,World,0 +olx.pt,fixeads.com,inbound,001,World,0 +olympiaedge.net,olympiaedge.net,inbound,001,World,0 +omahasteaks.com,omahasteaks.com,inbound,001,World,0.006139 +oneindia.in,infimail.com,inbound,001,World,0 +oneindia.in,mailurja.com,inbound,001,World,0 +onekingslane.com,onekingslane.com,inbound,001,World,0 +onepatriotplace.com,britecast.com,inbound,001,World,0 +onestopplus.com,neolane.net,inbound,001,World,0 +onetravelspecials.com,onetravelspecials.com,inbound,001,World,0.365386 +online.com,cnet.com,inbound,001,World,0 +onlive.com,ipost.com,inbound,001,World,0 +onmicrosoft.com,outlook.com,inbound,001,World,1 +onthecitymail.org,onthecitymail.org,inbound,001,World,1 +oo155.com,bsftransmit7.com,inbound,001,World,0 +oo155.com,oo155.com,inbound,001,World,0 +openstack.org,openstack.org,inbound,001,World,0.996884 +openstackmail.com,infimail.com,inbound,001,World,0 +opentable.com,opentable.com,inbound,001,World,0.000662 +opinionoutpost.com,opinionoutpost.com,inbound,001,World,0 +opinionsquare.com,opinionsquare.com,inbound,001,World,0 +oprah.com,oprah.com,inbound,001,World,0 +opticsplanet.com,opticsplanet.com,inbound,001,World,0 +optimusmail.in,iaires.com,inbound,001,World,0 +optonline.net,cv.net,inbound,001,World,0 +optonline.net,optonline.net,outbound,001,World,0 +orange.fr,orange.fr,inbound,001,World,0 +orange.fr,orange.fr,outbound,001,World,0 +orderscatalog.com,orderscatalog.com,inbound,001,World,0 +oriental-trading.com,oriental-trading.com,inbound,001,World,0 +oroscopofree.com,adsender.us,inbound,001,World,0 +oroscopofree.com,oroscopofree.com,inbound,001,World,0 +orsay.com,emp-mail.de,inbound,001,World,0 +os-email.com,os-email.com,inbound,001,World,0 +oshkoshbgosh.com,oshkoshbgosh.com,inbound,001,World,6e-06 +osu.edu,outlook.com,inbound,001,World,1 +otto.de,eccluster.com,inbound,001,World,1 +ouffer.com,ouffer.com,inbound,001,World,0.01022 +ourtime.com,seniorpeoplemeet.com,inbound,001,World,0 +outback.com,outback.com,inbound,001,World,0 +outlook.com,hotmail.{...},inbound,001,World,0.999929 +outlook.com,hotmail.{...},outbound,001,World,1 +outspot.be,teneo.be,inbound,001,World,0 +outspot.nl,teneo.be,inbound,001,World,0 +ovenmail.com,iaires.com,inbound,001,World,0 +overnightprints.com,chtah.net,inbound,001,World,0 +overstock.com,overstock.com,inbound,001,World,0 +ovh.net,ovh.net,inbound,001,World,0.207527 +ovuline.com,ovuline.com,inbound,001,World,1 +oxfam.org.uk,msgfocus.com,inbound,001,World,0 +ozsale.com.au,ozsale.com.au,inbound,001,World,0.000192 +p-world.co.jp,p-world.co.jp,inbound,001,World,0 +pagoda.com,zales.com,inbound,001,World,0 +pagseguro.com.br,uol.com.br,inbound,001,World,0 +pair.com,pair.com,inbound,001,World,0.880779 +palmscasinoresort.com,palmscasinoresort.com,inbound,001,World,0 +pampers.com,bfi0.com,inbound,001,World,0 +panasonic.jp,panasonic.jp,inbound,001,World,0 +pandaresearch.com,pandaresearch.com,inbound,001,World,0 +pandora.com,pandora.com,inbound,001,World,1 +pandora.net,pandora.net,inbound,001,World,0.999666 +panelplace.com,smtp.com,inbound,001,World,0 +panerabreadnews.com,panerabreadnews.com,inbound,001,World,0 +pantaloondirect.net,iaires.com,inbound,001,World,0 +papajohns-specials.com,papajohns-specials.com,inbound,001,World,0 +paradisepublishers.com,paradisepublishers.com,inbound,001,World,0.999626 +parents.com,meredith.com,inbound,001,World,0 +parkmobileglobal.com,parkmobile.us,inbound,001,World,0 +path.com,path.com,inbound,001,World,1 +patriotupdate.com,inboxfirst.com,inbound,001,World,0 +payback.de,artegic.net,inbound,001,World,0.999986 +payback.in,chtah.net,inbound,001,World,0 +payback.in,eccluster.com,inbound,001,World,0 +payback.in,ecm-cluster.com,inbound,001,World,0 +payback.in,ecmcluster.com,inbound,001,World,0 +paypal.co.uk,paypal.com,inbound,001,World,1 +paypal.com,paypal.com,inbound,001,World,0.608856 +paypal.com.au,paypal.com,inbound,001,World,1 +paypal.de,paypal.com,inbound,001,World,1 +paytm.com,paytm.com,inbound,001,World,1 +pbteen.com,pbteen.com,inbound,001,World,0 +pccomponentes.com,pccomponentes.com,inbound,001,World,0 +pch.com,ed10.com,inbound,001,World,0 +pchfrontpage.com,ed10.com,inbound,001,World,0 +pchlotto.com,ed10.com,inbound,001,World,0 +pchplayandwin.com,ed10.com,inbound,001,World,0 +pchsearch.com,ed10.com,inbound,001,World,0 +pcmag.com,ittoolbox.com,inbound,001,World,0 +pcworld.com,pcworld.com,inbound,001,World,0 +pd25.com,pd25.com,inbound,001,World,1 +pd25.com,pd27.com,inbound,001,World,1 +peanuthome.info,adopterc.info,inbound,001,World,0 +peanuthome.info,aguitytr.info,inbound,001,World,0 +peanuthome.info,bevest.info,inbound,001,World,0 +peanuthome.info,bluester.info,inbound,001,World,0 +peanuthome.info,burror.info,inbound,001,World,0 +peanuthome.info,bursion.info,inbound,001,World,0 +peanuthome.info,cantexi.info,inbound,001,World,0 +peanuthome.info,caserhi.info,inbound,001,World,0 +peanuthome.info,celect.info,inbound,001,World,0 +peanuthome.info,chintone.info,inbound,001,World,0 +peanuthome.info,citery.info,inbound,001,World,0 +peanuthome.info,cleathal.info,inbound,001,World,0 +peanuthome.info,coherentrequittal.info,inbound,001,World,0 +peanuthome.info,colicom.info,inbound,001,World,0 +peanuthome.info,complec.info,inbound,001,World,0 +peanuthome.info,cyprinoidkaiserdom.info,inbound,001,World,0 +peanuthome.info,deciarc.info,inbound,001,World,0 +peanuthome.info,declaws.info,inbound,001,World,0 +peanuthome.info,dewest.info,inbound,001,World,0 +peanuthome.info,epconce.info,inbound,001,World,0 +peanuthome.info,fnotec.info,inbound,001,World,0 +peanuthome.info,folkswor.info,inbound,001,World,0 +peanuthome.info,forepert.info,inbound,001,World,0 +peanuthome.info,gurgaro.info,inbound,001,World,0 +peanuthome.info,heallyps.info,inbound,001,World,0 +peanuthome.info,holeph.info,inbound,001,World,0 +peanuthome.info,homewor.info,inbound,001,World,0 +peanuthome.info,hydroni.info,inbound,001,World,0 +peanuthome.info,ingenbu.info,inbound,001,World,0 +peanuthome.info,kinklybotaurus.info,inbound,001,World,0 +peanuthome.info,ninetiethwhiffet.info,inbound,001,World,0 +peanuthome.info,unglibshudder.info,inbound,001,World,0 +peanutwebmaster.info,adcrent.info,inbound,001,World,0 +peanutwebmaster.info,addmiel.info,inbound,001,World,0 +peanutwebmaster.info,agilhe.info,inbound,001,World,0 +peanutwebmaster.info,allegap.info,inbound,001,World,0 +peanutwebmaster.info,andvore.info,inbound,001,World,0 +peanutwebmaster.info,angogl.info,inbound,001,World,0 +peanutwebmaster.info,animass.info,inbound,001,World,0 +peanutwebmaster.info,arettery.info,inbound,001,World,0 +peanutwebmaster.info,aribank.info,inbound,001,World,0 +peanutwebmaster.info,arkefoc.info,inbound,001,World,0 +peanutwebmaster.info,avenog.info,inbound,001,World,0 +peanutwebmaster.info,barrave.info,inbound,001,World,0 +peanutwebmaster.info,bindowmo.info,inbound,001,World,0 +peanutwebmaster.info,bitravit.info,inbound,001,World,0 +peanutwebmaster.info,borsand.info,inbound,001,World,0 +peanutwebmaster.info,branti.info,inbound,001,World,0 +peanutwebmaster.info,breaserp.info,inbound,001,World,0 +peanutwebmaster.info,bredogly.info,inbound,001,World,0 +peanutwebmaster.info,briantra.info,inbound,001,World,0 +peanutwebmaster.info,bridea.info,inbound,001,World,0 +peanutwebmaster.info,carial.info,inbound,001,World,0 +peanutwebmaster.info,castac.info,inbound,001,World,0 +peanutwebmaster.info,chedoner.info,inbound,001,World,0 +peanutwebmaster.info,chiquent.info,inbound,001,World,0 +peanutwebmaster.info,cinchoi.info,inbound,001,World,0 +peanutwebmaster.info,cliate.info,inbound,001,World,0 +peanutwebmaster.info,cognn.info,inbound,001,World,0 +peanutwebsite.info,abjibbin.info,inbound,001,World,0 +peanutwebsite.info,audiette.info,inbound,001,World,0 +peanutwebsite.info,blancer.info,inbound,001,World,0 +peanutwebsite.info,bluester.info,inbound,001,World,0 +peanutwebsite.info,burror.info,inbound,001,World,0 +peanutwebsite.info,bursion.info,inbound,001,World,0 +peanutwebsite.info,caserhi.info,inbound,001,World,0 +peanutwebsite.info,celect.info,inbound,001,World,0 +peanutwebsite.info,cleathal.info,inbound,001,World,0 +peanutwebsite.info,coherentrequittal.info,inbound,001,World,0 +peanutwebsite.info,complec.info,inbound,001,World,0 +peanutwebsite.info,condost.info,inbound,001,World,0 +peanutwebsite.info,cyprinoidkaiserdom.info,inbound,001,World,0 +peanutwebsite.info,deciarc.info,inbound,001,World,0 +peanutwebsite.info,declaws.info,inbound,001,World,0 +peanutwebsite.info,epconce.info,inbound,001,World,0 +peanutwebsite.info,ferrayer.info,inbound,001,World,0 +peanutwebsite.info,fnotec.info,inbound,001,World,0 +peanutwebsite.info,folkswor.info,inbound,001,World,0 +peanutwebsite.info,forepert.info,inbound,001,World,0 +peanutwebsite.info,gurgaro.info,inbound,001,World,0 +peanutwebsite.info,holeph.info,inbound,001,World,0 +peanutwebsite.info,homewor.info,inbound,001,World,0 +peanutwebsite.info,hydroni.info,inbound,001,World,0 +peanutwebsite.info,ingenbu.info,inbound,001,World,0 +peanutwebsite.info,kinklybotaurus.info,inbound,001,World,0 +peanutwebsite.info,ninetiethwhiffet.info,inbound,001,World,0 +pearlsofwealth.com,pearlsofwealth.com,inbound,001,World,1 +peartreegreetings.com,rexcraft.com,inbound,001,World,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,001,World,0 +pennwell.com,pennwell.com,inbound,001,World,0 +pepabo.com,pepabo.com,inbound,001,World,0 +pepboys.com,pepboys.com,inbound,001,World,0 +peperoni.de,peperoni.de,inbound,001,World,1 +pepperfry.com,epidm.net,inbound,001,World,0 +perfectpriceindia.com,infimail.com,inbound,001,World,0 +perfectworld.com,perfectworld.com,inbound,001,World,0.000162 +perfora.net,perfora.net,inbound,001,World,0.920896 +permissionresearch.com,permissionresearch.com,inbound,001,World,0 +personalliberty.com,personalliberty.com,inbound,001,World,1 +personare.com.br,personare.com.br,inbound,001,World,0 +peytz.dk,peytz.dk,inbound,001,World,4e-06 +pga.com,pga.com,inbound,001,World,0 +pge.com,pge.com,inbound,001,World,0.149792 +pgeveryday.com,bfi0.com,inbound,001,World,0 +philosophy.com,philosophy.com,inbound,001,World,0 +phoenix.edu,phoenix.edu,inbound,001,World,0 +photobox.com,photobox.com,inbound,001,World,0 +photoprintit.com,photoprintit.com,inbound,001,World,0 +phpclasses.org,phpclasses.org,inbound,001,World,0 +phsmtpbox.com,phsmtpbox.com,inbound,001,World,0 +pia.jp,pia.jp,inbound,001,World,0 +pinger.com,pinger.com,inbound,001,World,0 +pinterest.com,pinterest.com,inbound,001,World,1 +pivotaltracker.com,pivotaltracker.com,inbound,001,World,1 +pixable.com,pixable.com,inbound,001,World,1 +pixum.com,pixum.com,inbound,001,World,0 +pizzahut.com,quikorder.com,inbound,001,World,0 +pizzahutoffers.com,pizzahutoffers.com,inbound,001,World,0 +placedestendances.com,placedestendances.com,inbound,001,World,0 +plaisio.gr,fagms.de,inbound,001,World,0 +planeo.com,planeo.com,inbound,001,World,0 +planeo.pt,planeo.pt,inbound,001,World,0 +playstation.com,playstation.com,inbound,001,World,0 +playstationmail.net,playstationmail.net,inbound,001,World,0 +playtika.com,emv8.com,inbound,001,World,0 +plexapp.com,plex.tv,inbound,001,World,1 +plumdistrict.com,plumdistrict.com,inbound,001,World,1 +pmailus.com,patrontechnology.com,inbound,001,World,0 +pnc.com,messagelabs.com,inbound,001,World,0.997194 +pnetweb.co.za,hosting.co.za,inbound,001,World,0 +pnetweb.co.za,salesnet.co.za,inbound,001,World,0 +pobox.com,pobox.com,inbound,001,World,0.975372 +pof.com,plentyoffish.co.uk,inbound,001,World,0 +pogo.com,pogo.com,inbound,001,World,0 +pointtown.com,gmo-media.jp,inbound,001,World,0 +poinx.com,poinx.com,inbound,001,World,0 +pokerstars.com,pokerstars.eu,inbound,001,World,0 +pokerstars.eu,pokerstars.eu,inbound,001,World,0 +pokupon.by,mailersend.com,inbound,001,World,0 +pokupon.ua,mailersend.com,inbound,001,World,0 +politicoemail.com,politicoemail.com,inbound,001,World,0 +polyvore.com,polyvore.com,inbound,001,World,1 +pontofrio.com.br,emv8.com,inbound,001,World,0 +popsugar.com,popsugar.com,inbound,001,World,0 +postcardfromhell.com,cyberthugs.com,inbound,001,World,1 +postgresql.org,postgresql.org,inbound,001,World,0.996805 +potterybarn.com,potterybarn.com,inbound,001,World,0 +potterybarnkids.com,potterybarnkids.com,inbound,001,World,0 +praca.pl,praca.pl,inbound,001,World,1 +pracuj.pl,pracuj.pl,inbound,001,World,0 +preferredpetclub.com,preferredpetclub.com,inbound,001,World,0 +presslaff.net,dat-e-baseonline.com,inbound,001,World,0 +pressmartmail.com,pressmartmail.com,inbound,001,World,0 +priceline.com,priceline.com,inbound,001,World,1 +princess.com,princess.com,inbound,001,World,0 +printvenue.com,fagms.de,inbound,001,World,0 +priorityoneemail.com,priorityoneemail.com,inbound,001,World,0 +privalia.com,privalia.com,inbound,001,World,3.94913202027326e-07 +private-elist.com,private-elist.com,inbound,001,World,0 +privoscite.si,privoscite.si,inbound,001,World,0 +profitcenteronline.com,groupdealtools.com,inbound,001,World,1 +progressive.com,progressive.com,inbound,001,World,0.897291 +progressiveagent.com,progressive.com,inbound,001,World,1 +promedmail.org,childrenshospital.org,inbound,001,World,1 +promod-news.fr,promod-news.com,inbound,001,World,0 +propertysolutions.com,propertysolutions.com,inbound,001,World,1 +prospectgeysercoop.com,prospectgeysercoop.com,inbound,001,World,1 +protopmail.com,protopmail.com,inbound,001,World,0 +providesupport.com,providesupport.com,inbound,001,World,0.000103 +proxyvote.com,adp-ics.com,inbound,001,World,0.908635 +psu.edu,psu.edu,inbound,001,World,0.073843 +pttplc.com,pttgrp.com,inbound,001,World,0 +publicators.com,publicators.com,inbound,001,World,0 +publix.com,publix.com,inbound,001,World,0.002567 +pucp.edu.pe,pucp.edu.pe,inbound,001,World,0.225541 +puffinmailer.com,zoothost.com,inbound,001,World,0 +pur3.net,pur3.net,inbound,001,World,0.001111 +purewow.com,purewow.com,inbound,001,World,0 +puritan.com,email-nbtyinc.com,inbound,001,World,0 +purlsmail.com,purlsmail.com,inbound,001,World,0 +pxsmail.com,pxsmail.com,inbound,001,World,0 +python.org,python.org,inbound,001,World,1 +q.com,synacor.com,inbound,001,World,0.999729 +qemailserver.com,qemailserver.com,inbound,001,World,0 +qoinpro.com,qoinpro.com,inbound,001,World,1 +qoo10.jp,qoo10.jp,inbound,001,World,2e-06 +qoo10.sg,qoo10.co.id,inbound,001,World,1.9e-05 +qoo10.sg,qoo10.com,inbound,001,World,0 +qoo10.sg,qoo10.my,inbound,001,World,2.2e-05 +qoo10.sg,qoo10.sg,inbound,001,World,8e-06 +qq.com,qq.com,inbound,001,World,0.999978 +qq.com,qq.com,outbound,001,World,1 +qtropnews.com,qtropnews.com,inbound,001,World,1.7e-05 +qualicorp.com.br,qualicorp.com.br,inbound,001,World,1 +quality.net.ua,quality.net.ua,inbound,001,World,0 +qualitysafelist.com,zoothost.com,inbound,001,World,0 +queopinas.com,confirmit.com,inbound,001,World,1 +quickbooks.com,intuit.com,inbound,001,World,0.99908 +quickrewards.net,quickrewards.net,inbound,001,World,0 +quikr.com,quikr.com,inbound,001,World,0.026599 +quinstreet.com,neoquin.com,inbound,001,World,0 +quora.com,quora.com,inbound,001,World,1 +quoramail.com,quoramail.com,inbound,001,World,1 +qvcemail.com,qvcemail.com,inbound,001,World,0 +r-project.org,ethz.ch,inbound,001,World,0.99999 +r51.it,musvc.com,inbound,001,World,0.092498 +r52.it,musvc.com,inbound,001,World,0.000221 +r57.it,musvc.com,inbound,001,World,0.139425 +r67.it,musvc.com,inbound,001,World,0.199845 +r70.it,musvc.com,inbound,001,World,0.311983 +rabota.ua,rabota.ua,inbound,001,World,0 +rackroom-email.com,rackroom-email.com,inbound,001,World,0 +radarsystems.net,radarsystems.net,inbound,001,World,1 +radiantretailapps.com,radiantretailapps.com,inbound,001,World,0 +radioshack.com,radioshack.com,inbound,001,World,0 +railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,001,World,0 +rakuten.co.jp,rakuten.co.jp,inbound,001,World,0 +rakuten.co.jp,shareee.jp,inbound,001,World,0 +rakuten.co.jp,yahoo.co.jp,inbound,001,World,0 +rakuten.com,rakuten.com,inbound,001,World,0 +rakuten.ne.jp,rakuten.co.jp,inbound,001,World,0 +rambler.ru,rambler.ru,inbound,001,World,0.08173 +randomhouse.com,randomhouse.com,inbound,001,World,0 +rapattoni.com,rapmls.com,inbound,001,World,0 +ratedpeople.com,ratedpeople.com,inbound,001,World,0.000979 +rax.ru,rax.ru,inbound,001,World,0.000258 +razerzone.com,chtah.net,inbound,001,World,0 +rbc.com,rbc.com,inbound,001,World,0.001897 +rdio.com,rdio.com,inbound,001,World,1 +reactiveadz.com,downlinebuilderdirect.com,inbound,001,World,0 +realage-mail.com,postdirect.com,inbound,001,World,0 +realestate.com.au,realestate.com.au,inbound,001,World,0 +realtor.org,realtor.org,inbound,001,World,0 +realtytrac.com,realtytrac.com,inbound,001,World,0.001681 +realus.co.jp,realus.co.jp,inbound,001,World,0 +recipe.com,meredith.com,inbound,001,World,0 +recochoku.jp,recochoku.jp,inbound,001,World,0 +recruit.net,recruit.net,inbound,001,World,0 +redbox.com,exacttarget.com,inbound,001,World,0 +redbox.com,redbox.com,inbound,001,World,4e-06 +redcross.org.uk,redcross.org.uk,inbound,001,World,0 +redfin.com,redfin.com,inbound,001,World,0 +rediffmail.com,akadns.net,outbound,001,World,0 +rediffmail.com,rediffmail.com,inbound,001,World,0 +redtri.com,redtri.com,inbound,001,World,0 +reebokusnews.com,reebokusnews.com,inbound,001,World,0 +reebonz.com,ed10.com,inbound,001,World,0 +reebonz.com,reebonz.com,inbound,001,World,0.857615 +reed.co.uk,reed.co.uk,inbound,001,World,0 +regie11.net,odiso.net,inbound,001,World,0 +regionalhelpwanted.com,regionalhelpwanted.com,inbound,001,World,1 +registrar-servers.com,registrar-servers.com,inbound,001,World,0.053619 +registro.br,registro.br,inbound,001,World,0 +relax7.hu,gruppi.hu,inbound,001,World,0 +relianceada.com,relianceada.com,inbound,001,World,0.276515 +rent.com,rent.com,inbound,001,World,0.000397 +rentalcars.com,rentalcars.com,inbound,001,World,1 +renweb.com,renweb.com,inbound,001,World,0 +repica.jp,kakaku.com,inbound,001,World,0 +repica.jp,repica.jp,inbound,001,World,0 +republicwireless.com,republicwireless.com,inbound,001,World,0.033564 +research-panel.jp,research-panel.jp,inbound,001,World,0 +researchgate.net,researchgate.net,inbound,001,World,0 +responder.co.il,responder.co.il,inbound,001,World,1 +restorationhardware.com,restorationhardware.com,inbound,001,World,0 +retailjobinsider.com,retailjobinsider.com,inbound,001,World,0 +retailmenot.com,retailmenot.com,inbound,001,World,3.54930373308668e-07 +reverbnation.com,reverbnation.com,inbound,001,World,0 +revolutiongolf.com,revolutiongolf.com,inbound,001,World,0 +rewardme.in,bfi0.com,inbound,001,World,0 +reyrey.net,reyrey.net,inbound,001,World,0.946402 +ricardoeletro.com.br,allin.com.br,inbound,001,World,0 +richersoundsvip.com,ibwmail.com,inbound,001,World,0 +richyrichmailer.com,maddog-productions.info,inbound,001,World,1 +rightmove.com,rightmove.com,inbound,001,World,0 +rigzonemail.com,rigzonemail.com,inbound,001,World,0 +rikunabi.com,rikunabi.com,inbound,001,World,1e-05 +ringcentral.com,ringcentral.com,inbound,001,World,1 +ripleyperu.com.pe,icommarketing.com,inbound,001,World,0 +riseup.net,riseup.net,inbound,001,World,1 +rivamail.com,mailurja.com,inbound,001,World,0 +rmtr.de,rapidmail.de,inbound,001,World,0 +rnmk.com,rnmk.com,inbound,001,World,0 +roadrunner.com,rr.com,inbound,001,World,0.005479 +roamans.com,neolane.net,inbound,001,World,0 +rocketmail.com,yahoo.{...},inbound,001,World,1 +rocketmail.com,yahoodns.net,outbound,001,World,1 +rockpath.info,rockpath.info,inbound,001,World,1 +rockwellcollins.com,rockwellcollins.com,inbound,001,World,1 +rogers.com,yahoo.{...},inbound,001,World,1 +rogers.com,yahoodns.net,outbound,001,World,1 +rookiestewsemails.com,rookiestewsemails.com,inbound,001,World,0 +roulartamail.be,roulartamail.be,inbound,001,World,0 +royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,001,World,0 +rpinow.org,app-info.net,inbound,001,World,0 +rpo9usa.email,rpo9usa.email,inbound,001,World,0 +rr.com,rr.com,inbound,001,World,0.008342 +rr.com,rr.com,outbound,001,World,0 +rsgsv.net,postini.com,inbound,001,World,0.093513 +rsgsv.net,rsgsv.net,inbound,001,World,0 +rsvpsv.net,rsvpsv.net,inbound,001,World,0 +rsvpsv.net,send.esp.br,inbound,001,World,0 +rsys2.com,amfam.com,inbound,001,World,0 +rsys2.com,cheaptickets.com,inbound,001,World,0 +rsys2.com,dishnetworkmail.com,inbound,001,World,0 +rsys2.com,e-comms.net,inbound,001,World,0 +rsys2.com,eharmony.com,inbound,001,World,0 +rsys2.com,fathead.com,inbound,001,World,0 +rsys2.com,intuit.com,inbound,001,World,0 +rsys2.com,kmart.com,inbound,001,World,0 +rsys2.com,kohlernews.com,inbound,001,World,0 +rsys2.com,lego.com,inbound,001,World,0 +rsys2.com,lenovo.com,inbound,001,World,0 +rsys2.com,modcloth.com,inbound,001,World,0 +rsys2.com,moxieinteractive.com,inbound,001,World,0 +rsys2.com,orbitz.com,inbound,001,World,0 +rsys2.com,payless.com,inbound,001,World,0 +rsys2.com,petsathome.com,inbound,001,World,0 +rsys2.com,quizzle.com,inbound,001,World,0 +rsys2.com,robeez.com,inbound,001,World,0 +rsys2.com,rsys1.com,inbound,001,World,0 +rsys2.com,rsys2.com,inbound,001,World,0 +rsys2.com,rsys3.com,inbound,001,World,0 +rsys2.com,rsys4.com,inbound,001,World,0 +rsys2.com,saucony.com,inbound,001,World,0 +rsys2.com,sears.com,inbound,001,World,0 +rsys2.com,shopbop.com,inbound,001,World,0 +rsys2.com,southwest.com,inbound,001,World,0 +rsys2.com,speeddatemail.com,inbound,001,World,0 +rsys2.com,thecompanystore.com,inbound,001,World,0 +rsys2.com,theknot.com,inbound,001,World,0 +rsys5.com,alibris.com,inbound,001,World,0 +rsys5.com,allstate-email.com,inbound,001,World,0 +rsys5.com,beachmint.com,inbound,001,World,0 +rsys5.com,belleandclive.com,inbound,001,World,0 +rsys5.com,br.dk,inbound,001,World,0 +rsys5.com,charlotterusse.com,inbound,001,World,0 +rsys5.com,comixology.com,inbound,001,World,0 +rsys5.com,cottonon.com,inbound,001,World,0 +rsys5.com,ediblearrangements.com,inbound,001,World,0 +rsys5.com,emailworldmarket.com,inbound,001,World,0 +rsys5.com,farfetch.com,inbound,001,World,0 +rsys5.com,frhiemailcommunications.com,inbound,001,World,0 +rsys5.com,harryanddavid.com,inbound,001,World,0 +rsys5.com,hollandandbarrett.com,inbound,001,World,0 +rsys5.com,icing.com,inbound,001,World,0 +rsys5.com,indigo.ca,inbound,001,World,0 +rsys5.com,jabong.com,inbound,001,World,0 +rsys5.com,jcrew.com,inbound,001,World,0 +rsys5.com,jjill.com,inbound,001,World,0 +rsys5.com,kanui.com.br,inbound,001,World,0 +rsys5.com,kirklands.com,inbound,001,World,0 +rsys5.com,lanebryant.com,inbound,001,World,0 +rsys5.com,lazada.com,inbound,001,World,0 +rsys5.com,leapfrog.com,inbound,001,World,0 +rsys5.com,llbean.com,inbound,001,World,0 +rsys5.com,lojascolombo.com.br,inbound,001,World,0 +rsys5.com,madewell.com,inbound,001,World,0 +rsys5.com,magazineluiza.com.br,inbound,001,World,0 +rsys5.com,missguided.co.uk,inbound,001,World,0 +rsys5.com,moma.org,inbound,001,World,0 +rsys5.com,nationalgeographic.com,inbound,001,World,0 +rsys5.com,neat.com,inbound,001,World,0 +rsys5.com,newbalance.com,inbound,001,World,0 +rsys5.com,news-voeazul.com.br,inbound,001,World,0 +rsys5.com,nordstrom.com,inbound,001,World,0 +rsys5.com,normthompson.com,inbound,001,World,0 +rsys5.com,novomundo.com.br,inbound,001,World,0 +rsys5.com,ourdeal.com.au,inbound,001,World,0 +rsys5.com,pier1.com,inbound,001,World,0 +rsys5.com,postini.com,inbound,001,World,0.0697 +rsys5.com,productmadness.com,inbound,001,World,0 +rsys5.com,rainbowshops.com,inbound,001,World,0 +rsys5.com,rei.com,inbound,001,World,0 +rsys5.com,roadrunnersports.com,inbound,001,World,0 +rsys5.com,rosettastone.com,inbound,001,World,0 +rsys5.com,seamless.com,inbound,001,World,0 +rsys5.com,serenaandlily.com,inbound,001,World,0 +rsys5.com,smiles.com.br,inbound,001,World,0 +rsys5.com,soubarato.com.br,inbound,001,World,0 +rsys5.com,strava.com,inbound,001,World,0 +rsys5.com,submarino.com.br,inbound,001,World,0 +rsys5.com,thewalkingcompany.com,inbound,001,World,0 +rsys5.com,tigerdirect.com,inbound,001,World,0 +rsys5.com,udemy.com,inbound,001,World,0 +rsys5.com,vitaminshoppe.com,inbound,001,World,0 +rsys5.com,vpusa.com,inbound,001,World,0 +rsys5.com,walmart.com.br,inbound,001,World,0 +rsys5.com,worldofwatches.com,inbound,001,World,0 +rsys5.com,xfinity.com,inbound,001,World,0 +rue21email.com,rue21email.com,inbound,001,World,0 +rueducommerce.com,groupe-rueducommerce.fr,inbound,001,World,0 +rummycirclemails.com,eccluster.com,inbound,001,World,0 +runkeeper.com,runkeeper.com,inbound,001,World,0 +runnet.jp,runnet.jp,inbound,001,World,0 +runtastic.com,runtastic.com,inbound,001,World,0 +rutenmail.com.tw,rutenmail.com.tw,inbound,001,World,0 +ruum.com,ruum.com,inbound,001,World,0 +ryanairmail.com,ryanairmail.com,inbound,001,World,0 +rzone.de,rzone.de,inbound,001,World,1 +s3s-br1.net,splio.com.br,inbound,001,World,0.908187 +s3s-main.net,splio.com,inbound,001,World,0.970428 +s4s-pl1.pl,splio.com,inbound,001,World,0.939032 +saavn.com,saavn.com,inbound,001,World,1 +safe-sender.net,safe-sender.net,inbound,001,World,0 +safelistextreme.com,quantumsafelist.com,inbound,001,World,0 +safelistpro.com,safelistpro.com,inbound,001,World,0.00014 +safeway.com,chtah.com,inbound,001,World,0 +safeway.com,safeway.com,inbound,001,World,0.000382 +sahibinden.com,sahibinden.com,inbound,001,World,0 +sailthru.com,sailthru.com,inbound,001,World,0 +saimails.in,infimail.com,inbound,001,World,0 +sainsburys.co.uk,emv5.com,inbound,001,World,0 +saisoncard.co.jp,saisoncard.co.jp,inbound,001,World,0 +saks.com,saks.com,inbound,001,World,0 +saksoff5th.com,saksoff5th.com,inbound,001,World,0 +sakura.ne.jp,sakura.ne.jp,inbound,001,World,0.640465 +salememail.net,salememail.net,inbound,001,World,0 +salesforce.com,postini.com,inbound,001,World,0.866057 +salesforce.com,salesforce.com,inbound,001,World,0.983322 +salesforce.com,salesforce.com,outbound,001,World,1 +salesmanago.pl,salesmanago.pl,inbound,001,World,0 +salliemae.com,salliemae.com,inbound,001,World,1 +salsalabs.net,salsalabs.net,inbound,001,World,0 +samashmusic.com,wc09.net,inbound,001,World,0 +samsclub.com,m0.net,inbound,001,World,0 +samsung.com,samsung.com,inbound,001,World,0.043941 +samsung.ru,samsung.ru,inbound,001,World,0 +samsungusa.com,samsungusa.com,inbound,001,World,0 +sanmina-sci.com,postini.com,inbound,001,World,0.999991 +sanmina.com,postini.com,inbound,001,World,0.9996 +sans.org,sans.org,inbound,001,World,0.497629 +santander.cl,santander.cl,inbound,001,World,0.999992 +santander.cl,santandersantiago.cl,inbound,001,World,1 +sapnetworkmail.com,sap-ag.de,inbound,001,World,1 +sapo.pt,sapo.pt,inbound,001,World,0.242839 +sapo.pt,sapo.pt,outbound,001,World,0 +saramin.co.kr,saramin.co.kr,inbound,001,World,0 +sassieshop.com,sassieshop.com,inbound,001,World,0.025887 +saturday.com,saturday.com,inbound,001,World,0 +savelivefresh.com,livesavemail.com,inbound,001,World,1 +savingdeals.in,infimail.com,inbound,001,World,0 +savingstar.com,savingstar.com,inbound,001,World,1 +sbcglobal.net,yahoo.{...},inbound,001,World,0.999991 +sbcglobal.net,yahoodns.net,outbound,001,World,1 +sbi.co.in,sbi.co.in,inbound,001,World,0 +sbr-inc.co.jp,hdemail.jp,inbound,001,World,0.000528 +sc.com,messagelabs.com,inbound,001,World,0.996156 +sc.com,sc.com,inbound,001,World,0.99683 +schwab.com,schwab.com,inbound,001,World,0.025055 +scmp.com,emarsys.net,inbound,001,World,0 +scoop.it,scoop.it,inbound,001,World,0 +scoopon.com.au,inxserver.de,inbound,001,World,1 +screwfix.info,fwdto.net,inbound,001,World,0 +sears.ca,sears.ca,inbound,001,World,0.005447 +searscard.com,searscard.com,inbound,001,World,1 +seaworld.com,seaworld.com,inbound,001,World,0 +secretescapes.com,secretescapes.com,inbound,001,World,0 +secure.ne.jp,secure.ne.jp,inbound,001,World,0.000356 +securence.com,securence.com,inbound,001,World,0.667957 +secureserver.net,secureserver.net,inbound,001,World,3.6e-05 +seek.com.au,seek.com.au,inbound,001,World,0 +seekingalpha.com,seekingalpha.com,inbound,001,World,1 +seekingalpha.com,sendgrid.net,inbound,001,World,1 +selectacast.net,selectacast.net,inbound,001,World,0.000561 +selection-priceminister.com,selection-priceminister.com,inbound,001,World,0 +semana.com,semana.com,inbound,001,World,0.005867 +senate.gov,senate.gov,inbound,001,World,0.992994 +sendearnings.com,sendearnings.com,inbound,001,World,0 +sender.lt,sritis.lt,inbound,001,World,0.000352 +sendgrid.info,sendgrid.net,inbound,001,World,0.999966 +sendgrid.me,sendgrid.net,inbound,001,World,1 +sendlane.com,sendlane.com,inbound,001,World,0 +sendpal.in,sendpal.in,inbound,001,World,0 +sendsmaily.info,sendsmaily.info,inbound,001,World,0 +seniorplanet.fr,seniorplanet.fr,inbound,001,World,0 +serpadres.es,chtah.net,inbound,001,World,0 +service-now.com,postini.com,inbound,001,World,0.988878 +service-now.com,service-now.com,inbound,001,World,0.998853 +serviciobancomer.com,serviciobancomer.com,inbound,001,World,0 +seznam.cz,seznam.cz,inbound,001,World,0.001781 +seznam.cz,seznam.cz,outbound,001,World,0.001741 +sfid01.com,sfid01.com,inbound,001,World,0 +sfimg.com,sfimarketing.com,inbound,001,World,0 +sfly.com,shutterfly.com,inbound,001,World,0 +sfr.fr,sfr.fr,inbound,001,World,0.236539 +sfr.fr,sfr.fr,outbound,001,World,0.004753 +shaadi.com,shaadi.com,inbound,001,World,0.000154 +shadowshopper.com,shadowshopper.com,inbound,001,World,5.6e-05 +shaw.ca,shaw.ca,inbound,001,World,0 +shaw.ca,shaw.ca,outbound,001,World,1 +sheplers.com,sheplers.com,inbound,001,World,0 +shiftplanning.com,shiftplanning.com,inbound,001,World,0 +shiksha.com,shiksha.com,inbound,001,World,0 +shinseibank.com,shinseibank.com,inbound,001,World,0 +shoedazzle.com,shoedazzle.com,inbound,001,World,0 +shoes.com,famousfootwear.com,inbound,001,World,0 +shop2gether.com.br,shop2gether.com.br,inbound,001,World,0 +shopbonton.com,shopbonton.com,inbound,001,World,0 +shopcluesemail.com,shopcluesemail.com,inbound,001,World,0 +shopcluesmail.com,shopcluesmail.com,inbound,001,World,0 +shophq.com,shophq.com,inbound,001,World,0 +shopjustice.com,shopjustice.com,inbound,001,World,0 +shopkick.com,shopkick.com,inbound,001,World,0 +shopko.com,shopko.com,inbound,001,World,0 +shopnineteenmails.in,iaires.com,inbound,001,World,0 +shoppersoptimum.ca,thindata.net,inbound,001,World,0 +shoppersstop.com,shoppersstop.com,inbound,001,World,0 +shoprite-email.com,email-mywebgrocer.com,inbound,001,World,0 +shoptime.com,shoptime.com,inbound,001,World,0 +shopto.net,shopto.net,inbound,001,World,1 +showingtime.com,showingtime.com,inbound,001,World,0 +showroomprive.com,showroomprive.be,inbound,001,World,0 +showroomprive.com,showroomprive.com,inbound,001,World,0.007102 +showroomprive.com,showroomprive.nl,inbound,001,World,0 +showroomprive.es,showroomprive.es,inbound,001,World,0 +showroomprive.it,showroomprive.pt,inbound,001,World,0 +showroomprive.pt,showroomprive.co.uk,inbound,001,World,0 +shtyle.fm,shtyle.fm,inbound,001,World,0 +shukatsu.jp,shukatsu.jp,inbound,001,World,0 +siella.jp,siella.jp,inbound,001,World,0.000199 +sierratradingpost.com,sierratradingpost.com,inbound,001,World,0.000885 +sigmabeauty.com,lstrk.net,inbound,001,World,1 +sii.cl,sii.cl,inbound,001,World,0 +simplesafelist.com,adminforfree.com,inbound,001,World,1 +simpletextadz.com,web-hosting.com,inbound,001,World,1 +simplyhired.com,simplyhired.com,inbound,001,World,0 +simplymarry.com,tbsl.in,inbound,001,World,0 +singsale.com.sg,singsale.com.sg,inbound,001,World,0 +siriusxm.com,xmradio.com,inbound,001,World,0 +sitecore-mailer.com,sendlabs.com,inbound,001,World,0 +sittercity.com,sittercity.com,inbound,001,World,0 +sixflags.com,sixflags.com,inbound,001,World,0 +skillpages-mailer.com,dynect.net,inbound,001,World,0 +skillpages-mailer.com,sendlabs.com,inbound,001,World,0 +skillpages-mailer.com,skillpagesmail.com,inbound,001,World,0 +sky.com,sky.com,inbound,001,World,8.8e-05 +skymall.com,skymall.com,inbound,001,World,0.001135 +skynet.be,belgacom.be,inbound,001,World,0.005903 +skype.com,delivery.net,inbound,001,World,0 +skype.com,skype.com,inbound,001,World,0 +skyscanner.net,skyscanner.net,inbound,001,World,1 +sld.cu,sld.cu,inbound,001,World,1 +slickdeals.net,slickdeals.net,inbound,001,World,4.9e-05 +slidesharemail.com,newslettergrid.com,inbound,001,World,1 +slidesharemail.com,slideshare.net,inbound,001,World,1 +slidesharemail.com,slidesharemail.com,inbound,001,World,1 +smartbrief.com,smartbrief.com,inbound,001,World,0 +smartdraw.com,smartdraw.com,inbound,001,World,0 +smartertravel.com,smartertravelmedia.com,inbound,001,World,0.021388 +smartphoneexperts.com,mailgun.net,inbound,001,World,1 +smartresponder.ru,smartresponder.ru,inbound,001,World,1 +smp.ne.jp,smp.ne.jp,inbound,001,World,0 +snagajob-email.com,snagajob-email.com,inbound,001,World,0 +snapdeal.com,snapdeal.com,inbound,001,World,0 +snapdealmail.in,snapdealmail.in,inbound,001,World,0 +snaphire.com,snaphire.com,inbound,001,World,0 +snapretail.com,snapretail.com,inbound,001,World,1 +socialappsmail.com,socialappsmail.com,inbound,001,World,1 +socialsex.biz,infinitypersonals.com,inbound,001,World,0 +sofmap.com,sofmap.com,inbound,001,World,0.0092 +softbank.jp,softbank.jp,inbound,001,World,0 +softbank.jp,softbank.jp,outbound,001,World,0 +softbank.ne.jp,softbank.ne.jp,inbound,001,World,0 +softbank.ne.jp,softbank.ne.jp,outbound,001,World,0 +solesociety.com,bronto.com,inbound,001,World,0 +solosenders.com,megasenders.com,inbound,001,World,8.2e-05 +solosenders.com,traxweb.org,inbound,001,World,0 +solveerrors.com,infimail.com,inbound,001,World,0 +soma.com,soma.com,inbound,001,World,0 +someecards.com,someecards.com,inbound,001,World,0 +songkick.com,songkick.com,inbound,001,World,1 +sony-latin.com,sony-latin.com,inbound,001,World,0.01735 +sony.com,sony.com,inbound,001,World,0.020225 +sony.jp,sony.jp,inbound,001,World,0.000654 +sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,001,World,0 +sonyrewards.com,sonyrewards.com,inbound,001,World,0 +soundcloudmail.com,soundcloudmail.com,inbound,001,World,0.999996 +sourceforge.net,sourceforge.net,inbound,001,World,1 +sourcenext.info,sourcenext.info,inbound,001,World,0 +southwest.com,southwest.com,inbound,001,World,0 +sp.gov.br,sp.gov.br,inbound,001,World,0.485842 +spanishdict.com,spanishdict.com,inbound,001,World,0.002698 +spareroom.co.uk,spareroom.co.uk,inbound,001,World,1 +sparklist.com,sparklist.com,inbound,001,World,0 +sparkpeople.com,sparkpeople.com,inbound,001,World,0 +spartoo.com,spartoo.com,inbound,001,World,0 +spectersoft.com,spectersoft.com,inbound,001,World,0 +speedyrewards-email.com,speedyrewards-email.com,inbound,001,World,0 +spencersonline.com,spencersonline.com,inbound,001,World,0 +spiritairlines.com,ctd004.net,inbound,001,World,0 +spiritairlines.com,ctd005.net,inbound,001,World,0 +splitwise.com,splitwise.com,inbound,001,World,1 +sportlobster.com,sportlobster.com,inbound,001,World,1 +sportsdirect.com,sportsdirect.com,inbound,001,World,0 +sportsline.com,cbsig.net,inbound,001,World,1 +sportsmansguide.com,sportsmansguide.com,inbound,001,World,0.736903 +spotifymail.com,spotifymail.com,inbound,001,World,1 +sprint.com,m0.net,inbound,001,World,0 +sqlservercentral.com,sqlservercentral.com,inbound,001,World,0 +square-enix.com,messagelabs.com,inbound,001,World,0.004493 +squareup.com,squareup.com,inbound,001,World,0.986452 +ssgadm.com,ssg.com,inbound,001,World,0 +staffeazymailers.com,iaires.com,inbound,001,World,0 +stakemail.com,iaires.com,inbound,001,World,0 +stampmail.in,iaires.com,inbound,001,World,0 +standaard.be,vummail.be,inbound,001,World,0 +standardbank.co.za,standardbank.co.za,inbound,001,World,0 +stanford.edu,highwire.org,inbound,001,World,1 +stanford.edu,stanford.edu,inbound,001,World,0.93025 +stansberryresearch.com,stansberry-re.net,inbound,001,World,0 +stansberryresearch.com,stansberryresearch.com,inbound,001,World,0.001008 +staples-pt.com,1-hostingservice.com,inbound,001,World,0 +staples.co.uk,ncrwebhost.de,inbound,001,World,0 +staples.com,staples.com,inbound,001,World,0.00638 +starbucks.com,iphmx.com,inbound,001,World,0.999476 +starbucks.com,starbucks.com,inbound,001,World,0 +stardockcorporation.com,stardockcorporation.com,inbound,001,World,0 +stardockentertainment.info,stardockentertainment.info,inbound,001,World,0 +starsports.com,eccluster.com,inbound,001,World,0 +startribune.com,startribune.com,inbound,001,World,0.001728 +startwire.com,jobsreport.com,inbound,001,World,1 +startwire.com,startwire.com,inbound,001,World,1 +starwoodhotels.com,outlook.com,inbound,001,World,1 +state-of-the-art-mailer.com,futurebanners.net,inbound,001,World,0 +state.gov,state.gov,inbound,001,World,0 +statefarm.com,statefarm.com,inbound,001,World,1 +stayfriends.de,stayfriends.de,inbound,001,World,0 +steampowered.com,steampowered.com,inbound,001,World,1 +steinmart.com,steinmart.com,inbound,001,World,0 +stelladot.com,stelladot.com,inbound,001,World,0 +stepstone.de,stepstone.com,inbound,001,World,0.001689 +stevemadden.com,stevemadden.com,inbound,001,World,0 +stjobs.sg,st701.com,inbound,001,World,1 +stjude.org,stjude.org,inbound,001,World,0.006716 +strava.com,strava.com,inbound,001,World,1 +streetauthoritydaily.com,streetauthoritydaily.com,inbound,001,World,0 +streeteasy.com,streeteasy.com,inbound,001,World,0 +striata.com,striata.com,inbound,001,World,0 +stubhub.com,stubhub.com,inbound,001,World,1 +studentbeans.com,emv8.com,inbound,001,World,0 +stumblemail.com,stumblemail.com,inbound,001,World,1 +stylecareers.com,stylecareers.com,inbound,001,World,0 +suafaturanet.com.br,suafaturanet.com.br,inbound,001,World,0.995846 +subito.it,subito.it,inbound,001,World,0 +subscribe.ru,subscribe.ru,inbound,001,World,0 +subscribermail.com,subscribermail.com,inbound,001,World,0 +subtend.info,subtend.info,inbound,001,World,1 +subway.com,subway.com,inbound,001,World,0 +sungard.com,postini.com,inbound,001,World,0.499946 +sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,001,World,1 +superbalist.com,sailthru.com,inbound,001,World,0 +superdeal.com.ua,mailersend.com,inbound,001,World,0 +superdrug.com,superdrug.com,inbound,001,World,0 +superjob.ru,superjob.ru,inbound,001,World,0 +supersafemailer.com,zoothost.com,inbound,001,World,0 +support-love.com,support-love.com,inbound,001,World,0 +supremelist.com,onlinehome-server.com,inbound,001,World,1 +surfmandelivery.com,surfmandelivery.com,inbound,001,World,0 +surlatable.com,surlatable.com,inbound,001,World,0 +surveyhelpcenter.com,jsmtp.net,inbound,001,World,0 +surveyjobopportunities.com,surveyjobopportunities.com,inbound,001,World,3.7e-05 +surveymonkey.com,surveymonkey.com,inbound,001,World,0 +surveysavvy.com,surveysavvy.com,inbound,001,World,0 +surveyspot.com,ssisurveys.com,inbound,001,World,0 +sut1.co.uk,sut1.co.uk,inbound,001,World,0.001789 +sut5.co.uk,sut5.co.uk,inbound,001,World,0 +swanson-vitamins.com,emv5.com,inbound,001,World,0 +sweepstakesalerts.com,sweepstakesalerts.com,inbound,001,World,0 +swimoutlet.com,isport.com,inbound,001,World,0 +sylectus.com,sylectus.com,inbound,001,World,0.361297 +sympatico.ca,hotmail.{...},inbound,001,World,1 +sympatico.ca,hotmail.{...},outbound,001,World,1 +synchronyfinancial.com,bigfootinteractive.com,inbound,001,World,0 +synergy360.jp,crmstyle.com,inbound,001,World,0 +t-online.de,t-online.de,inbound,001,World,1 +t-online.de,t-online.de,outbound,001,World,0.999939 +tadtopmails.com,tadtopmails.com,inbound,001,World,0 +taggedmail.com,taggedmail.com,inbound,001,World,0 +taipeifubon.com.tw,taipeifubon.com.tw,inbound,001,World,0 +taishinbank.com.tw,taishinbank.com.tw,inbound,001,World,0 +take2games.com,take2games.com,inbound,001,World,0 +talkmatch.com,talkmatch.com,inbound,001,World,0 +tamu.edu,tamu.edu,inbound,001,World,0.249656 +tanga.com,tanga.com,inbound,001,World,0 +tangeroutletsusa.com,bronto.com,inbound,001,World,0 +tanningmail.com,tanningmail.com,inbound,001,World,0 +tappingsolutionemail.com,tappingsolutionemail.com,inbound,001,World,0 +target-safelist.com,safelistpro.com,inbound,001,World,0.000215 +target.com,bigfootinteractive.com,inbound,001,World,0 +targetproblaster.com,targetproblaster.com,inbound,001,World,0 +targetx.com,targetx.com,inbound,001,World,0 +tarot.com,tarot.com,inbound,001,World,0 +tastefullysimpleparty.com,bigfootinteractive.com,inbound,001,World,0 +tasteofhome.com,tasteofhome.com,inbound,001,World,0 +tastingtable.com,tastingtable.com,inbound,001,World,0 +tatrabanka.sk,tatrabanka.sk,inbound,001,World,0 +taxi4sure.net,infimail.com,inbound,001,World,0 +tchibo.com.tr,euromsg.net,inbound,001,World,0 +tchibo.de,srv2.de,inbound,001,World,0.980445 +teach12.net,teach12.net,inbound,001,World,0 +teambuymail.com,teambuymail.com,inbound,001,World,0 +teamo.ru,teamo.ru,inbound,001,World,0 +teamsnap.com,teamsnap.com,inbound,001,World,1 +teamviewer.com,teamviewer.com,inbound,001,World,0 +teapartyinfo.org,teapartyinfo.org,inbound,001,World,0 +techgig.com,tbsl.in,inbound,001,World,0 +technolutions.net,technolutions.net,inbound,001,World,1 +techtarget.com,techtarget.com,inbound,001,World,0.001771 +telefonica.com,telefonica.com,inbound,001,World,0.998662 +telegraph.co.uk,telegraph.co.uk,inbound,001,World,0 +telenet.be,telenet-ops.be,inbound,001,World,0.000128 +teleportmyjob.com,clara.net,inbound,001,World,0 +telus.com,telus.com,inbound,001,World,0.000118 +telus.net,telus.net,inbound,001,World,0.006226 +templeandwebster.com.au,templeandwebster.com.au,inbound,001,World,5e-06 +ten24mail.com,ten24mail.com,inbound,001,World,0 +terra.com,terra.com,inbound,001,World,0.000463 +terra.com.br,terra.com,inbound,001,World,0 +terra.com.br,terra.com,outbound,001,World,0 +tesco.com,tesco.com,inbound,001,World,0 +testfunda.com,testfunda.com,inbound,001,World,0 +texasjobdepartment.com,texasjobdepartment.com,inbound,001,World,0 +textnow.me,textnow.me,inbound,001,World,0 +tgw.com,tgw.com,inbound,001,World,0 +theanimalrescuesite.com,theanimalrescuesite.com,inbound,001,World,0 +theatermania.com,wc09.net,inbound,001,World,0 +thebay.com,thebay.com,inbound,001,World,0 +thebodyshop-usa.com,email-bodyshop.com,inbound,001,World,0 +thebodyshop-usa.com,postdirect.com,inbound,001,World,0 +thecarousell.com,thecarousell.com,inbound,001,World,1 +thegrommet.com,lstrk.net,inbound,001,World,1 +theguardian.com,theguardian.com,inbound,001,World,0 +thehut.com,thehut.com,inbound,001,World,0 +theleadmagnet.com,your-server.de,inbound,001,World,1 +thelimited.com,thelimited.com,inbound,001,World,0 +themailbagsafelist.com,thomas-j-brown.com,inbound,001,World,0 +theoutnet.com,theoutnet.com,inbound,001,World,0 +thepamperedchef.com,thepamperedchef.com,inbound,001,World,0 +thephonehouse.es,splio.com,inbound,001,World,0.983578 +thephonehouse.es,splio.es,inbound,001,World,0.983266 +therealreal.com,email-realreal.com,inbound,001,World,0 +theskimm.com,theskimm.com,inbound,001,World,0 +thesource.ca,thesource.ca,inbound,001,World,0.00923 +thesovereigninvestor.com,sovereignsociety.com,inbound,001,World,0 +thewarehouse.co.nz,thewarehouse.co.nz,inbound,001,World,0 +thinkgeek.com,thinkgeek.com,inbound,001,World,1e-06 +thinkvidya.com,thinkvidya.com,inbound,001,World,0 +thirtyonegifts.com,thirtyonegifts.com,inbound,001,World,0 +thomascook.com,eccluster.com,inbound,001,World,0 +thoughtful-mind.com,thoughtful-mind.com,inbound,001,World,0 +thumbtack.com,thumbtack.com,inbound,001,World,1 +ticketmaster.com,ticketmaster.com,inbound,001,World,0.251337 +ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,001,World,0 +ticketmonster.co.kr,ticketmonster.co.kr,inbound,001,World,0 +timehop.com,timehop.com,inbound,001,World,1 +timeout.com,ec-cluster.com,inbound,001,World,0 +timesjobs.com,tbsl.in,inbound,001,World,0 +timesjobsmail.com,tbsl.in,inbound,001,World,0 +timesofindia.com,indiatimes.com,inbound,001,World,0 +timewarnercable.com,bigfootinteractive.com,inbound,001,World,0 +timeweb.ru,timeweb.ru,inbound,001,World,1 +tinyletterapp.com,tinyletterapp.com,inbound,001,World,0 +tiscali.it,tiscali.it,outbound,001,World,0 +tmart.com,chtah.net,inbound,001,World,0 +tmp.com,tmpw.com,inbound,001,World,0 +tobi.com,messagebus.com,inbound,001,World,0 +tobizaru.jp,tobizaru.jp,inbound,001,World,0 +tocoo.jp,aics.ne.jp,inbound,001,World,0 +toluna.com,toluna.com,inbound,001,World,0 +tomtommailer.com,tomtommailer.com,inbound,001,World,0 +topface.com,topface.com,inbound,001,World,1e-06 +topica.com,topica-silver-y.com,inbound,001,World,0 +topqpon.si,topqpon.si,inbound,001,World,0 +topspin.net,topspin.net,inbound,001,World,1 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 +touchbase2.com,infimail.com,inbound,001,World,0 +touchbase2.com,mailurja.com,inbound,001,World,0 +touchbasepro.com,touchbasepro.com,inbound,001,World,0 +tower.jp,tower.jp,inbound,001,World,0 +townhallmail.com,townhallmail.com,inbound,001,World,0 +townnews-mail.com,townnews-mail.com,inbound,001,World,0 +townsquaremedia.info,sailthru.com,inbound,001,World,0 +toysrus.com,epsl1.com,inbound,001,World,0 +trabajar.com,trabajo.org,inbound,001,World,0.999997 +trabalhar.com,trabajo.org,inbound,001,World,0.999994 +tradeloop.com,tradeloop.com,inbound,001,World,1 +trademe.co.nz,trademe.co.nz,inbound,001,World,0.991916 +trafficboostermailer.com,trafficboostermailer.com,inbound,001,World,1 +trafficleads2incomevm.com,zoothost.com,inbound,001,World,0 +trafficprolist.com,thomas-j-brown.com,inbound,001,World,0 +trafficwave.net,trafficwave.net,inbound,001,World,0 +transittraveljobinsider.com,transittraveljobinsider.com,inbound,001,World,0 +transportexchangegroup.com,transportexchangegroup.com,inbound,001,World,0.998703 +transversal.net,transversal.net,inbound,001,World,1 +travelchannel.com,travelchannel.com,inbound,001,World,0 +travelocity.com,travelocity.com,inbound,001,World,0.000194 +travelzoo.com,travelzoo.com,inbound,001,World,0 +travisperkins.co.uk,travisperkins.co.uk,inbound,001,World,0 +trclient.com,trclient.com,inbound,001,World,0 +treemall.com.tw,symphox.com,inbound,001,World,0 +trello.com,mandrillapp.com,inbound,001,World,1 +trialsmith.com,membercentral.com,inbound,001,World,0 +tricaemidia.com.br,tricaemidia.com.br,inbound,001,World,0 +triongames.com,triongames.com,inbound,001,World,0.085718 +tripadvisor.com,tripadvisor.com,inbound,001,World,0.000588 +tripit.com,tripit.com,inbound,001,World,0 +tripolis.com,tripolis.com,inbound,001,World,0 +trovit.com,trovit.com,inbound,001,World,0 +trulia.com,trulia.com,inbound,001,World,0 +tsite.jp,tsite.jp,inbound,001,World,0 +tsmmail.com,tsmmail.com,inbound,001,World,0 +tsutaya.co.jp,tsutaya.co.jp,inbound,001,World,0 +tucasa.com,grupodtm.com,inbound,001,World,0 +tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,001,World,0 +tumblr.com,tumblr.com,inbound,001,World,1 +turbine.com,turbine.com,inbound,001,World,0.013592 +turkcell.com.tr,turkcell.com.tr,inbound,001,World,0.043492 +turner.com,cnn.com,inbound,001,World,0 +twe-safelist.com,adminforfree.com,inbound,001,World,1 +twitch.tv,justin.tv,inbound,001,World,0 +twitter.com,postini.com,inbound,001,World,0.765163 +twitter.com,twitter.com,inbound,001,World,0.999969 +twoomail.com,netlogmail.com,inbound,001,World,0 +twoomail.com,twoomail.com,inbound,001,World,0 +type.jp,type.jp,inbound,001,World,0 +u-shopping.com.tw,u-shopping.com.tw,inbound,001,World,0 +uber.com,uber.com,inbound,001,World,1 +ubi.com,ubi.com,inbound,001,World,0 +ubivox.com,ubivox.com,inbound,001,World,0.956275 +ubuntu.com,canonical.com,inbound,001,World,0.000129 +ucla.edu,ucla.edu,inbound,001,World,0.806204 +udnpaper.com,udnpaper.com,inbound,001,World,0.998858 +udnshopping.com,udnshopping.com,inbound,001,World,0 +uga.edu,outlook.com,inbound,001,World,1 +uhaul.com,uhaul.com,inbound,001,World,0.997947 +uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,001,World,0 +uiuc.edu,illinois.edu,inbound,001,World,0.999815 +ukr.net,fwdcdn.com,inbound,001,World,1 +ukr.net,ukr.net,outbound,001,World,0.999992 +ulta.com,exacttarget.com,inbound,001,World,0 +ulta.com,ulta.com,inbound,001,World,0.034861 +ulteem.com,ulteem.com,inbound,001,World,0 +ultimateadsites.net,ultimateadsites.net,inbound,001,World,1 +umd.edu,umd.edu,inbound,001,World,0.021709 +umn.edu,umn.edu,inbound,001,World,0.966992 +umpiredigital.com,inboxmarketer.com,inbound,001,World,0 +unilever.com,mxlogic.net,inbound,001,World,1 +uniqlo-usa.com,uniqlo-usa.com,inbound,001,World,0 +united.com,coair.com,inbound,001,World,1 +united.com,united.com,inbound,001,World,0 +unitedrepublic.org,unitedrepublic.org,inbound,001,World,1 +universalorlando.com,universalorlando.com,inbound,001,World,0 +unosinsidersclub.com,unosinsidersclub.com,inbound,001,World,0 +unrollmail.com,unrollmail.com,inbound,001,World,1 +uol.com.br,uol.com.br,inbound,001,World,0 +uol.com.br,uol.com.br,outbound,001,World,0 +upenn.edu,upenn.edu,inbound,001,World,0.16521 +upromise.com,delivery.net,inbound,001,World,0 +ups.com,ups.com,inbound,001,World,0.997955 +urbanoutfitters.com,freepeople.com,inbound,001,World,0 +urx.com.br,urx.com.br,inbound,001,World,0 +usaa.com,usaa.com,inbound,001,World,0.639434 +usafisnews.org,usafisnews.org,inbound,001,World,0 +usahockey-email.com,usahockey-email.com,inbound,001,World,0 +usajobs.gov,opm.gov,inbound,001,World,0.931948 +usc.edu,usc.edu,inbound,001,World,0.300314 +uscourts.gov,uscourts.gov,inbound,001,World,0 +uslargestsafelist.com,zoothost.com,inbound,001,World,0 +usndr.com,unisender.com,inbound,001,World,1 +usndr.com,usndr.com,inbound,001,World,1 +usp.br,usp.br,inbound,001,World,0.044176 +usps.com,usps.gov,inbound,001,World,0.909588 +usps.gov,usps.gov,inbound,001,World,0.940315 +ustream.tv,mailgun.net,inbound,001,World,1 +ustream.tv,mailgun.us,inbound,001,World,1 +usx.com.br,uqx.com.br,inbound,001,World,0 +usx.com.br,usx.com.br,inbound,001,World,0 +usx.com.br,utx.com.br,inbound,001,World,0 +utfsm.cl,utfsm.cl,inbound,001,World,0.002318 +utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,001,World,0 +utx.com.br,utx.com.br,inbound,001,World,0 +uvarosa.com.br,uvarosa.com.br,inbound,001,World,0 +vacationstogo.com,vacationstogo.com,inbound,001,World,0 +vagas.com.br,vagas.com.br,inbound,001,World,0 +vakifbank.com.tr,vakifbank.com.tr,inbound,001,World,1 +valuedopinions.co.uk,researchnow-usa.com,inbound,001,World,1 +vanheusenrewards.com,vanheusenrewards.com,inbound,001,World,0 +vd.nl,emsecure.net,inbound,001,World,0 +velocityfrequentflyer.com,virginaustralia.com,inbound,001,World,0.003442 +venca.es,eccluster.com,inbound,001,World,0 +venere.com,kiwari.com,inbound,001,World,0 +vente-exclusive.com,vente-exclusive.com,inbound,001,World,0.000463 +venteprivee.com,venteprivee.com,inbound,001,World,0 +venusswimwear.net,venusswimwear.net,inbound,001,World,0 +verabradleymail.com,verabradleymail.com,inbound,001,World,0 +verizon.com,verizon.com,inbound,001,World,0.999793 +verizon.net,verizon.net,inbound,001,World,0.00713 +verizon.net,verizon.net,outbound,001,World,0 +verizon.net,yahoo.{...},inbound,001,World,0.999435 +verizonwireless.com,verizonwireless.com,inbound,001,World,0.972245 +vfoutletvip.com,vfoutletvip.com,inbound,001,World,0 +vhmnetworkemail.com,jobdiagnosis.com,inbound,001,World,0 +viagogo.com,chtah.net,inbound,001,World,0 +viajanet.com.br,viajanet.com.br,inbound,001,World,0.001345 +vicinity.nl,picsrv.net,inbound,001,World,0.022107 +victoriassecret.com,victoriassecret.com,inbound,001,World,0 +videotron.ca,videotron.ca,inbound,001,World,0.005886 +vietcombank.com.vn,vietcombank.com.vn,inbound,001,World,1 +vikingrivercruises.com,bfi0.com,inbound,001,World,0 +vimeo.com,vimeo.com,inbound,001,World,0 +vindale.com,vindale.com,inbound,001,World,0 +vipvoice.com,npdor.com,inbound,001,World,0 +viraladmagnet.com,viraladmagnet.com,inbound,001,World,1 +viralsender.com,viralsender.com,inbound,001,World,0 +virgilio.it,virgilio.net,inbound,001,World,0 +virginia.edu,virginia.edu,inbound,001,World,0.055153 +virtualtarget.com.br,virtualtarget.com.br,inbound,001,World,0 +vistaprint.com,vistaprint.com,inbound,001,World,0 +vistaprint.com.au,vistaprint.com.au,inbound,001,World,0 +visualsoft.co.uk,visualsoft.co.uk,inbound,001,World,0 +vitacost.com,vitacost.com,inbound,001,World,0 +vitaladviews.com,zoothost.com,inbound,001,World,0 +vitaminworld.com,email-nbtyinc.com,inbound,001,World,0 +vivastreet.com,viwii.net,inbound,001,World,0 +vk.com,vkontakte.ru,inbound,001,World,0 +vocus.com,vocus.com,inbound,001,World,0 +vodafone.com,vodafone.in,inbound,001,World,0.617518 +vonage.com,vonagenetworks.net,inbound,001,World,0.006531 +votervoice.net,votervoice.net,inbound,001,World,0 +vouchercloud.com,vouchercloud.com,inbound,001,World,0 +vovici.com,vovici.com,inbound,001,World,0 +voyageprive.com,cccampaigns.net,inbound,001,World,0 +voyageprive.es,ccmdcampaigns.net,inbound,001,World,0 +voyageprive.it,ccmdcampaigns.net,inbound,001,World,0 +voyages-sncf.com,neolane.net,inbound,001,World,0 +vpass.ne.jp,clickmailer.jp,inbound,001,World,0 +vpcontact.com,vpcontact.com,inbound,001,World,0 +vresp.com,verticalresponse.com,inbound,001,World,0 +vt.edu,vt.edu,inbound,001,World,0.978548 +vtext.com,vtext.com,inbound,001,World,0 +vtext.com,vtext.com,outbound,001,World,1 +vudu.com,vudu.com,inbound,001,World,0 +vueling.com,vueling.com,inbound,001,World,0 +vuezone.com,vuezone.com,inbound,001,World,0 +vzwpix.com,vtext.com,inbound,001,World,0 +wagjag.com,wagjag.com,inbound,001,World,0 +walgreens.com,walgreens.com,inbound,001,World,0.006128 +wallst.com,wallst.com,inbound,001,World,0 +wallstreetdaily.com,wallstreetdaily.com,inbound,001,World,0 +walmart.ca,walmart.ca,inbound,001,World,0 +walmart.com,walmart.com,inbound,001,World,0.315059 +wanadoo.fr,orange.fr,inbound,001,World,0 +wanadoo.fr,orange.fr,outbound,001,World,0 +warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,001,World,0 +waterstones.com,waterstones.com,inbound,001,World,0 +wattpad.com,wattpad.com,inbound,001,World,1 +waves-audio.com,emv8.com,inbound,001,World,0 +way2movies.net,way2movies.net,inbound,001,World,0 +way2sms.biz,way2sms.biz,inbound,001,World,0 +way2sms.in,way2sms.in,inbound,001,World,0 +way2smsemail.com,way2smsemail.com,inbound,001,World,0 +way2smsemails.com,way2smsemails.com,inbound,001,World,0 +way2smsmail.in,way2smsmail.in,inbound,001,World,0 +way2smsmails.com,way2smsmails.com,inbound,001,World,0 +wayfair.com,csnstores.com,inbound,001,World,0 +wealthyaffiliate.com,wealthyaffiliate.com,inbound,001,World,1 +web.de,web.de,inbound,001,World,0.999986 +web.de,web.de,outbound,001,World,1 +webcas.net,webcas.net,inbound,001,World,0 +webmd.com,webmd.com,inbound,001,World,2e-06 +webs.com,epsl1.com,inbound,001,World,0 +websaver.ca,websaver.ca,inbound,001,World,0 +websitesettings.com,stabletransit.com,inbound,001,World,0 +websitewelcome.com,websitewelcome.com,inbound,001,World,1 +webstars2k.com,webstars2k.com,inbound,001,World,1 +wechat.com,qq.com,inbound,001,World,1 +weebly.com,weeblymail.com,inbound,001,World,0 +wegottickets.com,wegottickets.com,inbound,001,World,0 +weheartit.com,weheartit.com,inbound,001,World,1 +wehkamp.nl,wehkamp.nl,inbound,001,World,0 +wellsfargo.com,wellsfargo.com,inbound,001,World,1.0 +wellsfargoadvisors.com,wellsfargo.com,inbound,001,World,1 +wemakeprice.com,wemakeprice.com,inbound,001,World,0 +westelm.com,westelm.com,inbound,001,World,0 +westmarine.com,westmarine.com,inbound,001,World,0 +westwing.com.br,cust-cluster.com,inbound,001,World,0 +westwing.es,ecm-cluster.com,inbound,001,World,0 +westwing.ru,ecm-cluster.com,inbound,001,World,0 +wetransfer.com,wetransfer.com,inbound,001,World,0.999751 +wetsealnewsletter.com,wetsealnewsletter.com,inbound,001,World,0 +wgbh.org,wgbh.org,inbound,001,World,0 +whaakky.com,whaakky.com,inbound,001,World,0 +whatcounts.com,wc09.net,inbound,001,World,0 +whentowork.com,whentowork.com,inbound,001,World,0 +whereareyounow.com,wayn.net,inbound,001,World,0 +whitehouse.gov,whitehouse.gov,inbound,001,World,0 +whitelabelpros.com,whitelabelpros.com,inbound,001,World,0 +wiggle.com,wiggle.com,inbound,001,World,0 +wikia.com,wikia.com,inbound,001,World,0.550228 +wikimedia.org,wikimedia.org,inbound,001,World,0 +william-reed.com,neolane.net,inbound,001,World,0 +williamhill.com,williamhill.com,inbound,001,World,0 +williams-sonoma.com,williams-sonoma.com,inbound,001,World,0 +windstream.net,windstream.net,inbound,001,World,0.002172 +wine.com,wine.com,inbound,001,World,0 +winkalmail.com,fnbox.com,inbound,001,World,0 +wisdomitservices.com,infimail.com,inbound,001,World,0 +wldemail-mailer.com,wldemail.com,inbound,001,World,0 +wldemail.com,emarsys.net,inbound,001,World,0 +wmtransfer.com,wmtransfer.com,inbound,001,World,0 +wnd.com,emv4.net,inbound,001,World,0 +wnd.com,worldnetdaily.com,inbound,001,World,0 +wolfmedia.us,wolfmedia.us,inbound,001,World,0 +womanwithin.com,womanwithin.com,inbound,001,World,0 +woodforest.com,woodforest.com,inbound,001,World,1 +wordfly.com,wordfly.com,inbound,001,World,0 +work.ua,work.ua,inbound,001,World,1 +workcircle.com,workcircle.net,inbound,001,World,0 +workhunter.net,workhunter.net,inbound,001,World,1 +workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,001,World,0 +worldsingles.co,worldsingles.com,inbound,001,World,0 +worldwinner.com,worldwinner.com,inbound,001,World,0.112191 +wotif.com,whatcounts.com,inbound,001,World,0 +wowcher.co.uk,wowcher.co.uk,inbound,001,World,0.000373 +wp.com,wordpress.com,inbound,001,World,0 +wp.pl,wp.pl,inbound,001,World,0.998027 +wp.pl,wp.pl,outbound,001,World,1 +wpengine.com,wpengine.com,inbound,001,World,1 +writers-community.com,writers-community.com,inbound,001,World,0 +writersstore.com,writersstore.com,inbound,001,World,0 +wsjemail.com,wsjemail.com,inbound,001,World,0 +wuaki.tv,chtah.net,inbound,001,World,0 +wustl.edu,app-info.net,inbound,001,World,0 +wwe.com,wwe.com,inbound,001,World,8e-06 +www.gov.tw,hinet.net,inbound,001,World,8e-05 +wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,001,World,0 +xbox.com,xbox.com,inbound,001,World,0 +xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,001,World,0 +xen.org,xen.org,inbound,001,World,1 +xing.com,xing.com,inbound,001,World,0 +xmailix.com,xmailix.com,inbound,001,World,0 +xmeeting.com,xmeeting.com,inbound,001,World,1 +xmr3.com,messagereach.com,inbound,001,World,0.999933 +xoom.com,xoom.com,inbound,001,World,0 +xpnews.com.br,xpnews.com.br,inbound,001,World,1 +xxxconnect.com,infinitypersonals.com,inbound,001,World,0 +yahoo-inc.com,yahoo.{...},inbound,001,World,0.999974 +yahoo.co.jp,yahoo.co.jp,inbound,001,World,1.5e-05 +yahoo.co.jp,yahoo.co.jp,outbound,001,World,0 +yahoo.{...},postini.com,inbound,001,World,0.767368 +yahoo.{...},yahoo.co.jp,inbound,001,World,0 +yahoo.{...},yahoo.{...},inbound,001,World,0.999416 +yahoo.{...},yahoodns.net,outbound,001,World,1 +yahoogroups.com,yahoodns.net,outbound,001,World,1 +yakala.co,euromsg.net,inbound,001,World,0 +yammer.com,yammer.com,inbound,001,World,1 +yandex.ru,yandex.net,inbound,001,World,0.999698 +yandex.ru,yandex.ru,outbound,001,World,1 +yapikredi.com.tr,yapikredi.com.tr,inbound,001,World,1 +yapstone.com,yapstone.com,inbound,001,World,0 +yelp.com,yelpcorp.com,inbound,001,World,0 +yesbank.in,yesbank.in,inbound,001,World,0.108676 +yipit.com,yipit.com,inbound,001,World,1 +ymail.com,yahoo.{...},inbound,001,World,1 +ymail.com,yahoodns.net,outbound,001,World,1 +ymlpserver.net,ymlpserver.net,inbound,001,World,0 +ymlpsrv.net,ymlpsrv.net,inbound,001,World,0 +yodobashi.com,yodobashi.com,inbound,001,World,0 +yoox.com,yoox.com,inbound,001,World,0 +youmail.com,youmail.com,inbound,001,World,0 +youravon.com,email-avonglobal.com,inbound,001,World,0 +youreletters3.com,equitymaster.com,inbound,001,World,0 +yourezads.com,yourezads.com,inbound,001,World,0.002974 +yourezlist.com,simplicityads.com,inbound,001,World,9.5e-05 +yourhostingaccount.com,yourhostingaccount.com,inbound,001,World,0 +yournewsletters.net,everydayhealth.com,inbound,001,World,0 +youversion.com,youversion.com,inbound,001,World,1 +zacks.com,zacks.com,inbound,001,World,0.999188 +zalando.be,fagms.de,inbound,001,World,0 +zalando.dk,fagms.de,inbound,001,World,0 +zalando.fi,fagms.de,inbound,001,World,0 +zalando.it,fagms.de,inbound,001,World,0 +zalando.nl,fagms.de,inbound,001,World,0 +zalando.pl,fagms.de,inbound,001,World,0 +zappos.com,zappos.com,inbound,001,World,0.626758 +zara.com,cheetahmail.com,inbound,001,World,0 +zattoo.com,sendnode.com,inbound,001,World,0 +zelonews.com.br,zelonews.com.br,inbound,001,World,0 +zendesk.com,zdsys.com,inbound,001,World,1 +zgalleriestyle.com,zgalleriestyle.com,inbound,001,World,0 +zibmail.info,zibmail.info,inbound,001,World,2e-06 +zillow.com,zillow.com,inbound,001,World,2.82542783334609e-07 +zinio.com,zinio.com,inbound,001,World,0.006193 +zinio.net,zinio.com,inbound,001,World,1 +zipalerts.com,sendgrid.net,inbound,001,World,1 +zipalerts.com,zipalerts.com,inbound,001,World,1 +ziprealty.com,ziprealty.com,inbound,001,World,0.994545 +ziprecruiter.com,ziprecruiter.com,inbound,001,World,1 +zivamewear.com,infimail.com,inbound,001,World,0 +zizigo.com,euromsg.net,inbound,001,World,0 +zlavadna.sk,zlavadna.sk,inbound,001,World,0 +zoom.com.br,zoom.com.br,inbound,001,World,0 +zoominternet.net,synacor.com,inbound,001,World,0 +zoosk.com,zoosk.com,inbound,001,World,0 +zoothost.com,zoothost.com,inbound,001,World,0.107168 +zorpia.com,zorpia.com,inbound,001,World,0.56104 +zovifashion.com,eccluster.com,inbound,001,World,0 +zulily.com,zulily.com,inbound,001,World,0 +zumzi.com,neogen.ro,inbound,001,World,0 +zumzi.com,zumzi.com,inbound,001,World,0 +zyngamail.com,zyngamail.com,inbound,001,World,0 +zzounds.com,zzounds.com,inbound,001,World,0 +careers24.com,careers24.com,inbound,002,Africa,0 +cpm.co.ma,cpm.co.ma,inbound,002,Africa,0 +fnb.co.za,fnb.co.za,inbound,002,Africa,0 +gmail.com,telkomadsl.co.za,inbound,002,Africa,0.999989 +gmail.com,vodacom.co.za,inbound,002,Africa,1 +gtbank.com,gtbank.com,inbound,002,Africa,0.056121 +pnetweb.co.za,hosting.co.za,inbound,002,Africa,0 +pnetweb.co.za,salesnet.co.za,inbound,002,Africa,0 +bigpond.com,bigpond.com,inbound,009,Oceania,0 +bigpond.com,bigpond.com,outbound,009,Oceania,1 +empoweredcomms.com.au,empoweredcomms.com.au,inbound,009,Oceania,0 +gmail.com,bigpond.net.au,inbound,009,Oceania,0.999907 +gmail.com,iinet.net.au,inbound,009,Oceania,0.966945 +gmail.com,optusnet.com.au,inbound,009,Oceania,0.992845 +gmail.com,tpgi.com.au,inbound,009,Oceania,0.999648 +mbounces.com,emdbms.com,inbound,009,Oceania,0 +realestate.com.au,realestate.com.au,inbound,009,Oceania,0 +snaphire.com,snaphire.com,inbound,009,Oceania,0 +trademe.co.nz,trademe.co.nz,inbound,009,Oceania,0.991916 +1-day.co.nz,1-day.co.nz,inbound,019,Americas,0 +12manage.com,netarrest.com,inbound,019,Americas,0.99998 +1800petmeds.com,1800petmeds.com,inbound,019,Americas,0.00599 +2touchbase.com,infimail.com,inbound,019,Americas,0 +4wheelparts.com,4wheelparts.com,inbound,019,Americas,0 +99acres.com,99acres.com,inbound,019,Americas,0 +aafes.com,aafes.com,inbound,019,Americas,0 +abercrombie-email.com,abercrombie-email.com,inbound,019,Americas,0 +abercrombiekids-email.com,abercrombie-email.com,inbound,019,Americas,0 +about.com,about.com,inbound,019,Americas,0 +about.com,sailthru.com,inbound,019,Americas,0 +acemserv.com,acemserv.com,inbound,019,Americas,0 +activesafelist.com,zoothost.com,inbound,019,Americas,0 +adidasusnews.com,adidasusnews.com,inbound,019,Americas,0 +adjockeys.com,thomas-j-brown.com,inbound,019,Americas,0 +admastersafelist.com,zoothost.com,inbound,019,Americas,0 +adorama.com,adorama.com,inbound,019,Americas,0 +adpirate.net,thomas-j-brown.com,inbound,019,Americas,0 +adtpulse.com,adtpulse.com,inbound,019,Americas,0 +ae.com,ae.com,inbound,019,Americas,0 +aexp.com,aexp.com,inbound,019,Americas,1 +affairalert.com,iverificationsystems.com,inbound,019,Americas,0 +agorafinancial.com,agorafinancial.com,inbound,019,Americas,0 +airbnb.com,airbnb.com,inbound,019,Americas,0.867975 +airbrake.io,mailgun.net,inbound,019,Americas,1 +airfarewatchdog.com,smartertravelmedia.com,inbound,019,Americas,0.012002 +airmiles.ca,bigfootinteractive.com,inbound,019,Americas,0 +akcijatau.lt,akcijatau.lt,inbound,019,Americas,0 +alarm.com,alarm.com,inbound,019,Americas,0 +alarmnet.com,alarmnet.com,inbound,019,Americas,0 +alaskaair.com,alaskaair.com,inbound,019,Americas,0 +alertid.com,alertid.com,inbound,019,Americas,0 +allheart.com,allheart.com,inbound,019,Americas,0 +allmodern.com,allmodern.com,inbound,019,Americas,0 +allout.org,allout.org,inbound,019,Americas,1 +allrecipes.com,allrecipes.com,inbound,019,Americas,0 +allstate.com,rsys1.com,inbound,019,Americas,0 +alm.com,sailthru.com,inbound,019,Americas,0 +alumniclass.com,alumniclass.com,inbound,019,Americas,0 +alumniconnections.com,alumniconnections.com,inbound,019,Americas,0 +amazon.{...},postini.com,inbound,019,Americas,0.658136 +amazonses.com,postini.com,inbound,019,Americas,0.733998 +americanas.com,americanas.com,inbound,019,Americas,0 +americanbar.org,abanet.org,inbound,019,Americas,0.00574 +amubm.com,amubm.com,inbound,019,Americas,1 +ancestry.com,ancestry.com,inbound,019,Americas,0 +angelbroking.in,infimail.com,inbound,019,Americas,0 +anghami.com,mailgun.net,inbound,019,Americas,1 +anthropologie.com,freepeople.com,inbound,019,Americas,0 +aol.com,aol.com,inbound,019,Americas,0.999528 +aol.com,aol.com,outbound,019,Americas,0.999984 +aol.com,sailthru.com,inbound,019,Americas,0 +aol.net,aol.com,inbound,019,Americas,1 +apache.org,apache.org,inbound,019,Americas,0 +apnacomplex.com,apnacomplex.com,inbound,019,Americas,0.999981 +apply-4-jobs.com,apply-4-jobs.com,inbound,019,Americas,0 +aptmail.in,mailurja.com,inbound,019,Americas,0 +arcamax.com,arcamax.com,inbound,019,Americas,1.4e-05 +aritzia.com,aritzia.com,inbound,019,Americas,0 +armaniexchange.com,bronto.com,inbound,019,Americas,0 +asadventure.com,asadventure.com,inbound,019,Americas,0 +asana.com,asana.com,inbound,019,Americas,1 +ashleymadison.com,ashleymadison.com,inbound,019,Americas,1 +asos.com,asos.com,inbound,019,Americas,0 +assembla.com,assembla.com,inbound,019,Americas,1 +astrology.com,astrology.com,inbound,019,Americas,0 +astrology.com,hsnlmailsvc.com,inbound,019,Americas,0 +astrology.com,webstakes.com,inbound,019,Americas,0 +astrology.com,wsafmailsvc.com,inbound,019,Americas,0 +atlassian.net,uc-inf.net,inbound,019,Americas,1 +atrapalo.cl,atrapalo.com,inbound,019,Americas,0 +atrapalo.com,atrapalo.com,inbound,019,Americas,0 +att-mail.com,att-mail.com,inbound,019,Americas,1.9e-05 +att-mail.com,att.com,inbound,019,Americas,0.999966 +att.net,att.net,outbound,019,Americas,0.204629 +att.net,mycingular.net,inbound,019,Americas,0.00021 +att.net,yahoo.{...},inbound,019,Americas,1 +australiagsm.net,australiagsm.net,inbound,019,Americas,0 +autoloop.us,loop28.com,inbound,019,Americas,0 +avaaz.org,avaaz.org,inbound,019,Americas,0 +avalanchesafelist.com,zoothost.com,inbound,019,Americas,0 +aveda.com,esteelauder.com,inbound,019,Americas,0 +avenue.com,avenue.com,inbound,019,Americas,0 +avg.com,avg.com,inbound,019,Americas,0 +avomail.com,avomail.com,inbound,019,Americas,0 +b2b-mail.net,b2b-mail.net,inbound,019,Americas,0 +b2b-mail.net,contact-list.net,inbound,019,Americas,0 +babycenter.com,rsys3.com,inbound,019,Americas,0 +babyoye.com,babyoye.com,inbound,019,Americas,0.011564 +badoo.com,monopost.com,inbound,019,Americas,1 +baligam.co.il,baligam.co.il,inbound,019,Americas,1 +banamex.com,ibrands.es,inbound,019,Americas,0 +bancoahorrofamsa.com,avantel.net.mx,inbound,019,Americas,1 +bancomercorreo.com,bancomercorreo.com,inbound,019,Americas,0 +bandsintown.com,bandsintown.com,inbound,019,Americas,1 +barclaycard.co.uk,barclays.co.uk,inbound,019,Americas,0 +barenecessities.com,barenecessities.com,inbound,019,Americas,0 +barleyment.ca,barleyment.ca,inbound,019,Americas,0 +barneys.com,barneys.com,inbound,019,Americas,0 +baseballsavings.com,baseballsavings.com,inbound,019,Americas,0 +basspronews.com,basspronews.com,inbound,019,Americas,0 +bathandbodyworks.com,bathandbodyworks.com,inbound,019,Americas,0 +baublebar.com,baublebar.com,inbound,019,Americas,0.011219 +bayt.com,bayt.com,inbound,019,Americas,2e-06 +bcp.com.pe,bcp.com.pe,inbound,019,Americas,1 +bellsouth.net,att.net,outbound,019,Americas,0 +bellsouth.net,yahoo.{...},inbound,019,Americas,1 +bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +bespokeoffers.co.uk,chtah.net,inbound,019,Americas,0 +bestbuy.ca,bestbuy.ca,inbound,019,Americas,0 +bestdealsforyou.in,elabs5.com,inbound,019,Americas,0 +bevmo.com,bevmo.com,inbound,019,Americas,0.000967 +bhcosmetics.com,bronto.com,inbound,019,Americas,0 +bhg.com,meredith.com,inbound,019,Americas,0 +bigfishgames.com,bigfishgames.com,inbound,019,Americas,0 +biglist.com,biglist.com,inbound,019,Americas,0 +bigtent.com,carezen.net,inbound,019,Americas,0 +bionexo.com,bionexo.com.br,inbound,019,Americas,0.999594 +birthdayalarm.com,monkeyinferno.net,inbound,019,Americas,0 +bitbucket.org,bitbucket.org,inbound,019,Americas,0 +bitlysupport.com,mailgun.info,inbound,019,Americas,1 +bitlysupport.com,mailgun.us,inbound,019,Americas,1 +bitslane.email,bitslane.email,inbound,019,Americas,0 +bitstatement.org,bitstatement.org,inbound,019,Americas,1 +bizjournals.com,bizjournals.com,inbound,019,Americas,0 +bizmailtoday.com,bizmailtoday.com,inbound,019,Americas,0 +bjs.com,bjs.com,inbound,019,Americas,0 +blablacar.com,blablacar.com,inbound,019,Americas,1 +blackberry.com,blackberry.com,inbound,019,Americas,0 +blackboard.com,blackboard.com,inbound,019,Americas,1 +blissworld.com,lstrk.net,inbound,019,Americas,1 +bloomingdales.com,bloomingdales.com,inbound,019,Americas,0 +bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,019,Americas,0 +bluehost.com,bluehost.com,inbound,019,Americas,0.000971 +bluehost.com,hostmonster.com,inbound,019,Americas,0 +bluehost.com,unifiedlayer.com,inbound,019,Americas,0 +blueshellgames.com,blueshellgames.com,inbound,019,Americas,0 +bluestatedigital.com,bluestatedigital.com,inbound,019,Americas,0 +bluestonemx.com,bluestonemx.com,inbound,019,Americas,1 +bm23.com,bronto.com,inbound,019,Americas,0 +bm324.com,bronto.com,inbound,019,Americas,0 +bmdeda99.com,bmdeda99.com,inbound,019,Americas,0 +bn.com,bn.com,inbound,019,Americas,0 +bnetmail.com,bnetmail.com,inbound,019,Americas,0 +bol.com.br,bol.com.br,inbound,019,Americas,0 +bol.com.br,bol.com.br,outbound,019,Americas,0 +boletinrenuevo.com,boletinrenuevo.com,inbound,019,Americas,0 +bomnegocio.com,bomnegocio.com,inbound,019,Americas,0.588708 +bonobos.com,bronto.com,inbound,019,Americas,0 +bookbub.com,bookbub.com,inbound,019,Americas,1 +booking.com,booking.com,inbound,019,Americas,1 +bookingbuddy.com,smartertravelmedia.com,inbound,019,Americas,0.000487 +boomtownroi.com,boomtownroi.com,inbound,019,Americas,0 +boscovs.com,boscovs.com,inbound,019,Americas,0 +bostonproper.com,bostonproper.com,inbound,019,Americas,0 +boutiquesecret.com,chtah.net,inbound,019,Americas,0 +bradfordexchange.com,bradfordexchange.com,inbound,019,Americas,0 +bradsdeals.com,bradsdeals.com,inbound,019,Americas,0 +brassring.com,brassring.com,inbound,019,Americas,1 +briantracyintl.com,briantracyintl.com,inbound,019,Americas,0 +brijj.com,brijj.com,inbound,019,Americas,0 +brincltd.com,brincltd.com,inbound,019,Americas,0 +bronto.com,bronto.com,inbound,019,Americas,0 +bsf01.com,bsftransmit33.com,inbound,019,Americas,0 +buffalo.edu,buffalo.edu,inbound,019,Americas,0.001059 +bumeran.com,bumeran.com,inbound,019,Americas,0 +burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,019,Americas,0 +burton.co.uk,burton.co.uk,inbound,019,Americas,0 +buscojobs.com,amazonaws.com,inbound,019,Americas,0 +buy123.com.tw,buy123.com.tw,inbound,019,Americas,1 +bzm.mobi,nmsrv.com,inbound,019,Americas,1 +c21stores.com,c21stores.com,inbound,019,Americas,0 +ca.gov,ca.gov,inbound,019,Americas,0.702758 +cafepress.com,cafepress.com,inbound,019,Americas,0.000778 +caixa.gov.br,caixa.gov.br,inbound,019,Americas,0 +californiapsychicsemail.com,californiapsychicsemail.com,inbound,019,Americas,0 +calottery.com,calottery.com,inbound,019,Americas,0.999426 +camel.com,rjrsignup.com,inbound,019,Americas,0 +canadianvisaexpert.net,canadianvisaexpert.net,inbound,019,Americas,0 +cancer.org,delivery.net,inbound,019,Americas,0 +capillary.co.in,capillary.co.in,inbound,019,Americas,1 +capitalone.com,bigfootinteractive.com,inbound,019,Americas,0 +capitalone360.com,ingdirect.com,inbound,019,Americas,0 +care.com,care.com,inbound,019,Americas,0 +care2.com,care2.com,inbound,019,Americas,0 +careerage.com,careerage.com,inbound,019,Americas,0 +careerflash.net,careerflash.net,inbound,019,Americas,1 +carmamail.com,carmamail.com,inbound,019,Americas,0 +carnivalfunmail.com,carnivalfunmail.com,inbound,019,Americas,0 +carolsdaughter.com,carolsdaughter.com,inbound,019,Americas,0 +carwale.com,carwale.com,inbound,019,Americas,0 +case.edu,cwru.edu,inbound,019,Americas,0.999942 +castingnetworks.com,castingnetworks.com,inbound,019,Americas,0.000102 +causes.com,causes.com,inbound,019,Americas,1 +cb2.com,cb2.com,inbound,019,Americas,0 +cbsig.net,cbsig.net,inbound,019,Americas,1 +ccbchurch.com,ccbchurch.com,inbound,019,Americas,1 +ccialerts.com,ccialerts.com,inbound,019,Americas,0 +ccs.com,footlocker.com,inbound,019,Americas,2.5e-05 +cenlat.com,cenlat.com,inbound,019,Americas,0.064459 +cerberusapp.com,cerberusapp.com,inbound,019,Americas,1 +cfmvmail.com,cfmvmail.com,inbound,019,Americas,0 +chabad.org,chabad.org,inbound,019,Americas,0 +champssports.com,footlocker.com,inbound,019,Americas,0.000131 +change.org,change.org,inbound,019,Americas,1 +charlestyrwhitt.com,charlestyrwhitt.com,inbound,019,Americas,0 +charter.net,charter.net,inbound,019,Americas,0 +charter.net,charter.net,outbound,019,Americas,0 +chase.com,jpmchase.com,inbound,019,Americas,0.999999 +chatcitynotifications.com,chatcitynotifications.com,inbound,019,Americas,0 +chaturbate.com,chaturbate.com,inbound,019,Americas,1 +cheapairmailer.com,cheapairmailer.com,inbound,019,Americas,0 +check.me,check.me,inbound,019,Americas,0 +cheekylovers.com,ropot.net,inbound,019,Americas,0 +chess.com,chess.com,inbound,019,Americas,1 +chicagotribune.com,latimes.com,inbound,019,Americas,0 +chicos.com,chicos.com,inbound,019,Americas,0.000156 +childrensplace.com,childrensplace.com,inbound,019,Americas,0 +christianbook.com,christianbook.com,inbound,019,Americas,1 +christianmingle.com,christianmingle.com,inbound,019,Americas,0 +chtah.com,chtah.net,inbound,019,Americas,0 +chtah.net,chtah.net,inbound,019,Americas,0 +cincghq.com,searchhomesingta.com,inbound,019,Americas,1 +circleofmomsmail.com,circleofmomsmail.com,inbound,019,Americas,0 +citibank.com,bigfootinteractive.com,inbound,019,Americas,0 +ck.com,ck.com,inbound,019,Americas,0 +clarks.com,clarks.com,inbound,019,Americas,0 +clickdimensions.com,clickdimensions.com,inbound,019,Americas,0 +clickexperts.net,clickexperts.net,inbound,019,Americas,0 +clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,019,Americas,0 +climber.com,climber.com,inbound,019,Americas,0 +clinique.com,esteelauder.com,inbound,019,Americas,0 +clubcupon.com.ar,clubcupon.com.ar,inbound,019,Americas,0 +cmm01.com,coremotivesmarketing.com,inbound,019,Americas,0 +coach.com,delivery.net,inbound,019,Americas,0 +codeproject.com,codeproject.com,inbound,019,Americas,0 +coldwatercreek.com,coldwatercreek.com,inbound,019,Americas,0 +collectionsetc.com,collectionsetc.com,inbound,019,Americas,0 +columbia.edu,columbia.edu,inbound,019,Americas,0.762355 +comcast.net,comcast.net,inbound,019,Americas,0.919244 +comcast.net,comcast.net,outbound,019,Americas,0.999998 +comenity.net,alldata.net,inbound,019,Americas,1 +comenity.net,bigfootinteractive.com,inbound,019,Americas,0 +commonfloor.com,commonfloor.com,inbound,019,Americas,1 +compute.internal,amazonaws.com,inbound,019,Americas,0.875148 +computerworld.com,computerworld.com,inbound,019,Americas,0 +comunicacaodemkt.com,locaweb.com.br,inbound,019,Americas,0 +constantcontact.com,confirmedcc.com,inbound,019,Americas,0 +constantcontact.com,constantcontact.com,inbound,019,Americas,5.3e-05 +constantcontact.com,postini.com,inbound,019,Americas,0.042128 +containerstore.com,containerstore.com,inbound,019,Americas,0 +convio.net,convio.net,inbound,019,Americas,0 +coremotivesmarketing.com,coremotivesmarketing.com,inbound,019,Americas,0 +cornell.edu,cornell.edu,inbound,019,Americas,0.192285 +correosocc.com,correosocc.com,inbound,019,Americas,1 +costco.co.uk,costco.com,inbound,019,Americas,0 +costco.com,costco.com,inbound,019,Americas,7e-06 +costcophotocenter.com,wc09.net,inbound,019,Americas,0 +costcoservices.com,costco.com,inbound,019,Americas,0 +cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,019,Americas,0 +couchsurfing.org,couchsurfing.com,inbound,019,Americas,0 +couponamama.com,couponamama.com,inbound,019,Americas,1 +cox.com,cox.com,inbound,019,Americas,0.001665 +cox.net,cox.net,inbound,019,Americas,0.009187 +cox.net,cox.net,outbound,019,Americas,0 +coyotelogistics.com,postini.com,inbound,019,Americas,0 +cp20.com,cp20.com,inbound,019,Americas,0 +cpbnc.com,cpbnc.com,inbound,019,Americas,0 +cpbnc.com,fye.com,inbound,019,Americas,0 +crackle.com,crackle.com,inbound,019,Americas,0 +craigslist.org,craigslist.org,inbound,019,Americas,0 +craigslist.org,craigslist.org,outbound,019,Americas,1 +crashlytics.com,crashlytics.com,inbound,019,Americas,1 +crashlytics.com,sendgrid.net,inbound,019,Americas,1 +crateandbarrel.com,crateandbarrel.com,inbound,019,Americas,0 +creationsrewards.net,creationsrewards.net,inbound,019,Americas,0 +creditkarma.com,creditkarma.com,inbound,019,Americas,1 +crosswalkmail.com,crosswalkmail.com,inbound,019,Americas,0 +crowdcut.com,crowdcut.com,inbound,019,Americas,1 +crunchyroll.com,crunchyroll.com,inbound,019,Americas,0 +cumulusdist.net,cumulusdist.net,inbound,019,Americas,0 +cuponatic.com.pe,cuponatic.com.pe,inbound,019,Americas,1 +cuponicamail.com,fnbox.com,inbound,019,Americas,0 +curbednetwork.com,curbednetwork.com,inbound,019,Americas,1 +cuspemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +custombriefings.com,custombriefings.com,inbound,019,Americas,0 +customercenter.net,customercenter.net,inbound,019,Americas,0.996453 +customeriomail.com,customeriomail.com,inbound,019,Americas,1 +cxomedia.com,cxomedia.com,inbound,019,Americas,0 +cybercoders.com,cybercoders.com,inbound,019,Americas,0 +dafiti.cl,dafiti.cl,inbound,019,Americas,0 +dailyhoroscope.com,tarot.com,inbound,019,Americas,0 +datehookup.com,datehookup.com,inbound,019,Americas,0 +datingvipnotifications.com,datingvipnotifications.com,inbound,019,Americas,0 +daviacalendar.com,daviacalendar.com,inbound,019,Americas,1 +davidsbridal.com,davidsbridal.com,inbound,019,Americas,0 +davidstea.com,bronto.com,inbound,019,Americas,0 +daz3d.com,bronto.com,inbound,019,Americas,0 +dealersocket.com,dealersocket.com,inbound,019,Americas,0 +dealsaver.com,secondstreetmedia.com,inbound,019,Americas,0.999823 +debshops.com,lstrk.net,inbound,019,Americas,1 +deliasshopemail.com,deliasshopemail.com,inbound,019,Americas,0 +delivery.net,delivery.net,inbound,019,Americas,0 +delivery.net,m0.net,inbound,019,Americas,0 +dell.com,bfi0.com,inbound,019,Americas,0 +dentalsenders.com,dentalsenders.com,inbound,019,Americas,0 +descontos.pt,descontos.pt,inbound,019,Americas,0 +designerapparel.com,myperfectsale.com,inbound,019,Americas,1 +despegar.com,despegar.com,inbound,019,Americas,0 +dhgate.com,chtah.net,inbound,019,Americas,0 +dice.com,dice.com,inbound,019,Americas,0 +digitalmailer.com,digitalmailer.com,inbound,019,Americas,0 +digitalmedia-comunicacion.es,chtah.net,inbound,019,Americas,0 +digitalromanceinc.com,digitalromanceinc.com,inbound,019,Americas,1 +directv.com,directv.com,inbound,019,Americas,0.026649 +discover.com,discoverfinancial.com,inbound,019,Americas,1 +disparadordeemails.com,locaweb.com.br,inbound,019,Americas,0 +disqus.net,disqus.net,inbound,019,Americas,0 +dn.net,naukri.com,inbound,019,Americas,0 +docusign.net,docusign.net,inbound,019,Americas,0.985435 +donationnet.net,donationnet.net,inbound,019,Americas,0 +dorothyperkins.com,dorothyperkins.com,inbound,019,Americas,0 +dowjones.info,dowjones.info,inbound,019,Americas,0 +dptagent.biz,dptagent.biz,inbound,019,Americas,0 +dptagent.net,dptagent.net,inbound,019,Americas,0 +dreamhost.com,dreamhost.com,inbound,019,Americas,0 +dreamwidth.org,dreamwidth.org,inbound,019,Americas,0 +drhinternet.net,drhinternet.net,inbound,019,Americas,0 +driftem.com,emce2.in,inbound,019,Americas,0 +driftem.com,mailurja.com,inbound,019,Americas,0 +dropbox.com,dropbox.com,inbound,019,Americas,1 +dropboxmail.com,dropbox.com,inbound,019,Americas,1 +dsw.com,dsw.com,inbound,019,Americas,0 +ducks.org,uptilt.com,inbound,019,Americas,0 +duke.edu,duke.edu,inbound,019,Americas,0.308101 +dukecareers.com,dukecareers.com,inbound,019,Americas,1 +dvor.com,dvor.com,inbound,019,Americas,0 +dynamite-safelist.com,thomas-j-brown.com,inbound,019,Americas,0 +dynect-mailer.net,sendlabs.com,inbound,019,Americas,0 +e-activist.com,e-activist.com,inbound,019,Americas,0 +e-beallsonline.com,e-stagestores.com,inbound,019,Americas,0 +e-costco.mx,costco.com,inbound,019,Americas,0 +e-goodysonline.com,e-stagestores.com,inbound,019,Americas,0 +e-peebles.com,e-stagestores.com,inbound,019,Americas,0 +e-rewards.net,e-rewards.net,inbound,019,Americas,1 +e-stagestores.com,e-stagestores.com,inbound,019,Americas,0 +e-travelclub.es,e-travelclub.es,inbound,019,Americas,0 +e2ma.net,e2ma.net,inbound,019,Americas,1 +ea.com,ea.com,inbound,019,Americas,0.001232 +eaccess.net,postini.com,inbound,019,Americas,0 +earn-e-miles.com,earn-e-miles.com,inbound,019,Americas,0 +earnerslist.com,traxweb.net,inbound,019,Americas,9e-06 +earthfare-email.com,edclient2.com,inbound,019,Americas,0 +earthlink.net,earthlink.net,inbound,019,Americas,0.031648 +earthlink.net,earthlink.net,outbound,019,Americas,0 +eastbay.com,footlocker.com,inbound,019,Americas,0 +easyhealthoptions.com,easyhealthoptions.com,inbound,019,Americas,1 +easyroommate.com,easyroommate.com,inbound,019,Americas,0 +ebates.com,bfi0.com,inbound,019,Americas,0 +ebay.{...},ebay.{...},inbound,019,Americas,0.99941 +ebizac2.com,ebizac2.com,inbound,019,Americas,0 +eblastengine.com,secondstreetmedia.com,inbound,019,Americas,0.999827 +ec2.internal,amazonaws.com,inbound,019,Americas,0.769271 +ecasend.com,ecasend.com,inbound,019,Americas,0 +ed.gov,leepfrog.com,inbound,019,Americas,0.106381 +ed10.net,ed10.com,inbound,019,Americas,0 +edirect1.com,ivytech.edu,inbound,019,Americas,0 +edmodo.com,edmodo.com,inbound,019,Americas,1.1e-05 +educationzone.co.in,iaires.com,inbound,019,Americas,0 +effectivesafelist.com,zoothost.com,inbound,019,Americas,0 +eharmony.com,eharmony.com,inbound,019,Americas,1e-06 +eigbox.net,eigbox.net,inbound,019,Americas,0 +elabs3.com,elabs3.com,inbound,019,Americas,0 +elabs3.com,meritline.com,inbound,019,Americas,0 +elabs5.com,elabs5.com,inbound,019,Americas,0 +elabs6.com,elabs6.com,inbound,019,Americas,0 +elanceonline.com,elanceonline.com,inbound,019,Americas,0 +eleadtrack.net,eleadtrack.net,inbound,019,Americas,0 +elitesafelist.com,elitesafelist.com,inbound,019,Americas,0 +email-cooking.com,email-cooking.com,inbound,019,Americas,0 +email-galls.com,email-galls.com,inbound,019,Americas,1 +email-od.com,smtprelayserver.com,inbound,019,Americas,0.999779 +email-ticketdada.com,email-ticketdada.com,inbound,019,Americas,1 +email4-beyond.com,email4-beyond.com,inbound,019,Americas,0 +emailcounts.com,secureserver.net,inbound,019,Americas,0 +emaildir2.com,emaildirect.net,inbound,019,Americas,0 +emaildir2.com,espsnd.com,inbound,019,Americas,0 +emailnotify.net,emailnotify.net,inbound,019,Americas,0.961424 +emailsbancoestado.cl,emailsbancoestado.cl,inbound,019,Americas,0 +emailsripley.cl,etarget.cl,inbound,019,Americas,0 +embluejet.com,embluejet.com,inbound,019,Americas,0 +embluejet.com,emblueuser.com,inbound,019,Americas,0 +emcsend.com,emcsend.com,inbound,019,Americas,0 +emergencyemail.org,emergencyemail.org,inbound,019,Americas,0 +emktsender.net,locaweb.com.br,inbound,019,Americas,0 +emma.cl,emma.cl,inbound,019,Americas,0.996895 +emobile.ad.jp,postini.com,inbound,019,Americas,0 +employboard.com,employboard.com,inbound,019,Americas,1 +entregadeemails.com,locaweb.com.br,inbound,019,Americas,0 +entregadordecampanhas.net,locaweb.com.br,inbound,019,Americas,0 +enviodecampanhas.net,locaweb.com.br,inbound,019,Americas,0 +enviodemkt.com.br,locaweb.com.br,inbound,019,Americas,0 +epriority.com,epriority.com,inbound,019,Americas,0 +equifax.com,equifax.com,inbound,019,Americas,1 +equussafelist.com,equussafelist.com,inbound,019,Americas,0.000265 +esri.com,esri.com,inbound,019,Americas,0.983104 +esteelauder.com,esteelauder.com,inbound,019,Americas,0 +evanguard.com,evanguard.com,inbound,019,Americas,0 +eventbrite.com,eventbrite.com,inbound,019,Americas,0 +eversavelocal.com,eversavelocal.com,inbound,019,Americas,0 +everydayhealthinc.com,waterfrontmedia.net,inbound,019,Americas,0 +everyjobforme.com,everyjobforme.com,inbound,019,Americas,0 +exchangesolutions.com,exchangesolutions.com,inbound,019,Americas,0.000143 +exec-u-net-mail.com,exec-u-net-mail.com,inbound,019,Americas,0 +exprpt.com,exprpt.com,inbound,019,Americas,0 +expvtinboxhub.net,expvtinboxhub.net,inbound,019,Americas,0 +fabletics.com,bronto.com,inbound,019,Americas,0 +facebook.com,facebook.com,inbound,019,Americas,0.719445 +facebook.com,facebook.com,outbound,019,Americas,1 +facebookappmail.com,facebook.com,inbound,019,Americas,1 +facebookmail.com,facebook.com,inbound,019,Americas,1 +facebookmail.com,postini.com,inbound,019,Americas,0.592681 +facebookmail.com,yahoo.{...},inbound,019,Americas,1 +familychristianmail.com,familychristianmail.com,inbound,019,Americas,0 +famousfootwear.com,famousfootwear.com,inbound,019,Americas,0 +fanfiction.com,fictionpress.com,inbound,019,Americas,1 +farmersonly.com,mailgun.us,inbound,019,Americas,1 +fashion2hub.in,mgenie.in,inbound,019,Americas,0 +fastgb.com,fastgb.com,inbound,019,Americas,0 +fastlistmailer.com,zoothost.com,inbound,019,Americas,0 +fastweb.com,fastweb.com,inbound,019,Americas,0 +fbi.gov,fbi.gov,inbound,019,Americas,0 +fbmta.com,fbmta.com,inbound,019,Americas,0 +fc2.com,fc2.com,inbound,019,Americas,0.000601 +fedoraproject.org,fedoraproject.org,inbound,019,Americas,0 +feedblitz.com,feedblitz.com,inbound,019,Americas,0 +fetlifemail.com,fetlifemail.com,inbound,019,Americas,0 +fibertel.com.ar,fibertel.com.ar,inbound,019,Americas,0.003898 +fidelizador.org,fidelizador.org,inbound,019,Americas,0 +findexpvtinbox.com,findexpvtinbox.com,inbound,019,Americas,0 +finishline.com,finishline.com,inbound,019,Americas,0 +firemountaingems.com,firemountaingems.com,inbound,019,Americas,0 +fisher-price.com,fisher-price.com,inbound,019,Americas,0 +fitbit.com,fitbit.com,inbound,019,Americas,1 +fitnessmagazine.com,meredith.com,inbound,019,Americas,0 +fiverr.com,fiverr.com,inbound,019,Americas,0 +flexmls.com,flexmls.com,inbound,019,Americas,0.999992 +flightaware.com,flightaware.com,inbound,019,Americas,0.020698 +flipkart.com,flipkart.com,inbound,019,Americas,1 +flirt.com,ropot.net,inbound,019,Americas,0 +flirthookup.com,flirthookup.com,inbound,019,Americas,1 +flyceb.com,flyceb.com,inbound,019,Americas,0 +flyfrontier.com,flyfrontier.com,inbound,019,Americas,0 +foolsubs.com,foolcs.com,inbound,019,Americas,0 +foolsubs.com,foolsubs.com,inbound,019,Americas,0 +footaction.com,footlocker.com,inbound,019,Americas,0 +footlocker.com,footlocker.com,inbound,019,Americas,0.000605 +forever21.com,forever21.com,inbound,019,Americas,0 +fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,019,Americas,0 +foursquare.com,foursquare.com,inbound,019,Americas,1 +foxnews.com,foxnews.com,inbound,019,Americas,0.0139 +fragrancenet.com,fragrancenet.com,inbound,019,Americas,0.000427 +francescas.com,bronto.com,inbound,019,Americas,0 +freeadsmailer.com,zoothost.com,inbound,019,Americas,0 +freebeesafelist.com,zoothost.com,inbound,019,Americas,0 +freebizmag.com,delivery.net,inbound,019,Americas,0 +freebsd.org,freebsd.org,inbound,019,Americas,0.999845 +freecycle.org,freecycle.org,inbound,019,Americas,1 +freedesktop.org,freedesktop.org,inbound,019,Americas,0 +freeflys.com,freeflys.com,inbound,019,Americas,0 +freelancer.com,freelancer.com,inbound,019,Americas,0 +freelancer.com,freelancernotify.com,inbound,019,Americas,0 +freelancer.com,getafreelancer.com,inbound,019,Americas,0 +freelists.org,iquest.net,inbound,019,Americas,0 +freelotto.com,plasmanetinc.com,inbound,019,Americas,0 +freepeople.com,freepeople.com,inbound,019,Americas,0 +freesafelistking.com,zoothost.com,inbound,019,Americas,0 +freshdesk.com,freshdesk.com,inbound,019,Americas,1 +freshers2015.com,secureserver.net,inbound,019,Americas,0 +freshlatesave.com,freshlatesave.com,inbound,019,Americas,1 +friskone.com,mailurja.com,inbound,019,Americas,0 +frontsight.com,frontsight.com,inbound,019,Americas,0 +frys.com,frys.com,inbound,019,Americas,0.00388 +frysmail.com,frysmail.com,inbound,019,Americas,0 +fspeletters.com,agorapub.co.uk,inbound,019,Americas,0 +fuelrewards.com,britecast.com,inbound,019,Americas,0 +futureshop.com,futureshop.com,inbound,019,Americas,0 +gaiaonline.com,gaiaonline.com,inbound,019,Americas,0 +gamehouse.com,gamehouse.com,inbound,019,Americas,0 +gbyguess.com,guess.com,inbound,019,Americas,0.001787 +gemoney.com,rsys1.com,inbound,019,Americas,0 +generalmills.com,boxtops4education.com,inbound,019,Americas,0 +generalmills.com,pillsbury.com,inbound,019,Americas,0 +gentoo.org,gentoo.org,inbound,019,Americas,1 +get-me-jobs.com,get-me-jobs.com,inbound,019,Americas,0 +gethired.com,gethired.com,inbound,019,Americas,1 +getitfree.us,getitfree.us,inbound,019,Americas,0 +getpaidsolutions.com,getpaidsolutions.com,inbound,019,Americas,1 +getpocket.com,bronto.com,inbound,019,Americas,0 +ghin.com,ghinconnect.com,inbound,019,Americas,0 +ghup.in,mgenie.in,inbound,019,Americas,0 +gillyhicks-email.com,abercrombie-email.com,inbound,019,Americas,0 +github.com,github.com,inbound,019,Americas,1 +github.com,postini.com,inbound,019,Americas,0.837872 +glassdoor.com,glassdoor.com,inbound,019,Americas,1 +glasses.com,glasses.com,inbound,019,Americas,0 +gliq.com,gliq.com,inbound,019,Americas,0.99852 +globalsafelist.com,globalsafelist.com,inbound,019,Americas,0 +globaltestmarket.com,globaltestmarket.com,inbound,019,Americas,0 +gmail.com,amazonaws.com,inbound,019,Americas,0.994381 +gmail.com,anteldata.net.uy,inbound,019,Americas,0.998653 +gmail.com,bell.ca,inbound,019,Americas,0.992481 +gmail.com,bellsouth.net,inbound,019,Americas,0.9996 +gmail.com,blackberry.com,inbound,019,Americas,0.992207 +gmail.com,brasiltelecom.net.br,inbound,019,Americas,0.99995 +gmail.com,centurytel.net,inbound,019,Americas,0.999415 +gmail.com,cgocable.net,inbound,019,Americas,0.99829 +gmail.com,charter.com,inbound,019,Americas,0.999109 +gmail.com,claro.net.br,inbound,019,Americas,1 +gmail.com,comcast.net,inbound,019,Americas,0.999621 +gmail.com,comcastbusiness.net,inbound,019,Americas,0.985462 +gmail.com,cox.net,inbound,019,Americas,0.962619 +gmail.com,embarqhsd.net,inbound,019,Americas,0.999554 +gmail.com,franchiseindia.com,inbound,019,Americas,1 +gmail.com,frontiernet.net,inbound,019,Americas,0.995233 +gmail.com,gvt.net.br,inbound,019,Americas,0.999513 +gmail.com,lorexddns.net,inbound,019,Americas,0 +gmail.com,majesticmoneymailer.com,inbound,019,Americas,1 +gmail.com,mchsi.com,inbound,019,Americas,0.999951 +gmail.com,movistar.cl,inbound,019,Americas,0.999361 +gmail.com,mycingular.net,inbound,019,Americas,0.999918 +gmail.com,myvzw.com,inbound,019,Americas,0.99992 +gmail.com,naukri.com,inbound,019,Americas,0.000998 +gmail.com,optonline.net,inbound,019,Americas,0.999776 +gmail.com,postini.com,inbound,019,Americas,0.650888 +gmail.com,qwest.net,inbound,019,Americas,0.997574 +gmail.com,rcn.com,inbound,019,Americas,0.999275 +gmail.com,rogers.com,inbound,019,Americas,0.999916 +gmail.com,rr.com,inbound,019,Americas,0.986894 +gmail.com,sbcglobal.net,inbound,019,Americas,0.998817 +gmail.com,shawcable.net,inbound,019,Americas,0.999998 +gmail.com,spcsdns.net,inbound,019,Americas,0.999998 +gmail.com,suddenlink.net,inbound,019,Americas,0.961593 +gmail.com,telecom.net.ar,inbound,019,Americas,0.999664 +gmail.com,telesp.net.br,inbound,019,Americas,0.999743 +gmail.com,telus.com,inbound,019,Americas,1 +gmail.com,telus.net,inbound,019,Americas,0.974663 +gmail.com,tmodns.net,inbound,019,Americas,1 +gmail.com,veloxzone.com.br,inbound,019,Americas,0.999969 +gmail.com,verizon.net,inbound,019,Americas,0.990214 +gmail.com,videotron.ca,inbound,019,Americas,0.967196 +gmail.com,vtr.net,inbound,019,Americas,0.999072 +gmail.com,websitewelcome.com,inbound,019,Americas,1 +gmail.com,wideopenwest.com,inbound,019,Americas,0.999729 +gmail.com,windstream.net,inbound,019,Americas,0.951847 +gmail.com,yahoo.{...},inbound,019,Americas,0.999992 +gmail.com,zoothost.com,inbound,019,Americas,0.033858 +gob.ar,gob.ar,inbound,019,Americas,0.160006 +gob.ec,gob.ec,inbound,019,Americas,0.623867 +godaddy.com,secureserver.net,inbound,019,Americas,0 +gogecapital.com,rsys1.com,inbound,019,Americas,0 +goldenopsafelist.com,zoothost.com,inbound,019,Americas,0 +goldstar.com,goldstar.com,inbound,019,Americas,1 +golfmnb.com,golfmnb.com,inbound,019,Americas,0 +google.com,postini.com,inbound,019,Americas,0.584887 +gopusamedia.com,gopusamedia.com,inbound,019,Americas,0 +govdelivery.com,govdelivery.com,inbound,019,Americas,0 +governmentjobs.com,governmentjobs.com,inbound,019,Americas,0 +gpmailer.com.br,parperfeito.com,inbound,019,Americas,0 +grassrootsaction.com,grassfire.net,inbound,019,Americas,0 +greatergood.com,greatergood.com,inbound,019,Americas,0 +groupon.{...},chtah.net,inbound,019,Americas,0 +groupon.{...},groupon.{...},inbound,019,Americas,1.0 +groupon.{...},postini.com,inbound,019,Americas,0.886672 +grouponmail.{...},grouponmail.{...},inbound,019,Americas,0 +grupos.com.br,grupos.com.br,inbound,019,Americas,0 +guess.ca,guess.com,inbound,019,Americas,0.000807 +guess.com,guess.com,inbound,019,Americas,0.003805 +guessfactory.com,guess.com,inbound,019,Americas,0.00164 +gustazos.com,cityoferta.com,inbound,019,Americas,1 +hallmark.com,hallmark.com,inbound,019,Americas,0 +hannaandersson.com,hannaandersson.com,inbound,019,Americas,0.013533 +harristeetermail.com,harristeetermail.com,inbound,019,Americas,0.99673 +harvard.edu,harvard.edu,inbound,019,Americas,0.274906 +hayneedle.com,hayneedle.com,inbound,019,Americas,0 +helpareporter.net,helpareporter.com,inbound,019,Americas,0 +herculist.com,herculist.com,inbound,019,Americas,0 +hilton.com,hiltonemail.com,inbound,019,Americas,0 +hipchat.com,hipchat.com,inbound,019,Americas,1 +hispavista.com,hispavista.com,inbound,019,Americas,0 +hollister-email.com,abercrombie-email.com,inbound,019,Americas,0 +homeaway.com,haspf.com,inbound,019,Americas,0 +homedecorators.com,homedecorators.com,inbound,019,Americas,0 +homedepot.com,homedepot.com,inbound,019,Americas,1 +hootsuite.com,hootsuite.com,inbound,019,Americas,1 +horchowemail.com,horchowemail.com,inbound,019,Americas,0 +hostelworld.com,bronto.com,inbound,019,Americas,0 +hostgator.com,hostgator.com,inbound,019,Americas,0.98061 +hostgator.com,websitewelcome.com,inbound,019,Americas,1 +hotmail.{...},hotmail.{...},inbound,019,Americas,1 +hotmail.{...},hotmail.{...},outbound,019,Americas,1 +hotmail.{...},postini.com,inbound,019,Americas,0.723143 +hotornot.com,monopost.com,inbound,019,Americas,1 +hotschedules.com,hotschedules.com,inbound,019,Americas,0 +house.gov,house.gov,inbound,019,Americas,0.999966 +houseoffraser.co.uk,houseoffraser.co.uk,inbound,019,Americas,0 +houzz.com,houzz.com,inbound,019,Americas,1 +hubspot.com,hubspot.com,inbound,019,Americas,1 +hungry-girl.com,hungry-girl.com,inbound,019,Americas,0 +iamlgnd2.com,iamlgnd2.com,inbound,019,Americas,1 +ibsys.com,ibsys.com,inbound,019,Americas,9e-05 +icbc.com.ar,clickexperts.net,inbound,019,Americas,0 +icbc.com.ar,standardbank.com.ar,inbound,019,Americas,0 +icims.com,icims.com,inbound,019,Americas,0.999939 +icors.org,lsoft.us,inbound,019,Americas,0 +icpbounce.com,icpbounce.com,inbound,019,Americas,0 +idc.email,nmsrv.com,inbound,019,Americas,1 +idgconnect-resources.com,idgconnect-resources.com,inbound,019,Americas,0 +ieee.org,ieee.org,inbound,019,Americas,0.999912 +ig.com.br,ig.com.br,inbound,019,Americas,0 +ig.com.br,ig.com.br,outbound,019,Americas,0 +igot-mails.com,zoothost.com,inbound,019,Americas,0 +iimjobs.com,iimjobs.com,inbound,019,Americas,1 +ikmultimedianews.com,ikmultimedianews.com,inbound,019,Americas,0.993319 +illinois.edu,illinois.edu,inbound,019,Americas,0.866481 +imageshost.ca,imageshost.ca,inbound,019,Americas,0 +imakenews.net,imakenews.com,inbound,019,Americas,0 +imo.im,imo.im,inbound,019,Americas,1 +imodules.com,imodules.com,inbound,019,Americas,0 +imvu.com,imvu.com,inbound,019,Americas,7e-06 +inboxdollars.com,inboxdollars.com,inbound,019,Americas,0 +inboxfirst.com,inboxfirst.com,inbound,019,Americas,0 +inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,019,Americas,0.999877 +inboxpays.com,inboxpays.com,inbound,019,Americas,0 +inboxpounds.co.uk,inboxpounds.co.uk,inbound,019,Americas,0 +indeed.com,indeed.com,inbound,019,Americas,0.000121 +indeedemail.com,indeedemail.com,inbound,019,Americas,0 +independentlivingbullion.com,independentlivingbullion.com,inbound,019,Americas,0 +infobradesco.com.br,infobradesco.com.br,inbound,019,Americas,0 +infojobs.com.br,anuntis.com,inbound,019,Americas,0 +infomoney.com.br,infomoney.com.br,inbound,019,Americas,1 +informz.net,informz.net,inbound,019,Americas,0 +infradead.org,infradead.org,inbound,019,Americas,1 +inman.com,inman.com,inbound,019,Americas,0 +innovyx.net,innovyx.net,inbound,019,Americas,0 +insidehook.com,sailthru.com,inbound,019,Americas,0 +instagram.com,facebook.com,inbound,019,Americas,1 +instantprofitlist.com,screenshotads.com,inbound,019,Americas,0 +interac.ca,certapay.com,inbound,019,Americas,0 +interactivebrokers.com,interactivebrokers.com,inbound,019,Americas,1 +interactiverealtyservices.com,interactiverealtyservices.com,inbound,019,Americas,0 +intercom.io,mailgun.info,inbound,019,Americas,1 +interealty.net,interealty.net,inbound,019,Americas,1 +interweave.com,interweave.com,inbound,019,Americas,0 +intliv2.net,internationalliving.com,inbound,019,Americas,0 +invalidemail.com,taleo.net,inbound,019,Americas,1 +investopedia.com,vclk.net,inbound,019,Americas,0.999999 +investorplace.com,investorplace.com,inbound,019,Americas,0.995093 +irctcshopping.com,chtah.net,inbound,019,Americas,0 +iridium.com,iridium.com,inbound,019,Americas,0.997882 +isendservice.com.br,isendservice.com.br,inbound,019,Americas,0 +itau-unibanco.com.br,itau.com.br,inbound,019,Americas,0 +ittoolbox.com,ittoolbox.com,inbound,019,Americas,0 +ittoolbox.com,toolbox.com,inbound,019,Americas,0 +itwhitepapers.com,itwhitepapers.com,inbound,019,Americas,0 +iwantoneofthose.com,thehut.com,inbound,019,Americas,0 +ixs1.net,ixs1.net,inbound,019,Americas,0.005131 +jcpenney.com,jcpenney.com,inbound,019,Americas,9e-06 +jeevansathi.com,jeevansathi.com,inbound,019,Americas,0 +jetsetter.com,smartertravelmedia.com,inbound,019,Americas,0.041888 +jibjab.com,storybots.com,inbound,019,Americas,0 +jira.com,uc-inf.net,inbound,019,Americas,1 +jobscentral.com.sg,mailgun.net,inbound,019,Americas,1 +jobson.com,jobsonmail.com,inbound,019,Americas,0 +jobsradar.com,jobsradar.com,inbound,019,Americas,0 +jockeycomfort.com,jockeycomfort.com,inbound,019,Americas,0 +johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,019,Americas,0 +jomashop.com,lstrk.net,inbound,019,Americas,1 +josbank.com,josbank.com,inbound,019,Americas,0 +jossandmain.com,jossandmain.com,inbound,019,Americas,0 +jtv.com,jtv.com,inbound,019,Americas,0 +juno.com,untd.com,inbound,019,Americas,0 +juno.com,untd.com,outbound,019,Americas,0 +justdial.com,mailurja.com,inbound,019,Americas,0 +justfab.com,bronto.com,inbound,019,Americas,0 +justfab.fr,bronto.com,inbound,019,Americas,0 +keek.com,keek.com,inbound,019,Americas,1 +kernel.org,kernel.org,inbound,019,Americas,0 +kgstores.com,kgstores.com,inbound,019,Americas,0 +kickstarter.com,kickstarter.com,inbound,019,Americas,1 +kidsfootlocker.com,footlocker.com,inbound,019,Americas,0 +kik.com,kik.com,inbound,019,Americas,1 +kimblegroup.com,kimblegroup.com,inbound,019,Americas,1 +kintera.com,kintera.com,inbound,019,Americas,0 +klaviyomail.com,klaviyomail.com,inbound,019,Americas,1 +klove.com,emfbroadcasting.com,inbound,019,Americas,0 +komando.com,komando.com,inbound,019,Americas,0 +kp.org,kp.org,inbound,019,Americas,0.999975 +krogermail.com,bigfootinteractive.com,inbound,019,Americas,0 +landmarketingmailer.com,zoothost.com,inbound,019,Americas,0 +landofnod.com,landofnod.com,inbound,019,Americas,0.002525 +languagepod101.com,eclient10.com,inbound,019,Americas,0 +languagepod101.com,eddlvr.com,inbound,019,Americas,0 +languagepod101.com,ednwsltr3.com,inbound,019,Americas,0 +languagepod101.com,ednwsltr8.com,inbound,019,Americas,0 +languagepod101.com,emaildirect.net,inbound,019,Americas,0 +lasenza.com,lasenza.com,inbound,019,Americas,0 +lastcallemail.com,lastcallemail.com,inbound,019,Americas,0 +latimes.com,latimes.com,inbound,019,Americas,0 +lauraashley.com,lauraashley.com,inbound,019,Americas,0 +leftlanesports.com,auspient.com,inbound,019,Americas,0.00705 +leftlanesports.com,leftlanesports.com,inbound,019,Americas,0 +legalshieldassociate.com,legalshield.com,inbound,019,Americas,0 +lexico.com,lexico.com,inbound,019,Americas,0 +life360.com,life360.com,inbound,019,Americas,1 +lindenlab.com,lindenlab.com,inbound,019,Americas,0.999444 +linkedin.com,linkedin.com,inbound,019,Americas,0.999873 +linkedin.com,postini.com,inbound,019,Americas,0.713391 +listeneremail.net,listeneremail.net,inbound,019,Americas,0 +listia.com,listia.com,inbound,019,Americas,1 +listnerds.com,listnerds.com,inbound,019,Americas,0 +listreturn.com,zoothost.com,inbound,019,Americas,0 +listserve.com,listserve.com,inbound,019,Americas,0 +listvolta.com,listvolta.com,inbound,019,Americas,0 +listwire.com,listwire.com,inbound,019,Americas,0 +live.{...},hotmail.{...},inbound,019,Americas,1 +live.{...},hotmail.{...},outbound,019,Americas,1 +livefyre.com,andbit.net,inbound,019,Americas,1 +livescribe.com,bronto.com,inbound,019,Americas,0 +livingsocial.com,livingsocial.com,inbound,019,Americas,0 +livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,019,Americas,0 +localhires.com,localhires.com,inbound,019,Americas,1 +logitech.com,dvsops.com,inbound,019,Americas,0 +lojasmarisa.com.br,lojasmarisa.com.br,inbound,019,Americas,0 +lolsolos.com,ultimateadsites.net,inbound,019,Americas,0.999996 +lonelywifehookup.com,iverificationsystems.com,inbound,019,Americas,0 +lordandtaylor.com,lordandtaylor.com,inbound,019,Americas,0 +loveaholics.com,ropot.net,inbound,019,Americas,0 +lovelywholesale.com,lovelywholesale.com,inbound,019,Americas,1 +lrsmail.com,lrsmail.com,inbound,019,Americas,0 +lt02.net,listrak.com,inbound,019,Americas,1 +lt02.net,lstrk.net,inbound,019,Americas,1 +ltdcommodities.com,ltdcomm.net,inbound,019,Americas,0 +luckymag.com,mkt4500.com,inbound,019,Americas,0 +lulu.com,bronto.com,inbound,019,Americas,0 +lulus.com,lstrk.net,inbound,019,Americas,1 +lumosity.com,lumosity.com,inbound,019,Americas,1 +lyst.com,lyst.com,inbound,019,Americas,1 +maccosmetics.com,esteelauder.com,inbound,019,Americas,0 +macupdate.com,mailgun.info,inbound,019,Americas,1 +madmels.info,ultimateadsites.net,inbound,019,Americas,1 +madmimi.com,madmimi.com,inbound,019,Americas,0 +magicjack.com,magicjack.com,inbound,019,Americas,1 +magnetdev.com,magnetmail.net,inbound,019,Americas,0 +mail-route.com,mail-route.com,inbound,019,Americas,0 +mail-thestreet.com,mail-thestreet.com,inbound,019,Americas,0 +mail.mil,mail.mil,inbound,019,Americas,0 +mailaccurate.com,mgenie.in,inbound,019,Americas,0 +mailfacil.com.br,md02.com,inbound,019,Americas,0 +mailfeast.com,mgenie.in,inbound,019,Americas,0 +mailgun.org,mailgun.info,inbound,019,Americas,1 +mailgun.org,mailgun.net,inbound,019,Americas,1 +mailgun.org,mailgun.us,inbound,019,Americas,1 +mailingathome.net,mailingathome.net,inbound,019,Americas,0.999995 +mailjayde.com,mailjayde.com,inbound,019,Americas,0 +mailmachine1050.com,mailmachine1050.com,inbound,019,Americas,0 +mailsend1.com,mailsend6.com,inbound,019,Americas,0 +mailsender.com.br,mailsender.com.br,inbound,019,Americas,0 +manager.com.br,manager.com.br,inbound,019,Americas,0 +mandrillapp.com,mandrillapp.com,inbound,019,Americas,1 +mandrillapp.com,myjobhelperalerts.com,inbound,019,Americas,1 +marcustheatres.com,movio.co,inbound,019,Americas,0 +markandgraham.com,markandgraham.com,inbound,019,Americas,0 +marketer-safelist.com,jsalfianmarketing.com,inbound,019,Americas,1 +marketinghq.net,elabs8.com,inbound,019,Americas,0 +marketingprofs.com,marketingprofs.com,inbound,019,Americas,0.004412 +marlboro.com,marlboro.com,inbound,019,Americas,0 +maropost.com,biotrustnews.com,inbound,019,Americas,0 +maropost.com,mailing-truthaboutabs.com,inbound,019,Americas,0 +maropost.com,maropost.com,inbound,019,Americas,0 +maropost.com,mp2201.com,inbound,019,Americas,0 +masivapp.com,masivapp.com,inbound,019,Americas,1 +massageenvyclinics.com,massageenvyclinics.com,inbound,019,Americas,0 +masterbase.com,masterbase.com,inbound,019,Americas,0 +mastercard-email.com,mastercard-email.com,inbound,019,Americas,0 +mate1.net,mate1.net,inbound,019,Americas,0 +matrixemailer.com,matrixemailer.com,inbound,019,Americas,0 +mbstrm.com,mobilestorm.com,inbound,019,Americas,0 +mcafee.com,mcafee.com,inbound,019,Americas,0.979126 +mcarthurglen.com,mcarthurglen.com,inbound,019,Americas,0 +mcdlv.net,mcdlv.net,inbound,019,Americas,0 +mckinsey.com,bigfootinteractive.com,inbound,019,Americas,0 +mcsv.net,mcsv.net,inbound,019,Americas,0 +mdlinx.com,mdlinx.com,inbound,019,Americas,0 +mec.gov.br,mec.gov.br,inbound,019,Americas,0 +mediabistro.com,iworld.com,inbound,019,Americas,0.006428 +medpagetoday.com,wc09.net,inbound,019,Americas,0 +meetmemail.com,meetmemail.com,inbound,019,Americas,0 +megasenders.com,megasenders.com,inbound,019,Americas,0.07662 +memberdealsusa.com,memberdealsusa.com,inbound,019,Americas,0 +menswearhouse.com,menswearhouse.com,inbound,019,Americas,0 +mercadojobs.com,sendgrid.net,inbound,019,Americas,1 +mercola.com,mercola.com,inbound,019,Americas,0.000369 +messagegears.net,messagegears.net,inbound,019,Americas,0 +met-art.com,hydentra.com,inbound,019,Americas,1 +mgo.com,bronto.com,inbound,019,Americas,0 +michaels.com,chtah.net,inbound,019,Americas,0 +michaels.com,michaels.com,inbound,019,Americas,0 +microcentermedia.com,bfi0.com,inbound,019,Americas,0 +microsoft.com,hotmail.{...},inbound,019,Americas,1 +microsoft.com,msn.com,inbound,019,Americas,1 +midnightsunsafelist.com,zoothost.com,inbound,019,Americas,0 +milfaholic.com,iverificationsystems.com,inbound,019,Americas,0 +miltnews.com,miltnews.com,inbound,019,Americas,0 +mindbodyonline.com,mindbodyonline.com,inbound,019,Americas,1 +mindfieldonline.com,mindfieldonline.com,inbound,019,Americas,0 +mindmoviesmail.com,mindmoviesmail.com,inbound,019,Americas,0.004239 +mindvalleymail3.com,mindvalleymail3.com,inbound,019,Americas,0 +mint.com,mint.com,inbound,019,Americas,0 +minted.com,messagelabs.com,inbound,019,Americas,0.999669 +missselfridge.com,wallis-fashion.com,inbound,019,Americas,0 +mistersafelist.com,zoothost.com,inbound,019,Americas,0 +mit.edu,mit.edu,inbound,019,Americas,0.869568 +mjinn.com,mailurja.com,inbound,019,Americas,0 +mkt015.com,mkt015.com,inbound,019,Americas,0 +mkt022.com,mkt022.com,inbound,019,Americas,0 +mkt063.com,mkt063.com,inbound,019,Americas,0 +mkt1136.com,mkt1136.com,inbound,019,Americas,0 +mkt1985.com,fmlinks.net,inbound,019,Americas,0 +mkt2010.com,mkt2010.com,inbound,019,Americas,0 +mkt2106.com,mkt2106.com,inbound,019,Americas,0 +mkt2170.com,mkt2170.com,inbound,019,Americas,0 +mkt2181.com,mkt2181.com,inbound,019,Americas,0 +mkt2615.com,mkt2615.com,inbound,019,Americas,0 +mkt2813.com,mkt2813.com,inbound,019,Americas,0 +mkt2944.com,mkt2944.com,inbound,019,Americas,0 +mkt3134.com,mkt3134.com,inbound,019,Americas,0 +mkt3142.com,mkt3142.com,inbound,019,Americas,0 +mkt3156.com,mkt3156.com,inbound,019,Americas,0 +mkt3203.com,mkt3203.com,inbound,019,Americas,0 +mkt346.com,mkt346.com,inbound,019,Americas,0 +mkt3544.com,mkt3544.com,inbound,019,Americas,0 +mkt3622.com,mkt3622.com,inbound,019,Americas,0 +mkt3682.com,mkt3682.com,inbound,019,Americas,0 +mkt3690.com,mkt3690.com,inbound,019,Americas,0 +mkt3695.com,mkt3695.com,inbound,019,Americas,0 +mkt3804.com,mkt3804.com,inbound,019,Americas,0 +mkt3815.com,mkt3815.com,inbound,019,Americas,0 +mkt3952.com,xoom.com,inbound,019,Americas,0 +mkt4355.com,mkt4355.com,inbound,019,Americas,0 +mkt4364.com,mkt4364.com,inbound,019,Americas,0 +mkt459.com,mkt459.com,inbound,019,Americas,0 +mkt4701.com,mkt4701.com,inbound,019,Americas,0 +mkt4728.com,mkt4728.com,inbound,019,Americas,0 +mkt4731.com,mkt4731.com,inbound,019,Americas,0 +mkt4738.com,mkt4738.com,inbound,019,Americas,0 +mkt5071.com,mkt5071.com,inbound,019,Americas,0 +mkt5098.com,mkt5098.com,inbound,019,Americas,0 +mkt5131.com,mkt5131.com,inbound,019,Americas,0 +mkt5144.com,mkt5144.com,inbound,019,Americas,0 +mkt5144.com,mkt5980.com,inbound,019,Americas,0 +mkt5144.com,mkt5981.com,inbound,019,Americas,0 +mkt5181.com,mkt5181.com,inbound,019,Americas,0 +mkt5269.com,mkt5269.com,inbound,019,Americas,0 +mkt529.com,mkt529.com,inbound,019,Americas,0 +mkt5297.com,mkt5297.com,inbound,019,Americas,0 +mkt5297.com,mkt5309.com,inbound,019,Americas,0 +mkt5371.com,mkt5371.com,inbound,019,Americas,0 +mkt5806.com,mkt5806.com,inbound,019,Americas,0 +mkt5934.com,mkt5934.com,inbound,019,Americas,0 +mkt5937.com,mkt5937.com,inbound,019,Americas,0 +mkt5970.com,mkt5970.com,inbound,019,Americas,0 +mkt6100.com,mkt6098.com,inbound,019,Americas,0 +mkt6276.com,mkt6276.com,inbound,019,Americas,0 +mkt6323.com,mkt6323.com,inbound,019,Americas,0 +mkt746.com,mkt746.com,inbound,019,Americas,0 +mkt824.com,mkt869.com,inbound,019,Americas,0 +mktdillards.com,mktdillards.com,inbound,019,Americas,0 +mo1send.com,mo1send.com,inbound,019,Americas,0 +mobly.com.br,mobly.com.br,inbound,019,Americas,0 +modellsemail.com,n-email.net,inbound,019,Americas,0 +moneymorning.com,moneymappress.com,inbound,019,Americas,0 +monster.com,monster.com,inbound,019,Americas,0 +monster.com,tmpw.net,inbound,019,Americas,0.000503 +moon-ray.com,moon-ray.com,inbound,019,Americas,0 +mooresclothing.com,mooresclothing.com,inbound,019,Americas,0 +moveon.org,moveon.org,inbound,019,Americas,0.996147 +mrmlsmatrix.com,mrmlsmatrix.com,inbound,019,Americas,0 +ms.com,ms.com,inbound,019,Americas,1 +msn.com,hotmail.{...},inbound,019,Americas,1 +msn.com,hotmail.{...},outbound,019,Americas,1 +mta.info,ealert.com,inbound,019,Americas,0 +mtasv.net,mtasv.net,inbound,019,Americas,0.999999 +musicnotes-alerts.com,mybuys.com,inbound,019,Americas,0 +mustanglist.com,mustanglist.com,inbound,019,Americas,0 +mycheapoair.com,mycheapoair.com,inbound,019,Americas,0.957931 +mydailymoment.biz,mydailymoment.biz,inbound,019,Americas,0 +mydailymoment.info,mydailymoment.info,inbound,019,Americas,0 +mydailymoment.net,mydailymoment.net,inbound,019,Americas,0 +mydailymoment.us,mydailymoment.us,inbound,019,Americas,0 +myfedloan.org,aessuccess.org,inbound,019,Americas,0.328917 +myfxbook.com,myfxbook.com,inbound,019,Americas,0 +mygreatlakes.org,glhec.org,inbound,019,Americas,0.000247 +myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,019,Americas,0 +mymeijer.com,mymeijer.com,inbound,019,Americas,0 +myngp.com,ngpweb.com,inbound,019,Americas,0 +myperfectsale.com,myperfectsale.com,inbound,019,Americas,1 +myprotein.com,thehut.com,inbound,019,Americas,0 +mysafelistmailer.com,mysafelistmailer.com,inbound,019,Americas,0.00033 +mysmartprice.com,itzwow.com,inbound,019,Americas,1 +myzamanamail.com,myzamanamail.com,inbound,019,Americas,0 +n-email.net,n-email.net,inbound,019,Americas,0 +n-email1.net,n-email1.net,inbound,019,Americas,0 +n-email4.net,n-email4.net,inbound,019,Americas,0 +nanomail.com.br,araie.com.br,inbound,019,Americas,1 +napitipp.hu,napitipp.hu,inbound,019,Americas,0 +nasa.gov,nasa.gov,inbound,019,Americas,0.101289 +nastygal.com,bronto.com,inbound,019,Americas,0 +nationbuilder.com,nationbuilder.com,inbound,019,Americas,1 +nature.com,nature.com,inbound,019,Americas,0 +naukri.com,naukri.com,inbound,019,Americas,0.000533 +nauta.cu,etecsa.net,inbound,019,Americas,0 +nba.com,nba.com,inbound,019,Americas,0.005829 +nbaa.org,nbaa.org,inbound,019,Americas,0 +ncl.com,ncl.com,inbound,019,Americas,0 +neimanmarcusemail.com,neimanmarcusemail.com,inbound,019,Americas,0 +net-a-porter.com,net-a-porter.com,inbound,019,Americas,0 +net-empregos.com,net-empregos.com,inbound,019,Americas,0 +netcommunity1.com,blackbaud.com,inbound,019,Americas,0 +netflix.com,netflix.com,inbound,019,Americas,1 +netopia.pt,netopia.pt,inbound,019,Americas,0 +netprosoftmail.com,netprosoftmail.com,inbound,019,Americas,0 +netsuite.com,netsuite.com,inbound,019,Americas,0.60682 +networkworld.com,networkworld.com,inbound,019,Americas,0 +newgrounds.com,newgrounds.com,inbound,019,Americas,0 +newrelic.com,sendlabs.com,inbound,019,Americas,0 +newspaperdirect.com,newspaperdirect.com,inbound,019,Americas,0.000177 +newyorktimesinfo.com,newyorktimesinfo.com,inbound,019,Americas,0 +nexcess.net,nexcess.net,inbound,019,Americas,0.039513 +nextdoor.com,mailgun.info,inbound,019,Americas,1 +nfl.com,bfi0.com,inbound,019,Americas,0 +nih.gov,nih.gov,inbound,019,Americas,8.8e-05 +ninewestmail.com,ninewestmail.com,inbound,019,Americas,0 +ning.com,ning.com,inbound,019,Americas,0 +nixle.com,nixle.com,inbound,019,Americas,0 +nl00.net,netline.com,inbound,019,Americas,5e-06 +nl00.net,nl00.net,inbound,019,Americas,0 +nongnu.org,gnu.org,inbound,019,Americas,1 +nordstrom.com,taleo.net,inbound,019,Americas,1 +nortonfromsymantec.com,rsys1.com,inbound,019,Americas,0 +novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,019,Americas,0 +numbersusa.com,numbersusa.com,inbound,019,Americas,0 +nyandcompany.com,nyandcompany.com,inbound,019,Americas,0 +ocadomail.com,ocadomail.com,inbound,019,Americas,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,019,Americas,0 +ofertasefacil.com.br,ofertasefacil.com.br,inbound,019,Americas,0 +ofertop.pe,icommarketing.com,inbound,019,Americas,0 +officedepot.com,officedepot.com,inbound,019,Americas,2.6e-05 +olympiaedge.net,olympiaedge.net,inbound,019,Americas,0 +oneindia.in,infimail.com,inbound,019,Americas,0 +oneindia.in,mailurja.com,inbound,019,Americas,0 +onepatriotplace.com,britecast.com,inbound,019,Americas,0 +onestopplus.com,neolane.net,inbound,019,Americas,0 +onetravelspecials.com,onetravelspecials.com,inbound,019,Americas,0.365386 +online.com,cnet.com,inbound,019,Americas,0 +onthecitymail.org,onthecitymail.org,inbound,019,Americas,1 +oo155.com,bsftransmit7.com,inbound,019,Americas,0 +oo155.com,oo155.com,inbound,019,Americas,0 +openstack.org,openstack.org,inbound,019,Americas,0.997933 +openstackmail.com,infimail.com,inbound,019,Americas,0 +opentable.com,opentable.com,inbound,019,Americas,2e-06 +opinionoutpost.com,opinionoutpost.com,inbound,019,Americas,0 +opinionsquare.com,opinionsquare.com,inbound,019,Americas,0 +oprah.com,oprah.com,inbound,019,Americas,0 +opticsplanet.com,opticsplanet.com,inbound,019,Americas,0 +optonline.net,cv.net,inbound,019,Americas,0 +optonline.net,optonline.net,outbound,019,Americas,0 +oriental-trading.com,oriental-trading.com,inbound,019,Americas,0 +outlook.com,hotmail.{...},inbound,019,Americas,1 +outlook.com,hotmail.{...},outbound,019,Americas,1 +overnightprints.com,chtah.net,inbound,019,Americas,0 +ovuline.com,ovuline.com,inbound,019,Americas,1 +pagoda.com,zales.com,inbound,019,Americas,0 +pagseguro.com.br,uol.com.br,inbound,019,Americas,0 +pair.com,pair.com,inbound,019,Americas,0.893803 +palmscasinoresort.com,palmscasinoresort.com,inbound,019,Americas,0 +pampers.com,bfi0.com,inbound,019,Americas,0 +pandaresearch.com,pandaresearch.com,inbound,019,Americas,0 +pandora.com,pandora.com,inbound,019,Americas,1 +pandora.net,pandora.net,inbound,019,Americas,1 +parents.com,meredith.com,inbound,019,Americas,0 +parkmobileglobal.com,parkmobile.us,inbound,019,Americas,0 +path.com,path.com,inbound,019,Americas,1 +patriotupdate.com,inboxfirst.com,inbound,019,Americas,0 +payback.in,chtah.net,inbound,019,Americas,0 +paytm.com,paytm.com,inbound,019,Americas,1 +pbteen.com,pbteen.com,inbound,019,Americas,0 +pch.com,ed10.com,inbound,019,Americas,0 +pchfrontpage.com,ed10.com,inbound,019,Americas,0 +pchlotto.com,ed10.com,inbound,019,Americas,0 +pchplayandwin.com,ed10.com,inbound,019,Americas,0 +pchsearch.com,ed10.com,inbound,019,Americas,0 +pcmag.com,ittoolbox.com,inbound,019,Americas,0 +pcworld.com,pcworld.com,inbound,019,Americas,0 +pd25.com,pd25.com,inbound,019,Americas,1 +pearlsofwealth.com,pearlsofwealth.com,inbound,019,Americas,1 +peartreegreetings.com,rexcraft.com,inbound,019,Americas,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,019,Americas,0 +pepboys.com,pepboys.com,inbound,019,Americas,0 +pepperfry.com,epidm.net,inbound,019,Americas,0 +perfectpriceindia.com,infimail.com,inbound,019,Americas,0 +perfora.net,perfora.net,inbound,019,Americas,1 +permissionresearch.com,permissionresearch.com,inbound,019,Americas,0 +personalliberty.com,personalliberty.com,inbound,019,Americas,1 +pga.com,pga.com,inbound,019,Americas,0 +pgeveryday.com,bfi0.com,inbound,019,Americas,0 +phoenix.edu,phoenix.edu,inbound,019,Americas,0 +phsmtpbox.com,phsmtpbox.com,inbound,019,Americas,0 +pinterest.com,pinterest.com,inbound,019,Americas,1 +pivotaltracker.com,pivotaltracker.com,inbound,019,Americas,1 +pixable.com,pixable.com,inbound,019,Americas,1 +pizzahut.com,quikorder.com,inbound,019,Americas,0 +plexapp.com,plex.tv,inbound,019,Americas,1 +pmailus.com,patrontechnology.com,inbound,019,Americas,0 +pnc.com,messagelabs.com,inbound,019,Americas,0.997194 +pobox.com,pobox.com,inbound,019,Americas,0.975372 +pof.com,plentyoffish.co.uk,inbound,019,Americas,0 +pogo.com,pogo.com,inbound,019,Americas,0 +polyvore.com,polyvore.com,inbound,019,Americas,1 +postcardfromhell.com,cyberthugs.com,inbound,019,Americas,1 +potterybarn.com,potterybarn.com,inbound,019,Americas,0 +potterybarnkids.com,potterybarnkids.com,inbound,019,Americas,0 +preferredpetclub.com,preferredpetclub.com,inbound,019,Americas,0 +presslaff.net,dat-e-baseonline.com,inbound,019,Americas,0 +pressmartmail.com,pressmartmail.com,inbound,019,Americas,0 +priorityoneemail.com,priorityoneemail.com,inbound,019,Americas,0 +private-elist.com,private-elist.com,inbound,019,Americas,0 +progressive.com,progressive.com,inbound,019,Americas,0.999893 +promedmail.org,childrenshospital.org,inbound,019,Americas,1 +propertysolutions.com,propertysolutions.com,inbound,019,Americas,1 +prospectgeysercoop.com,prospectgeysercoop.com,inbound,019,Americas,1 +providesupport.com,providesupport.com,inbound,019,Americas,0.000103 +proxyvote.com,adp-ics.com,inbound,019,Americas,0.908635 +psu.edu,psu.edu,inbound,019,Americas,0.073843 +puffinmailer.com,zoothost.com,inbound,019,Americas,0 +purewow.com,purewow.com,inbound,019,Americas,0 +purlsmail.com,purlsmail.com,inbound,019,Americas,0 +pxsmail.com,pxsmail.com,inbound,019,Americas,0 +q.com,synacor.com,inbound,019,Americas,0.999734 +qemailserver.com,qemailserver.com,inbound,019,Americas,0 +qtropnews.com,qtropnews.com,inbound,019,Americas,1.7e-05 +qualicorp.com.br,qualicorp.com.br,inbound,019,Americas,1 +qualitysafelist.com,zoothost.com,inbound,019,Americas,0 +queopinas.com,confirmit.com,inbound,019,Americas,1 +quickrewards.net,quickrewards.net,inbound,019,Americas,0 +quikr.com,quikr.com,inbound,019,Americas,0 +quora.com,quora.com,inbound,019,Americas,1 +qvcemail.com,qvcemail.com,inbound,019,Americas,0 +radarsystems.net,radarsystems.net,inbound,019,Americas,1 +radiantretailapps.com,radiantretailapps.com,inbound,019,Americas,0 +radioshack.com,radioshack.com,inbound,019,Americas,0 +rapattoni.com,rapmls.com,inbound,019,Americas,0 +razerzone.com,chtah.net,inbound,019,Americas,0 +rbc.com,rbc.com,inbound,019,Americas,0.003061 +rdio.com,rdio.com,inbound,019,Americas,1 +realtor.org,realtor.org,inbound,019,Americas,0 +realtytrac.com,realtytrac.com,inbound,019,Americas,0.001681 +recipe.com,meredith.com,inbound,019,Americas,0 +recruit.net,recruit.net,inbound,019,Americas,0 +redbox.com,redbox.com,inbound,019,Americas,0 +redfin.com,redfin.com,inbound,019,Americas,0 +redtri.com,redtri.com,inbound,019,Americas,0 +reebokusnews.com,reebokusnews.com,inbound,019,Americas,0 +reebonz.com,ed10.com,inbound,019,Americas,0 +reebonz.com,reebonz.com,inbound,019,Americas,1 +registro.br,registro.br,inbound,019,Americas,0 +rent.com,rent.com,inbound,019,Americas,0.000397 +rentalcars.com,rentalcars.com,inbound,019,Americas,1 +renweb.com,renweb.com,inbound,019,Americas,0 +researchgate.net,researchgate.net,inbound,019,Americas,0 +restorationhardware.com,restorationhardware.com,inbound,019,Americas,0 +retailmenot.com,retailmenot.com,inbound,019,Americas,0 +reverbnation.com,reverbnation.com,inbound,019,Americas,0 +rewardme.in,bfi0.com,inbound,019,Americas,0 +ricardoeletro.com.br,allin.com.br,inbound,019,Americas,0 +rigzonemail.com,rigzonemail.com,inbound,019,Americas,0 +ripleyperu.com.pe,icommarketing.com,inbound,019,Americas,0 +riseup.net,riseup.net,inbound,019,Americas,1 +rivamail.com,mailurja.com,inbound,019,Americas,0 +rnmk.com,rnmk.com,inbound,019,Americas,0 +roamans.com,neolane.net,inbound,019,Americas,0 +rockwellcollins.com,rockwellcollins.com,inbound,019,Americas,1 +rpinow.org,app-info.net,inbound,019,Americas,0 +rr.com,rr.com,inbound,019,Americas,0.03696 +rr.com,rr.com,outbound,019,Americas,0 +rsgsv.net,rsgsv.net,inbound,019,Americas,0 +rsvpsv.net,rsvpsv.net,inbound,019,Americas,0 +rsvpsv.net,send.esp.br,inbound,019,Americas,0 +rsys2.com,amfam.com,inbound,019,Americas,0 +rsys2.com,cheaptickets.com,inbound,019,Americas,0 +rsys2.com,dishnetworkmail.com,inbound,019,Americas,0 +rsys2.com,e-comms.net,inbound,019,Americas,0 +rsys2.com,eharmony.com,inbound,019,Americas,0 +rsys2.com,fathead.com,inbound,019,Americas,0 +rsys2.com,intuit.com,inbound,019,Americas,0 +rsys2.com,kmart.com,inbound,019,Americas,0 +rsys2.com,kohlernews.com,inbound,019,Americas,0 +rsys2.com,lego.com,inbound,019,Americas,0 +rsys2.com,lenovo.com,inbound,019,Americas,0 +rsys2.com,modcloth.com,inbound,019,Americas,0 +rsys2.com,moxieinteractive.com,inbound,019,Americas,0 +rsys2.com,orbitz.com,inbound,019,Americas,0 +rsys2.com,payless.com,inbound,019,Americas,0 +rsys2.com,petsathome.com,inbound,019,Americas,0 +rsys2.com,quizzle.com,inbound,019,Americas,0 +rsys2.com,robeez.com,inbound,019,Americas,0 +rsys2.com,rsys1.com,inbound,019,Americas,0 +rsys2.com,rsys2.com,inbound,019,Americas,0 +rsys2.com,rsys3.com,inbound,019,Americas,0 +rsys2.com,rsys4.com,inbound,019,Americas,0 +rsys2.com,saucony.com,inbound,019,Americas,0 +rsys2.com,sears.com,inbound,019,Americas,0 +rsys2.com,shopbop.com,inbound,019,Americas,0 +rsys2.com,southwest.com,inbound,019,Americas,0 +rsys2.com,speeddatemail.com,inbound,019,Americas,0 +rsys2.com,thecompanystore.com,inbound,019,Americas,0 +rsys2.com,theknot.com,inbound,019,Americas,0 +rsys5.com,alibris.com,inbound,019,Americas,0 +rsys5.com,allstate-email.com,inbound,019,Americas,0 +rsys5.com,beachmint.com,inbound,019,Americas,0 +rsys5.com,belleandclive.com,inbound,019,Americas,0 +rsys5.com,br.dk,inbound,019,Americas,0 +rsys5.com,charlotterusse.com,inbound,019,Americas,0 +rsys5.com,comixology.com,inbound,019,Americas,0 +rsys5.com,cottonon.com,inbound,019,Americas,0 +rsys5.com,ediblearrangements.com,inbound,019,Americas,0 +rsys5.com,emailworldmarket.com,inbound,019,Americas,0 +rsys5.com,farfetch.com,inbound,019,Americas,0 +rsys5.com,frhiemailcommunications.com,inbound,019,Americas,0 +rsys5.com,harryanddavid.com,inbound,019,Americas,0 +rsys5.com,hollandandbarrett.com,inbound,019,Americas,0 +rsys5.com,icing.com,inbound,019,Americas,0 +rsys5.com,indigo.ca,inbound,019,Americas,0 +rsys5.com,jabong.com,inbound,019,Americas,0 +rsys5.com,jcrew.com,inbound,019,Americas,0 +rsys5.com,jjill.com,inbound,019,Americas,0 +rsys5.com,kanui.com.br,inbound,019,Americas,0 +rsys5.com,kirklands.com,inbound,019,Americas,0 +rsys5.com,lanebryant.com,inbound,019,Americas,0 +rsys5.com,lazada.com,inbound,019,Americas,0 +rsys5.com,leapfrog.com,inbound,019,Americas,0 +rsys5.com,llbean.com,inbound,019,Americas,0 +rsys5.com,lojascolombo.com.br,inbound,019,Americas,0 +rsys5.com,madewell.com,inbound,019,Americas,0 +rsys5.com,magazineluiza.com.br,inbound,019,Americas,0 +rsys5.com,missguided.co.uk,inbound,019,Americas,0 +rsys5.com,moma.org,inbound,019,Americas,0 +rsys5.com,nationalgeographic.com,inbound,019,Americas,0 +rsys5.com,neat.com,inbound,019,Americas,0 +rsys5.com,newbalance.com,inbound,019,Americas,0 +rsys5.com,news-voeazul.com.br,inbound,019,Americas,0 +rsys5.com,nordstrom.com,inbound,019,Americas,0 +rsys5.com,normthompson.com,inbound,019,Americas,0 +rsys5.com,novomundo.com.br,inbound,019,Americas,0 +rsys5.com,ourdeal.com.au,inbound,019,Americas,0 +rsys5.com,pier1.com,inbound,019,Americas,0 +rsys5.com,productmadness.com,inbound,019,Americas,0 +rsys5.com,rainbowshops.com,inbound,019,Americas,0 +rsys5.com,rei.com,inbound,019,Americas,0 +rsys5.com,roadrunnersports.com,inbound,019,Americas,0 +rsys5.com,rosettastone.com,inbound,019,Americas,0 +rsys5.com,seamless.com,inbound,019,Americas,0 +rsys5.com,serenaandlily.com,inbound,019,Americas,0 +rsys5.com,smiles.com.br,inbound,019,Americas,0 +rsys5.com,soubarato.com.br,inbound,019,Americas,0 +rsys5.com,strava.com,inbound,019,Americas,0 +rsys5.com,submarino.com.br,inbound,019,Americas,0 +rsys5.com,thewalkingcompany.com,inbound,019,Americas,0 +rsys5.com,tigerdirect.com,inbound,019,Americas,0 +rsys5.com,udemy.com,inbound,019,Americas,0 +rsys5.com,vitaminshoppe.com,inbound,019,Americas,0 +rsys5.com,vpusa.com,inbound,019,Americas,0 +rsys5.com,walmart.com.br,inbound,019,Americas,0 +rsys5.com,worldofwatches.com,inbound,019,Americas,0 +rsys5.com,xfinity.com,inbound,019,Americas,0 +rue21email.com,rue21email.com,inbound,019,Americas,0 +ruum.com,ruum.com,inbound,019,Americas,0 +saavn.com,saavn.com,inbound,019,Americas,1 +safelistextreme.com,quantumsafelist.com,inbound,019,Americas,0 +safelistpro.com,safelistpro.com,inbound,019,Americas,0.00014 +safeway.com,chtah.com,inbound,019,Americas,0 +sailthru.com,sailthru.com,inbound,019,Americas,0 +saks.com,saks.com,inbound,019,Americas,0 +saksoff5th.com,saksoff5th.com,inbound,019,Americas,0 +salememail.net,salememail.net,inbound,019,Americas,0 +salesforce.com,postini.com,inbound,019,Americas,0.816952 +salesforce.com,salesforce.com,inbound,019,Americas,1 +salesforce.com,salesforce.com,outbound,019,Americas,1 +salliemae.com,salliemae.com,inbound,019,Americas,1 +salsalabs.net,salsalabs.net,inbound,019,Americas,0 +samashmusic.com,wc09.net,inbound,019,Americas,0 +samsclub.com,m0.net,inbound,019,Americas,0 +sans.org,sans.org,inbound,019,Americas,0.497629 +santander.cl,santander.cl,inbound,019,Americas,0.999992 +santander.cl,santandersantiago.cl,inbound,019,Americas,1 +sassieshop.com,sassieshop.com,inbound,019,Americas,0.025887 +savelivefresh.com,livesavemail.com,inbound,019,Americas,1 +savingstar.com,savingstar.com,inbound,019,Americas,1 +sbcglobal.net,yahoo.{...},inbound,019,Americas,1 +sbcglobal.net,yahoodns.net,outbound,019,Americas,1 +sc.com,messagelabs.com,inbound,019,Americas,0.996156 +schwab.com,schwab.com,inbound,019,Americas,0.025055 +seaworld.com,seaworld.com,inbound,019,Americas,0 +securence.com,securence.com,inbound,019,Americas,0.670246 +secureserver.net,secureserver.net,inbound,019,Americas,4.4e-05 +seekingalpha.com,seekingalpha.com,inbound,019,Americas,1 +seekingalpha.com,sendgrid.net,inbound,019,Americas,1 +senate.gov,senate.gov,inbound,019,Americas,0.992994 +sendearnings.com,sendearnings.com,inbound,019,Americas,0 +sendgrid.info,sendgrid.net,inbound,019,Americas,1 +sendgrid.me,sendgrid.net,inbound,019,Americas,1 +sendlane.com,sendlane.com,inbound,019,Americas,0 +serpadres.es,chtah.net,inbound,019,Americas,0 +service-now.com,postini.com,inbound,019,Americas,0.9858 +service-now.com,service-now.com,inbound,019,Americas,0.998562 +sfimg.com,sfimarketing.com,inbound,019,Americas,0 +sfly.com,shutterfly.com,inbound,019,Americas,0 +shaadi.com,shaadi.com,inbound,019,Americas,0.00037 +shadowshopper.com,shadowshopper.com,inbound,019,Americas,5.6e-05 +shaw.ca,shaw.ca,inbound,019,Americas,0 +shaw.ca,shaw.ca,outbound,019,Americas,1 +sheplers.com,sheplers.com,inbound,019,Americas,0 +shiftplanning.com,shiftplanning.com,inbound,019,Americas,0 +shiksha.com,shiksha.com,inbound,019,Americas,0 +shoedazzle.com,shoedazzle.com,inbound,019,Americas,0 +shoes.com,famousfootwear.com,inbound,019,Americas,0 +shopbonton.com,shopbonton.com,inbound,019,Americas,0 +shopcluesemail.com,shopcluesemail.com,inbound,019,Americas,0 +shopcluesmail.com,shopcluesmail.com,inbound,019,Americas,0 +shophq.com,shophq.com,inbound,019,Americas,0 +shopkick.com,shopkick.com,inbound,019,Americas,0 +shopko.com,shopko.com,inbound,019,Americas,0 +shoppersoptimum.ca,thindata.net,inbound,019,Americas,0 +shoptime.com,shoptime.com,inbound,019,Americas,0 +shtyle.fm,shtyle.fm,inbound,019,Americas,0 +sierratradingpost.com,sierratradingpost.com,inbound,019,Americas,0.000885 +sigmabeauty.com,lstrk.net,inbound,019,Americas,1 +sii.cl,sii.cl,inbound,019,Americas,0 +simplyhired.com,simplyhired.com,inbound,019,Americas,0 +siriusxm.com,xmradio.com,inbound,019,Americas,0 +sitecore-mailer.com,sendlabs.com,inbound,019,Americas,0 +sittercity.com,sittercity.com,inbound,019,Americas,0 +sixflags.com,sixflags.com,inbound,019,Americas,0 +skillpages-mailer.com,sendlabs.com,inbound,019,Americas,0 +skillpages-mailer.com,skillpagesmail.com,inbound,019,Americas,0 +sky.com,sky.com,inbound,019,Americas,0 +skype.com,delivery.net,inbound,019,Americas,0 +sld.cu,sld.cu,inbound,019,Americas,1 +slidesharemail.com,newslettergrid.com,inbound,019,Americas,1 +slidesharemail.com,slidesharemail.com,inbound,019,Americas,1 +smartbrief.com,smartbrief.com,inbound,019,Americas,0 +smartdraw.com,smartdraw.com,inbound,019,Americas,0 +smartertravel.com,smartertravelmedia.com,inbound,019,Americas,0.021434 +smartphoneexperts.com,mailgun.net,inbound,019,Americas,1 +snapretail.com,snapretail.com,inbound,019,Americas,1 +socialappsmail.com,socialappsmail.com,inbound,019,Americas,1 +solesociety.com,bronto.com,inbound,019,Americas,0 +solosenders.com,megasenders.com,inbound,019,Americas,8.2e-05 +solosenders.com,traxweb.org,inbound,019,Americas,0 +soma.com,soma.com,inbound,019,Americas,0 +someecards.com,someecards.com,inbound,019,Americas,0 +songkick.com,songkick.com,inbound,019,Americas,1 +sony-latin.com,sony-latin.com,inbound,019,Americas,0.01735 +sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,019,Americas,0 +sourceforge.net,sourceforge.net,inbound,019,Americas,1 +southwest.com,southwest.com,inbound,019,Americas,0 +sp.gov.br,sp.gov.br,inbound,019,Americas,0.496357 +sparkpeople.com,sparkpeople.com,inbound,019,Americas,0 +spectersoft.com,spectersoft.com,inbound,019,Americas,0 +spencersonline.com,spencersonline.com,inbound,019,Americas,0 +spiritairlines.com,ctd004.net,inbound,019,Americas,0 +spiritairlines.com,ctd005.net,inbound,019,Americas,0 +splitwise.com,splitwise.com,inbound,019,Americas,1 +sportlobster.com,sportlobster.com,inbound,019,Americas,1 +sportsdirect.com,sportsdirect.com,inbound,019,Americas,0 +sportsline.com,cbsig.net,inbound,019,Americas,1 +sportsmansguide.com,sportsmansguide.com,inbound,019,Americas,0.736903 +sprint.com,m0.net,inbound,019,Americas,0 +squareup.com,squareup.com,inbound,019,Americas,0.986452 +stanford.edu,highwire.org,inbound,019,Americas,1 +stanford.edu,stanford.edu,inbound,019,Americas,0.93025 +stansberryresearch.com,stansberryresearch.com,inbound,019,Americas,0.001541 +staples.com,staples.com,inbound,019,Americas,0.006379 +starbucks.com,starbucks.com,inbound,019,Americas,0 +stardockcorporation.com,stardockcorporation.com,inbound,019,Americas,0 +stardockentertainment.info,stardockentertainment.info,inbound,019,Americas,0 +startribune.com,startribune.com,inbound,019,Americas,0.001728 +startwire.com,jobsreport.com,inbound,019,Americas,1 +startwire.com,startwire.com,inbound,019,Americas,1 +state.gov,state.gov,inbound,019,Americas,0 +statefarm.com,statefarm.com,inbound,019,Americas,1 +steinmart.com,steinmart.com,inbound,019,Americas,0 +stevemadden.com,stevemadden.com,inbound,019,Americas,0 +streetauthoritydaily.com,streetauthoritydaily.com,inbound,019,Americas,0 +stumblemail.com,stumblemail.com,inbound,019,Americas,1 +suafaturanet.com.br,suafaturanet.com.br,inbound,019,Americas,0.995846 +subscribermail.com,subscribermail.com,inbound,019,Americas,0 +sungard.com,postini.com,inbound,019,Americas,0.498265 +sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,019,Americas,1 +superbalist.com,sailthru.com,inbound,019,Americas,0 +supersafemailer.com,zoothost.com,inbound,019,Americas,0 +supremelist.com,onlinehome-server.com,inbound,019,Americas,1 +surlatable.com,surlatable.com,inbound,019,Americas,0 +surveyjobopportunities.com,surveyjobopportunities.com,inbound,019,Americas,3.7e-05 +surveymonkey.com,surveymonkey.com,inbound,019,Americas,0 +surveysavvy.com,surveysavvy.com,inbound,019,Americas,0 +sylectus.com,sylectus.com,inbound,019,Americas,0.361297 +sympatico.ca,hotmail.{...},outbound,019,Americas,1 +taggedmail.com,taggedmail.com,inbound,019,Americas,0 +tamu.edu,tamu.edu,inbound,019,Americas,0.249876 +tangeroutletsusa.com,bronto.com,inbound,019,Americas,0 +target-safelist.com,safelistpro.com,inbound,019,Americas,0.000215 +targetproblaster.com,targetproblaster.com,inbound,019,Americas,0 +targetx.com,targetx.com,inbound,019,Americas,0 +tarot.com,tarot.com,inbound,019,Americas,0 +tastefullysimpleparty.com,bigfootinteractive.com,inbound,019,Americas,0 +tastingtable.com,tastingtable.com,inbound,019,Americas,0 +taxi4sure.net,infimail.com,inbound,019,Americas,0 +teach12.net,teach12.net,inbound,019,Americas,0 +techtarget.com,techtarget.com,inbound,019,Americas,0.001771 +telus.net,telus.net,inbound,019,Americas,0.006226 +ten24mail.com,ten24mail.com,inbound,019,Americas,0 +terra.com,terra.com,inbound,019,Americas,0.000463 +terra.com.br,terra.com,inbound,019,Americas,0 +terra.com.br,terra.com,outbound,019,Americas,0 +tesco.com,tesco.com,inbound,019,Americas,0 +testfunda.com,testfunda.com,inbound,019,Americas,0 +textnow.me,textnow.me,inbound,019,Americas,0 +tgw.com,tgw.com,inbound,019,Americas,0 +theanimalrescuesite.com,theanimalrescuesite.com,inbound,019,Americas,0 +theatermania.com,wc09.net,inbound,019,Americas,0 +thebay.com,thebay.com,inbound,019,Americas,0 +thegrommet.com,lstrk.net,inbound,019,Americas,1 +thehut.com,thehut.com,inbound,019,Americas,0 +thelimited.com,thelimited.com,inbound,019,Americas,0 +themailbagsafelist.com,thomas-j-brown.com,inbound,019,Americas,0 +theoutnet.com,theoutnet.com,inbound,019,Americas,0 +theskimm.com,theskimm.com,inbound,019,Americas,0 +thesovereigninvestor.com,sovereignsociety.com,inbound,019,Americas,0 +thirtyonegifts.com,thirtyonegifts.com,inbound,019,Americas,0 +thoughtful-mind.com,thoughtful-mind.com,inbound,019,Americas,0 +thumbtack.com,thumbtack.com,inbound,019,Americas,1 +ticketmaster.com,ticketmaster.com,inbound,019,Americas,0.769059 +tmart.com,chtah.net,inbound,019,Americas,0 +tmp.com,tmpw.com,inbound,019,Americas,0 +toluna.com,toluna.com,inbound,019,Americas,0 +tomtommailer.com,tomtommailer.com,inbound,019,Americas,0 +topspin.net,topspin.net,inbound,019,Americas,1 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,019,Americas,0 +touchbase2.com,infimail.com,inbound,019,Americas,0 +touchbase2.com,mailurja.com,inbound,019,Americas,0 +touchbasepro.com,touchbasepro.com,inbound,019,Americas,0 +townhallmail.com,townhallmail.com,inbound,019,Americas,0 +townsquaremedia.info,sailthru.com,inbound,019,Americas,0 +toysrus.com,epsl1.com,inbound,019,Americas,0 +trafficboostermailer.com,trafficboostermailer.com,inbound,019,Americas,1 +trafficleads2incomevm.com,zoothost.com,inbound,019,Americas,0 +trafficprolist.com,thomas-j-brown.com,inbound,019,Americas,0 +trialsmith.com,membercentral.com,inbound,019,Americas,0 +tricaemidia.com.br,tricaemidia.com.br,inbound,019,Americas,0 +tripit.com,tripit.com,inbound,019,Americas,0 +tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,019,Americas,0 +tumblr.com,tumblr.com,inbound,019,Americas,1 +twitch.tv,justin.tv,inbound,019,Americas,0 +uber.com,uber.com,inbound,019,Americas,1 +ubi.com,ubi.com,inbound,019,Americas,0 +ubivox.com,ubivox.com,inbound,019,Americas,0.935106 +ucla.edu,ucla.edu,inbound,019,Americas,0.806204 +uhaul.com,uhaul.com,inbound,019,Americas,0.997947 +uiuc.edu,illinois.edu,inbound,019,Americas,0.999815 +ultimateadsites.net,ultimateadsites.net,inbound,019,Americas,1 +umd.edu,umd.edu,inbound,019,Americas,0.021709 +umn.edu,umn.edu,inbound,019,Americas,0.966964 +umpiredigital.com,inboxmarketer.com,inbound,019,Americas,0 +united.com,coair.com,inbound,019,Americas,1 +united.com,united.com,inbound,019,Americas,0 +universalorlando.com,universalorlando.com,inbound,019,Americas,0 +unrollmail.com,unrollmail.com,inbound,019,Americas,1 +uol.com.br,uol.com.br,inbound,019,Americas,0 +uol.com.br,uol.com.br,outbound,019,Americas,0 +upenn.edu,upenn.edu,inbound,019,Americas,0.16521 +upromise.com,delivery.net,inbound,019,Americas,0 +ups.com,ups.com,inbound,019,Americas,0.997955 +urbanoutfitters.com,freepeople.com,inbound,019,Americas,0 +usaa.com,usaa.com,inbound,019,Americas,0.072606 +usafisnews.org,usafisnews.org,inbound,019,Americas,0 +usajobs.gov,opm.gov,inbound,019,Americas,0.931948 +usc.edu,usc.edu,inbound,019,Americas,0.300314 +uscourts.gov,uscourts.gov,inbound,019,Americas,0 +uslargestsafelist.com,zoothost.com,inbound,019,Americas,0 +usndr.com,unisender.com,inbound,019,Americas,1 +usp.br,usp.br,inbound,019,Americas,0.044595 +usps.com,usps.gov,inbound,019,Americas,0.909591 +usps.gov,usps.gov,inbound,019,Americas,0.940315 +ustream.tv,mailgun.net,inbound,019,Americas,1 +ustream.tv,mailgun.us,inbound,019,Americas,1 +utfsm.cl,utfsm.cl,inbound,019,Americas,0.000352 +vacationstogo.com,vacationstogo.com,inbound,019,Americas,0 +vagas.com.br,vagas.com.br,inbound,019,Americas,0 +valuedopinions.co.uk,researchnow-usa.com,inbound,019,Americas,1 +vanheusenrewards.com,vanheusenrewards.com,inbound,019,Americas,0 +velocityfrequentflyer.com,virginaustralia.com,inbound,019,Americas,0 +vente-exclusive.com,vente-exclusive.com,inbound,019,Americas,0 +venusswimwear.net,venusswimwear.net,inbound,019,Americas,0 +verabradleymail.com,verabradleymail.com,inbound,019,Americas,0 +verizon.net,verizon.net,inbound,019,Americas,0.00713 +verizon.net,verizon.net,outbound,019,Americas,0 +verizonwireless.com,verizonwireless.com,inbound,019,Americas,0.972245 +vhmnetworkemail.com,jobdiagnosis.com,inbound,019,Americas,0 +viagogo.com,chtah.net,inbound,019,Americas,0 +viajanet.com.br,viajanet.com.br,inbound,019,Americas,0 +victoriassecret.com,victoriassecret.com,inbound,019,Americas,0 +videotron.ca,videotron.ca,inbound,019,Americas,0.005886 +vikingrivercruises.com,bfi0.com,inbound,019,Americas,0 +vimeo.com,vimeo.com,inbound,019,Americas,0 +vindale.com,vindale.com,inbound,019,Americas,0 +vipvoice.com,npdor.com,inbound,019,Americas,0 +viraladmagnet.com,viraladmagnet.com,inbound,019,Americas,1 +virginia.edu,virginia.edu,inbound,019,Americas,0.055153 +virtualtarget.com.br,virtualtarget.com.br,inbound,019,Americas,0 +vitacost.com,vitacost.com,inbound,019,Americas,0 +vitaladviews.com,zoothost.com,inbound,019,Americas,0 +vivastreet.com,viwii.net,inbound,019,Americas,0 +votervoice.net,votervoice.net,inbound,019,Americas,0 +vresp.com,verticalresponse.com,inbound,019,Americas,0 +vt.edu,vt.edu,inbound,019,Americas,0.978548 +vtext.com,vtext.com,inbound,019,Americas,0 +vtext.com,vtext.com,outbound,019,Americas,1 +vuezone.com,vuezone.com,inbound,019,Americas,0 +vzwpix.com,vtext.com,inbound,019,Americas,0 +wagjag.com,wagjag.com,inbound,019,Americas,0 +walgreens.com,walgreens.com,inbound,019,Americas,0.006128 +wallst.com,wallst.com,inbound,019,Americas,0 +wallstreetdaily.com,wallstreetdaily.com,inbound,019,Americas,0 +waterstones.com,waterstones.com,inbound,019,Americas,0 +wattpad.com,wattpad.com,inbound,019,Americas,1 +way2movies.net,way2movies.net,inbound,019,Americas,0 +wayfair.com,csnstores.com,inbound,019,Americas,0 +webs.com,epsl1.com,inbound,019,Americas,0 +websaver.ca,websaver.ca,inbound,019,Americas,0 +websitesettings.com,stabletransit.com,inbound,019,Americas,0 +websitewelcome.com,websitewelcome.com,inbound,019,Americas,1 +webstars2k.com,webstars2k.com,inbound,019,Americas,1 +weebly.com,weeblymail.com,inbound,019,Americas,0 +wellsfargo.com,wellsfargo.com,inbound,019,Americas,1.0 +wellsfargoadvisors.com,wellsfargo.com,inbound,019,Americas,1 +westelm.com,westelm.com,inbound,019,Americas,0 +westmarine.com,westmarine.com,inbound,019,Americas,0 +wetransfer.com,wetransfer.com,inbound,019,Americas,1 +wetsealnewsletter.com,wetsealnewsletter.com,inbound,019,Americas,0 +whatcounts.com,wc09.net,inbound,019,Americas,0 +whentowork.com,whentowork.com,inbound,019,Americas,0 +wikia.com,wikia.com,inbound,019,Americas,1 +williams-sonoma.com,williams-sonoma.com,inbound,019,Americas,0 +windstream.net,windstream.net,inbound,019,Americas,0.002172 +wine.com,wine.com,inbound,019,Americas,0 +winkalmail.com,fnbox.com,inbound,019,Americas,0 +wisdomitservices.com,infimail.com,inbound,019,Americas,0 +wldemail-mailer.com,wldemail.com,inbound,019,Americas,0 +womanwithin.com,womanwithin.com,inbound,019,Americas,0 +woodforest.com,woodforest.com,inbound,019,Americas,1 +workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,019,Americas,0 +worldsingles.co,worldsingles.com,inbound,019,Americas,0 +wotif.com,whatcounts.com,inbound,019,Americas,0 +wp.com,wordpress.com,inbound,019,Americas,0 +wuaki.tv,chtah.net,inbound,019,Americas,0 +wustl.edu,app-info.net,inbound,019,Americas,0 +wwe.com,wwe.com,inbound,019,Americas,8e-06 +xen.org,xen.org,inbound,019,Americas,1 +xmeeting.com,xmeeting.com,inbound,019,Americas,1 +xmr3.com,messagereach.com,inbound,019,Americas,0.999933 +xoom.com,xoom.com,inbound,019,Americas,0 +xpnews.com.br,xpnews.com.br,inbound,019,Americas,1 +yahoo.{...},postini.com,inbound,019,Americas,0.664066 +yahoo.{...},yahoo.{...},inbound,019,Americas,0.999 +yahoo.{...},yahoodns.net,outbound,019,Americas,1 +yelp.com,yelpcorp.com,inbound,019,Americas,0 +yipit.com,yipit.com,inbound,019,Americas,1 +ymail.com,yahoo.{...},inbound,019,Americas,1 +ymail.com,yahoodns.net,outbound,019,Americas,1 +yoox.com,yoox.com,inbound,019,Americas,0 +youmail.com,youmail.com,inbound,019,Americas,0 +youreletters3.com,equitymaster.com,inbound,019,Americas,0 +yourezads.com,yourezads.com,inbound,019,Americas,0.002974 +yourezlist.com,simplicityads.com,inbound,019,Americas,9.5e-05 +yourhostingaccount.com,yourhostingaccount.com,inbound,019,Americas,0 +youversion.com,youversion.com,inbound,019,Americas,1 +zacks.com,zacks.com,inbound,019,Americas,0.999687 +zara.com,cheetahmail.com,inbound,019,Americas,0 +zendesk.com,zdsys.com,inbound,019,Americas,1 +zgalleriestyle.com,zgalleriestyle.com,inbound,019,Americas,0 +zibmail.info,zibmail.info,inbound,019,Americas,9e-06 +zinio.com,zinio.com,inbound,019,Americas,0 +zipalerts.com,zipalerts.com,inbound,019,Americas,1 +ziprealty.com,ziprealty.com,inbound,019,Americas,0.994545 +ziprecruiter.com,ziprecruiter.com,inbound,019,Americas,1 +zivamewear.com,infimail.com,inbound,019,Americas,0 +zoothost.com,zoothost.com,inbound,019,Americas,0.107168 +zorpia.com,zorpia.com,inbound,019,Americas,1 +zulily.com,zulily.com,inbound,019,Americas,0 +zzounds.com,zzounds.com,inbound,019,Americas,0 +0101.co.jp,0101.co.jp,inbound,142,Asia,0 +04auto.biz,01auto.biz,inbound,142,Asia,0 +123.com.tw,123.com.tw,inbound,142,Asia,0 +160by2.us,160by2.us,inbound,142,Asia,0 +163.com,163.com,inbound,142,Asia,0.534088 +163.com,netease.com,outbound,142,Asia,1 +17life.com.tw,17life.com.tw,inbound,142,Asia,0 +1lejend.com,asumeru.com,inbound,142,Asia,0 +1lejend.com,asumeru001.com,inbound,142,Asia,0 +33go.com.tw,33go.com.tw,inbound,142,Asia,0 +518.com.tw,518.com.tw,inbound,142,Asia,0 +7net.com.tw,7net.com.tw,inbound,142,Asia,0 +ab0.jp,altovision.co.jp,inbound,142,Asia,0 +activetrail.com,atmailsvr.net,inbound,142,Asia,0 +activetrail.com,mymarketing.co.il,inbound,142,Asia,0 +adchiever.com,kinder-rash-marketing.com,inbound,142,Asia,0 +adityabirla.com,adityabirla.com,inbound,142,Asia,0.00615 +agora.co.il,1host.co.il,inbound,142,Asia,0 +airtel.com,airtel.in,inbound,142,Asia,0.064377 +alertsindia.in,alertsindia.in,inbound,142,Asia,1 +alibaba.com,alibaba.com,inbound,142,Asia,0 +aliexpress.com,alibaba.com,inbound,142,Asia,0 +alipay.com,alipay.com,inbound,142,Asia,1 +alljob.co.il,alljob.co.il,inbound,142,Asia,0 +amazon.{...},amazon.{...},inbound,142,Asia,0 +ana.co.jp,ana.co.jp,inbound,142,Asia,0.534416 +anghami.com,mailgun.net,inbound,142,Asia,1 +apple.com,apple.com,inbound,142,Asia,0.999817 +artscow.com,dyxnet.com,inbound,142,Asia,0.000355 +asus.com,asus.com,inbound,142,Asia,0 +baycrews.co.jp,webcas.net,inbound,142,Asia,0 +beamtele.com,beamtele.com,inbound,142,Asia,0 +beanfun.com,beanfun.com,inbound,142,Asia,1 +belluna.net,belluna.net,inbound,142,Asia,0 +betrend.com,betrend.com,inbound,142,Asia,0 +bharatmatrimony.com,bharatmatrimony.com,inbound,142,Asia,1 +blayn.jp,bserver.jp,inbound,142,Asia,0 +bme.jp,bserver.jp,inbound,142,Asia,0 +bookoffonline.co.jp,bookoffonline.co.jp,inbound,142,Asia,0 +brands4friends.jp,webcas.net,inbound,142,Asia,0 +brandsfever.com,mailgun.net,inbound,142,Asia,1 +buyma.com,buyma.com,inbound,142,Asia,0 +cam2life.com,hinet.net,inbound,142,Asia,0 +camsonline.com,camsonline.com,inbound,142,Asia,0.136261 +careesma.in,careesma.in,inbound,142,Asia,0 +ccavenue.com,avenues.info,inbound,142,Asia,1 +chance.com,data-hotel.net,inbound,142,Asia,0 +chiangcn.com,chiangcn.com,inbound,142,Asia,0 +chinatrust.com.tw,chinatrust.com.tw,inbound,142,Asia,0.001272 +cityheaven.net,cityheaven.net,inbound,142,Asia,0 +clickmailer.jp,clickmailer.jp,inbound,142,Asia,9e-05 +cocacola.co.jp,cocacola.co.jp,inbound,142,Asia,0 +combzmail.jp,combzmail.jp,inbound,142,Asia,0 +communitymatrimony.com,communitymatrimony.com,inbound,142,Asia,1 +conrepmail.com,conrepmail.com,inbound,142,Asia,0 +cookpad.com,cookpad.com,inbound,142,Asia,0 +cpc.gov.in,cpc.gov.in,inbound,142,Asia,0 +crmstyle.com,crmstyle.com,inbound,142,Asia,0 +crocos.jp,crocos.jp,inbound,142,Asia,0 +ctrip.com,ctrip.com,inbound,142,Asia,0.019448 +cuenote.jp,cuenote.jp,inbound,142,Asia,0 +cw.com.tw,cw.com.tw,inbound,142,Asia,0.002535 +cyberlinkmember.com,cyberlinkmember.com,inbound,142,Asia,0 +dena.ne.jp,dena.ne.jp,inbound,142,Asia,0.000249 +dietnavi.com,data-hotel.net,inbound,142,Asia,0 +dip-net.co.jp,dip-net.co.jp,inbound,142,Asia,0 +directresponsemanager.com,wide.ne.jp,inbound,142,Asia,0 +disc.co.jp,disc.co.jp,inbound,142,Asia,0.001051 +dishtv.co.in,dishtv.co.in,inbound,142,Asia,0.047664 +disqus.net,disqus.net,inbound,142,Asia,0 +dks.com.tw,dks.com.tw,inbound,142,Asia,0.018134 +dmm.com,dmm.com,inbound,142,Asia,0 +docomo.ne.jp,docomo.ne.jp,inbound,142,Asia,0 +docomo.ne.jp,docomo.ne.jp,outbound,142,Asia,0 +dreammail.ne.jp,dreammail.jp,inbound,142,Asia,0 +drushim.co.il,drushim.co.il,inbound,142,Asia,0 +dstyleweb.com,dstyleweb.com,inbound,142,Asia,0.001099 +ec21.com,ec21.com,inbound,142,Asia,0.002263 +ecnavi.jp,ecnavi.jp,inbound,142,Asia,0 +en-japan.com,en-japan.com,inbound,142,Asia,0.000204 +eonet.ne.jp,eonet.ne.jp,inbound,142,Asia,0.99955 +epaper.com.tw,epaper.com.tw,inbound,142,Asia,0 +eplus.jp,eplus.jp,inbound,142,Asia,0 +eslitebooks.com,eslitebooks.com,inbound,142,Asia,0 +euromsg.net,euromsg.net,inbound,142,Asia,0 +evaair.com,evaair.com,inbound,142,Asia,0.007363 +ezweb.ne.jp,ezweb.ne.jp,inbound,142,Asia,0.282076 +ezweb.ne.jp,ezweb.ne.jp,outbound,142,Asia,0 +farmersonly.com,mailgun.net,inbound,142,Asia,1 +felissimo.jp,felissimo.jp,inbound,142,Asia,0 +finansbank.com.tr,finansbank.com.tr,inbound,142,Asia,0.927 +fmworld.net,fmworld.net,inbound,142,Asia,0 +fofa.jp,mpme.jp,inbound,142,Asia,0 +freeml.com,gmo-media.jp,inbound,142,Asia,0 +ftchinese.com,ftchinese.com,inbound,142,Asia,0 +fubonshop.com,fubonshop.com,inbound,142,Asia,0 +gamecity.ne.jp,gamecity.ne.jp,inbound,142,Asia,0 +garanti.com.tr,garanti.com.tr,inbound,142,Asia,0.421936 +gilt.jp,gilt.jp,inbound,142,Asia,0 +globalsources.com,globalsources.com,inbound,142,Asia,0.003024 +globetel.com.ph,globetel.com.ph,inbound,142,Asia,1 +gmail.com,asianet.co.th,inbound,142,Asia,0.998511 +gmail.com,au-net.ne.jp,inbound,142,Asia,1 +gmail.com,bbtec.net,inbound,142,Asia,1 +gmail.com,data-hotel.net,inbound,142,Asia,0.000607 +gmail.com,hinet.net,inbound,142,Asia,0.973114 +gmail.com,mtnl.net.in,inbound,142,Asia,0.999527 +gmail.com,netvigator.com,inbound,142,Asia,0.818263 +gmail.com,ocn.ne.jp,inbound,142,Asia,0.99466 +gmail.com,panda-world.ne.jp,inbound,142,Asia,1 +gmail.com,seed.net.tw,inbound,142,Asia,0.992252 +gmail.com,singnet.com.sg,inbound,142,Asia,0.968389 +gmail.com,totbb.net,inbound,142,Asia,0.999876 +gmo.jp,gmo-media.jp,inbound,142,Asia,0 +gmoes.jp,gmoes.jp,inbound,142,Asia,0 +gmt.ne.jp,gmt.ne.jp,inbound,142,Asia,0 +gnavi.co.jp,gnavi.co.jp,inbound,142,Asia,0.002441 +gomaji.com,gomaji.com,inbound,142,Asia,0 +gree.jp,gree.jp,inbound,142,Asia,0 +groupon.{...},groupon.{...},inbound,142,Asia,0 +gunosy.com,gunosy.com,inbound,142,Asia,0.998682 +gurunavi.jp,gurunavi.jp,inbound,142,Asia,0 +hdfcbank.com,powerelay.com,inbound,142,Asia,1 +hdfcbank.net,powerelay.com,inbound,142,Asia,1 +hdfcbank.net,quickvmail.com,inbound,142,Asia,0 +heteml.jp,heteml.jp,inbound,142,Asia,0.950296 +hinet.net,hinet.net,inbound,142,Asia,0.007064 +hinet.net,hinet.net,outbound,142,Asia,0.00565 +home.ne.jp,zaq.ne.jp,inbound,142,Asia,9e-06 +i-part.com.tw,i-part.com.tw,inbound,142,Asia,0.001865 +ibps.in,sify.net,inbound,142,Asia,0 +ibpsorg.org,sify.net,inbound,142,Asia,0 +icicibank.com,icicibank.com,inbound,142,Asia,0.009835 +icicisecurities.com,icicibank.com,inbound,142,Asia,0.000803 +icloud.com,apple.com,inbound,142,Asia,1 +imi.ne.jp,lifemedia.jp,inbound,142,Asia,0 +indiaproperty.com,indiaproperty.com,inbound,142,Asia,0 +intage.co.jp,intage.co.jp,inbound,142,Asia,0.00557 +intuit.com,intuit.com,inbound,142,Asia,0.983698 +isbank.com.tr,isbank.com.tr,inbound,142,Asia,0.009655 +itsmyascent.com,itsmyascent.com,inbound,142,Asia,0 +itunes.com,apple.com,inbound,142,Asia,1 +jcity.com,jcity.com,inbound,142,Asia,0 +jobinthailand.com,jobinthailand.com,inbound,142,Asia,1 +jobmaster.co.il,jobmaster1.co.il,inbound,142,Asia,0 +jobsdbalert.co.id,jobsdbalert.co.id,inbound,142,Asia,0 +jobsdbalert.com,jobsdbalert.com,inbound,142,Asia,0 +jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,142,Asia,0 +jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,142,Asia,0 +jobstreet.com,jobstreet.com,inbound,142,Asia,0 +joshin.co.jp,joshin.co.jp,inbound,142,Asia,8e-06 +kagoya.net,kagoya.net,inbound,142,Asia,0.013052 +karvy.com,karvy.com,inbound,142,Asia,0.045186 +kasikornbank.com,kasikornbank.com,inbound,142,Asia,0 +kotak.com,kotak.com,inbound,142,Asia,0.005566 +krs.bz,tricorn.net,inbound,142,Asia,0 +kvbmail.com,kvbmail.com,inbound,142,Asia,0 +lancers.jp,lancers.jp,inbound,142,Asia,0 +lelong.my,lelong.com.my,inbound,142,Asia,1 +lelong.my,lelong.net.my,inbound,142,Asia,1 +line.me,naver.com,inbound,142,Asia,1 +livedoor.com,livedoor.com,inbound,142,Asia,0 +luxa.jp,luxa.jp,inbound,142,Asia,0 +m3.com,m3.com,inbound,142,Asia,0 +mag2.com,tandem-m.com,inbound,142,Asia,0 +magicbricks.com,tbsl.in,inbound,142,Asia,0 +mail-boss.com,mail-boss.com,inbound,142,Asia,0 +mailgun.org,mailgun.net,inbound,142,Asia,1 +mbga.jp,mbga.jp,inbound,142,Asia,0 +mixi.jp,mixi.jp,inbound,142,Asia,0 +mmagic.jp,mmagic.jp,inbound,142,Asia,0.000531 +mobile01.com,mobile01.com,inbound,142,Asia,0 +monex.co.jp,monex.co.jp,inbound,142,Asia,0.019424 +moneycontrol.com,active18.com,inbound,142,Asia,0 +moneyforward.com,moneyforward.com,inbound,142,Asia,0 +monipla.jp,aainc.co.jp,inbound,142,Asia,0 +monster.co.in,monster.co.in,inbound,142,Asia,0 +morhipo.com,euromsg.net,inbound,142,Asia,0 +mpme.jp,mpme.jp,inbound,142,Asia,0 +mpse.jp,emsaqua.jp,inbound,142,Asia,0 +mpse.jp,emsbeige.jp,inbound,142,Asia,0 +mpse.jp,emsbrown.jp,inbound,142,Asia,0 +mpse.jp,emscyan.jp,inbound,142,Asia,0 +mpse.jp,emsgold.jp,inbound,142,Asia,0 +mpse.jp,emslime.jp,inbound,142,Asia,0 +mpse.jp,emsnavy.jp,inbound,142,Asia,0 +mpse.jp,emspink.jp,inbound,142,Asia,0 +mpse.jp,emssnow.jp,inbound,142,Asia,0 +mpse.jp,mpme.jp,inbound,142,Asia,0 +mpse.jp,yahoo.co.jp,inbound,142,Asia,0 +mynavi.jp,mynavi.jp,inbound,142,Asia,3.5e-05 +naver.com,naver.com,inbound,142,Asia,1 +naver.com,naver.com,outbound,142,Asia,1 +nesinemail.com,euromsg.net,inbound,142,Asia,0 +net-survey.jp,net-survey.jp,inbound,142,Asia,0 +netbk.co.jp,netbk.co.jp,inbound,142,Asia,0.010995 +next-engine.org,next-engine.org,inbound,142,Asia,1 +nextdoor.com,nextdoor.com,inbound,142,Asia,1 +nic.in,relayout.nic.in,inbound,142,Asia,0 +nicovideo.jp,nicovideo.jp,inbound,142,Asia,0 +nifty.com,nifty.com,inbound,142,Asia,0.762068 +nikkei.com,nikkei.co.jp,inbound,142,Asia,0 +nikkeibp.co.jp,nikkeibp.co.jp,inbound,142,Asia,4.9e-05 +nissen.jp,nissen.jp,inbound,142,Asia,0 +ocn.ad.jp,ocn.ad.jp,inbound,142,Asia,0 +ocn.ne.jp,ocn.ad.jp,inbound,142,Asia,0 +p-world.co.jp,p-world.co.jp,inbound,142,Asia,0 +panasonic.jp,panasonic.jp,inbound,142,Asia,0 +pepabo.com,pepabo.com,inbound,142,Asia,0 +pia.jp,pia.jp,inbound,142,Asia,0 +playstation.com,playstation.com,inbound,142,Asia,0 +pointtown.com,gmo-media.jp,inbound,142,Asia,0 +pttplc.com,pttgrp.com,inbound,142,Asia,0 +publicators.com,publicators.com,inbound,142,Asia,0 +qoo10.jp,qoo10.jp,inbound,142,Asia,0 +qq.com,qq.com,inbound,142,Asia,0.999973 +quickbooks.com,intuit.com,inbound,142,Asia,0.99908 +rakuten.co.jp,rakuten.co.jp,inbound,142,Asia,0 +rakuten.co.jp,yahoo.co.jp,inbound,142,Asia,0 +rakuten.ne.jp,rakuten.co.jp,inbound,142,Asia,0 +realus.co.jp,realus.co.jp,inbound,142,Asia,0 +recochoku.jp,recochoku.jp,inbound,142,Asia,0 +rediffmail.com,akadns.net,outbound,142,Asia,0 +rediffmail.com,rediffmail.com,inbound,142,Asia,0 +relianceada.com,relianceada.com,inbound,142,Asia,0.276515 +research-panel.jp,research-panel.jp,inbound,142,Asia,0 +responder.co.il,responder.co.il,inbound,142,Asia,1 +runnet.jp,runnet.jp,inbound,142,Asia,0 +rutenmail.com.tw,rutenmail.com.tw,inbound,142,Asia,0 +sahibinden.com,sahibinden.com,inbound,142,Asia,0 +saisoncard.co.jp,saisoncard.co.jp,inbound,142,Asia,0 +sakura.ne.jp,sakura.ne.jp,inbound,142,Asia,0.673305 +samsung.com,samsung.com,inbound,142,Asia,0.008685 +saramin.co.kr,saramin.co.kr,inbound,142,Asia,0 +sbi.co.in,sbi.co.in,inbound,142,Asia,0 +sbr-inc.co.jp,hdemail.jp,inbound,142,Asia,0 +sc.com,sc.com,inbound,142,Asia,0.99683 +secure.ne.jp,secure.ne.jp,inbound,142,Asia,3.7e-05 +secureserver.net,secureserver.net,inbound,142,Asia,0.000386 +shinseibank.com,shinseibank.com,inbound,142,Asia,0 +shukatsu.jp,shukatsu.jp,inbound,142,Asia,0 +simplymarry.com,tbsl.in,inbound,142,Asia,0 +smartphoneexperts.com,mailgun.net,inbound,142,Asia,1 +smp.ne.jp,smp.ne.jp,inbound,142,Asia,0 +snapdealmail.in,snapdealmail.in,inbound,142,Asia,0 +sofmap.com,sofmap.com,inbound,142,Asia,0.0092 +softbank.jp,softbank.jp,inbound,142,Asia,0 +sony.jp,sony.jp,inbound,142,Asia,0.000654 +sourcenext.info,sourcenext.info,inbound,142,Asia,0 +surfmandelivery.com,surfmandelivery.com,inbound,142,Asia,0 +synergy360.jp,crmstyle.com,inbound,142,Asia,0 +taipeifubon.com.tw,taipeifubon.com.tw,inbound,142,Asia,0 +techgig.com,tbsl.in,inbound,142,Asia,0 +thinkvidya.com,thinkvidya.com,inbound,142,Asia,0 +ticketmonster.co.kr,ticketmonster.co.kr,inbound,142,Asia,0 +timesjobs.com,tbsl.in,inbound,142,Asia,0 +timesjobsmail.com,tbsl.in,inbound,142,Asia,0 +timesofindia.com,indiatimes.com,inbound,142,Asia,0 +tobizaru.jp,tobizaru.jp,inbound,142,Asia,0 +tocoo.jp,aics.ne.jp,inbound,142,Asia,0 +tower.jp,tower.jp,inbound,142,Asia,0 +treemall.com.tw,symphox.com,inbound,142,Asia,0 +tsite.jp,tsite.jp,inbound,142,Asia,0 +tsutaya.co.jp,tsutaya.co.jp,inbound,142,Asia,0 +turkcell.com.tr,turkcell.com.tr,inbound,142,Asia,0.043492 +type.jp,type.jp,inbound,142,Asia,0 +u-shopping.com.tw,u-shopping.com.tw,inbound,142,Asia,0 +udnpaper.com,udnpaper.com,inbound,142,Asia,0.998858 +udnshopping.com,udnshopping.com,inbound,142,Asia,0 +vodafone.com,vodafone.in,inbound,142,Asia,0.617518 +vpass.ne.jp,clickmailer.jp,inbound,142,Asia,0 +vpcontact.com,vpcontact.com,inbound,142,Asia,0 +way2sms.biz,way2sms.biz,inbound,142,Asia,0 +way2sms.in,way2sms.in,inbound,142,Asia,0 +webcas.net,webcas.net,inbound,142,Asia,0 +wechat.com,qq.com,inbound,142,Asia,1 +www.gov.tw,hinet.net,inbound,142,Asia,8e-05 +yahoo.co.jp,yahoo.co.jp,inbound,142,Asia,8e-06 +yahoo.co.jp,yahoo.co.jp,outbound,142,Asia,0 +yahoo.{...},yahoo.co.jp,inbound,142,Asia,0 +yakala.co,euromsg.net,inbound,142,Asia,0 +yesbank.in,yesbank.in,inbound,142,Asia,0.241591 +yodobashi.com,yodobashi.com,inbound,142,Asia,0 +zizigo.com,euromsg.net,inbound,142,Asia,0 +3suisses.be,3suisses.be,inbound,150,Europe,0 +3suisses.fr,3suisses.fr,inbound,150,Europe,0 +aanotifier.nl,aanotifier.nl,inbound,150,Europe,0 +adidas.com,neolane.net,inbound,150,Europe,0 +admail.hu,sanomaonline.hu,inbound,150,Europe,0 +adsender.us,adsender.us,inbound,150,Europe,0 +adverts.ie,adverts.ie,inbound,150,Europe,1 +advfn.com,advfn.com,inbound,150,Europe,0.635382 +agnitas.de,agnitas.de,inbound,150,Europe,0.999265 +airliquide.com,airliquide.com,inbound,150,Europe,1 +alerteimmo.com,alerteimmo.com,inbound,150,Europe,0 +alinea.fr,bp06.net,inbound,150,Europe,0 +allegro.pl,allegro.pl,inbound,150,Europe,0 +allegroup.hu,allegroup.hu,inbound,150,Europe,0 +alza.cz,alza.cz,inbound,150,Europe,0.039449 +alza.sk,alza.cz,inbound,150,Europe,0.027725 +andrewchristian.com,emv8.com,inbound,150,Europe,0 +anpasia.com,anpasia.com,inbound,150,Europe,0 +aprovaconcursos.com.br,eadunicid.com.br,inbound,150,Europe,0 +aruba.it,aruba.it,inbound,150,Europe,0.055666 +ashampoo.com,ashampoo.com,inbound,150,Europe,1 +aswatson.com,emarsys.net,inbound,150,Europe,0 +avira.com,avira.com,inbound,150,Europe,0.042528 +avito.ru,avito.ru,inbound,150,Europe,0.000631 +badoo.com,monopost.com,inbound,150,Europe,1 +balsamik.fr,balsamik.fr,inbound,150,Europe,0 +bancomer.com,postini.com,inbound,150,Europe,1e-05 +bbvacompass.com,postini.com,inbound,150,Europe,0.997006 +be2.com,nmp1.net,inbound,150,Europe,0 +biglion.ru,biglion.ru,inbound,150,Europe,0.999775 +bigmailsender.com,bigmailsender.com,inbound,150,Europe,0 +bioagri.com.br,postini.com,inbound,150,Europe,0.991765 +biomedcentral.com,emv5.com,inbound,150,Europe,0 +blackberry.com,blackberry.com,inbound,150,Europe,0 +blinkboxmusic.com,mediagraft.com,inbound,150,Europe,1 +blogtrottr.com,blogtrottr.com,inbound,150,Europe,0 +blue-compass.com,blue-compass.com,inbound,150,Europe,0 +bmdeda99.com,bmdeda99.com,inbound,150,Europe,0 +bolsfr.fr,colt.net,inbound,150,Europe,0 +bonuszbrigad.hu,bonuszbrigad.hu,inbound,150,Europe,0 +boohooemail.com,smartfocusdigital.net,inbound,150,Europe,0 +booking.com,booking.com,inbound,150,Europe,1 +bouncemanager.it,musvc.com,inbound,150,Europe,0.362026 +br.com,cmailsys.com,inbound,150,Europe,0 +brandalley.com,brandalley.com,inbound,150,Europe,0 +brands4friends.de,emv5.com,inbound,150,Europe,0 +brandsvillage.net,brandsvillage.net,inbound,150,Europe,0 +bt.com,bt.com,inbound,150,Europe,0.409404 +bweeble.com,adlabsinc.com,inbound,150,Europe,0 +cabestan.com,cab07.net,inbound,150,Europe,0 +cadremploi.fr,cadremploi.fr,inbound,150,Europe,0 +cardsys.at,cardsys.at,inbound,150,Europe,1 +carmamail.com,carmamail.com,inbound,150,Europe,0 +casasbahia.com.br,casasbahia.com.br,inbound,150,Europe,0 +catchoftheday.com.au,inxserver.de,inbound,150,Europe,1 +catererglobal.com,madgexjb.com,inbound,150,Europe,0 +caterermail.com,totaljobsmail.co.uk,inbound,150,Europe,0 +cathkidston.com,cathkidston.co.uk,inbound,150,Europe,0 +cccampaigns.com,emv5.com,inbound,150,Europe,0 +cccampaigns.com,emv8.com,inbound,150,Europe,0 +cccampaigns.net,01net.com,inbound,150,Europe,0 +cccampaigns.net,cccampaigns.net,inbound,150,Europe,0 +cccampaigns.net,emv4.net,inbound,150,Europe,0 +ccmbg.com,benchmark.fr,inbound,150,Europe,0 +cdongroup.com,cdongroup.com,inbound,150,Europe,0.000147 +cheapflights.co.uk,cheapflights.co.uk,inbound,150,Europe,0 +cheapflights.com,cheapflights.com,inbound,150,Europe,0 +chelseafc.com,chelseafc.com,inbound,150,Europe,0.003566 +cinesa.es,cccampaigns.com,inbound,150,Europe,0 +cipherzone.com,infimail.com,inbound,150,Europe,0 +citybrands.hu,webinform.hu,inbound,150,Europe,1 +clickon.com.br,clickon.com.br,inbound,150,Europe,0 +clicplan.com,dmdelivery.com,inbound,150,Europe,0 +cobone.com,emarsys.net,inbound,150,Europe,0 +communicatoremail.com,communicatoremail.com,inbound,150,Europe,0 +compute.internal,amazonaws.com,inbound,150,Europe,0.81478 +confirmedoptin.com,confirmedoptin.com,inbound,150,Europe,0 +continente.pt,1-hostingservice.com,inbound,150,Europe,0 +cratusservices.in,ramcorp.in,inbound,150,Europe,0 +cricinfo.com,cricinfo.com,inbound,150,Europe,0 +critsend.com,critsend.com,inbound,150,Europe,0 +crsend.com,crsend.com,inbound,150,Europe,0.008688 +csas.cz,csas.cz,inbound,150,Europe,0.999971 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,150,Europe,1 +cv-library.co.uk,cv-library.co.uk,inbound,150,Europe,0 +cvbankas.lt,efadm.eu,inbound,150,Europe,0 +cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 +d-reizen.nl,dmdelivery.com,inbound,150,Europe,0 +dafiti.com.br,fagms.de,inbound,150,Europe,0 +datingfactory.com,caerussolutions.net,inbound,150,Europe,0 +dbgi.co.uk,emc1.co.uk,inbound,150,Europe,0 +deal.com.sg,emarsys.net,inbound,150,Europe,0 +debian.org,debian.org,inbound,150,Europe,1 +deezer.com,dms30.com,inbound,150,Europe,0 +directcrm.ru,directcrm.ru,inbound,150,Europe,0 +disney.co.uk,emv9.com,inbound,150,Europe,0 +dominosemail.co.uk,dominosemail.co.uk,inbound,150,Europe,0 +doodle.com,doodle.com,inbound,150,Europe,1 +dotmailer-email.com,dotmailer.com,inbound,150,Europe,0 +dotmailer.co.uk,dotmailer.com,inbound,150,Europe,0 +dpapp.nl,prikbordmailer.nl,inbound,150,Europe,0 +dpapp.nl,sslsecuref.nl,inbound,150,Europe,0 +dreivip.com,dreivip.com,inbound,150,Europe,0.000301 +dress-for-less.de,privalia.com,inbound,150,Europe,0 +drweb.com,drweb.com,inbound,150,Europe,0.969315 +e-boks.dk,e-boks.dk,inbound,150,Europe,1 +e-ebuyer.com,e-ebuyer.com,inbound,150,Europe,0 +e-mark.nl,e-mark.nl,inbound,150,Europe,0 +e-ngine.nl,e-ngine.nl,inbound,150,Europe,0 +ebay-kleinanzeigen.de,mobile.de,inbound,150,Europe,1 +ecommzone.com,ecommzone.com,inbound,150,Europe,0 +edarling.fr,fagms.de,inbound,150,Europe,0 +edima.hu,edima.hu,inbound,150,Europe,0 +efox-shop.com,dmdelivery.com,inbound,150,Europe,0 +ejobs.ro,ejobs.ro,inbound,150,Europe,8.5e-05 +elaine-asp.de,artegic.net,inbound,150,Europe,0.999997 +elcorteingles.es,elcorteingles.es,inbound,150,Europe,0 +elektronskaposta.si,eprvak.si,inbound,150,Europe,0 +elettershop.de,servicemail24.de,inbound,150,Europe,1 +emag.ro,emag.ro,inbound,150,Europe,0.005013 +email-comparethemarket.com,smartfocusdigital.net,inbound,150,Europe,0 +email360api.com,email360api.com,inbound,150,Europe,0 +emarsys.net,emarsys.net,inbound,150,Europe,0 +embluejet.com,emblueuser.com,inbound,150,Europe,0 +emsecure.net,emsecure.net,inbound,150,Europe,0 +emsmtp.com,emsmtp.com,inbound,150,Europe,0 +enewsletter.pl,enewsletter.pl,inbound,150,Europe,0.007349 +enewsletter.pl,sare25.com,inbound,150,Europe,0 +espmp-agfr.net,bp06.net,inbound,150,Europe,0 +esprit-friends.com,esprit-friends.com,inbound,150,Europe,0 +evanscycles.com,msgfocus.com,inbound,150,Europe,0 +experteer.com,experteer.com,inbound,150,Europe,0 +extra.com.br,emv8.com,inbound,150,Europe,0 +eyepin.com,eyepin.com,inbound,150,Europe,0 +fabfurnish.com,fagms.de,inbound,150,Europe,0 +facilisimo.com,facilisimo.com,inbound,150,Europe,1 +fagms.net,fagms.de,inbound,150,Europe,0 +finn.no,schibsted-it.no,inbound,150,Europe,0.002893 +fixeads.com,fixeads.com,inbound,150,Europe,0 +flirchi.com,flirchi.com,inbound,150,Europe,1.0 +flymonarchemail.com,flymonarchemail.com,inbound,150,Europe,0 +follow-up.se,follow-up.se,inbound,150,Europe,1 +fotocasa.es,fotocasa.es,inbound,150,Europe,0 +fotostrana.ru,fotocdn.net,inbound,150,Europe,3.4e-05 +free-lance.ru,free-lance.ru,inbound,150,Europe,0 +free.fr,free.fr,inbound,150,Europe,0.984012 +free.fr,free.fr,outbound,150,Europe,6.9e-05 +freecycle.org,freecycle.org,inbound,150,Europe,0.999865 +freemail.hu,freemail.hu,outbound,150,Europe,0 +freshmail.pl,freshmail.pl,inbound,150,Europe,0 +gardeningclubmail.co.uk,msgfocus.com,inbound,150,Europe,0 +giffgaff.com,giffgaff.com,inbound,150,Europe,0 +globasemail.com,globasemail.com,inbound,150,Europe,0.951088 +gmail.com,02.net,inbound,150,Europe,0.999617 +gmail.com,as13285.net,inbound,150,Europe,0.999943 +gmail.com,bbox.fr,inbound,150,Europe,0.919849 +gmail.com,belgacom.be,inbound,150,Europe,0.999444 +gmail.com,blackberry.com,inbound,150,Europe,0.998923 +gmail.com,bluewin.ch,inbound,150,Europe,0.93294 +gmail.com,btcentralplus.com,inbound,150,Europe,0.999969 +gmail.com,chello.nl,inbound,150,Europe,0.999951 +gmail.com,fastwebnet.it,inbound,150,Europe,0.984706 +gmail.com,jazztel.es,inbound,150,Europe,0.999701 +gmail.com,net24.it,inbound,150,Europe,0.999964 +gmail.com,netcabo.pt,inbound,150,Europe,0.998164 +gmail.com,numericable.fr,inbound,150,Europe,0.999723 +gmail.com,ono.com,inbound,150,Europe,0.995991 +gmail.com,orange.es,inbound,150,Europe,0.998742 +gmail.com,orange.fr,inbound,150,Europe,0 +gmail.com,otenet.gr,inbound,150,Europe,0.957917 +gmail.com,postini.com,inbound,150,Europe,0.924066 +gmail.com,proxad.net,inbound,150,Europe,0.998094 +gmail.com,rima-tde.net,inbound,150,Europe,0.99915 +gmail.com,sfr.net,inbound,150,Europe,0.999877 +gmail.com,skybroadband.com,inbound,150,Europe,0.999854 +gmail.com,t-ipconnect.de,inbound,150,Europe,0.999841 +gmail.com,tdc.net,inbound,150,Europe,0.999591 +gmail.com,telecomitalia.it,inbound,150,Europe,0.997 +gmail.com,telekom.hu,inbound,150,Europe,0.999977 +gmail.com,telenet.be,inbound,150,Europe,1 +gmail.com,telepac.pt,inbound,150,Europe,0.999584 +gmail.com,telia.com,inbound,150,Europe,1 +gmail.com,threembb.co.uk,inbound,150,Europe,1 +gmail.com,tpnet.pl,inbound,150,Europe,0.999761 +gmail.com,virginm.net,inbound,150,Europe,0.99661 +gmail.com,vodafone-ip.de,inbound,150,Europe,1 +gmail.com,vodafone.pt,inbound,150,Europe,0.999563 +gmail.com,vodafonedsl.it,inbound,150,Europe,0.999006 +gmail.com,wanadoo.fr,inbound,150,Europe,0.999752 +gmail.com,ziggo.nl,inbound,150,Europe,1 +gmx.de,gmx.net,inbound,150,Europe,1 +gmx.de,gmx.net,outbound,150,Europe,1 +gmx.net,gmx.net,inbound,150,Europe,1 +goalunited.org,ccmdcampaigns.net,inbound,150,Europe,0 +gog.com,gog.com,inbound,150,Europe,0 +gogroopie.com,gogroopie.com,inbound,150,Europe,0.000283 +goldenline.pl,goldenline.pl,inbound,150,Europe,1 +goodgame.com,emsmtp.com,inbound,150,Europe,0 +goodlife.pt,emv8.com,inbound,150,Europe,0 +google.com,postini.com,inbound,150,Europe,0.810567 +googlemail.com,t-ipconnect.de,inbound,150,Europe,0.999957 +gumtree.com,marktplaats.nl,inbound,150,Europe,0 +gumtree.com.au,kijiji.com,inbound,150,Europe,0 +gymglish.com,gymglish.com,inbound,150,Europe,0.000788 +haskell.org,haskell.org,inbound,150,Europe,0.000703 +hazteoir.org,hazteoir.org,inbound,150,Europe,0.31478 +hh.ru,hh.ru,inbound,150,Europe,0.996236 +hotel.de,emp-mail.de,inbound,150,Europe,0 +hotornot.com,monopost.com,inbound,150,Europe,0.983953 +hotukdeals.com,hotukdeals.com,inbound,150,Europe,1 +hpnotifier.nl,hpnotifier.nl,inbound,150,Europe,0 +icelandmail.co.uk,emsg-live.co.uk,inbound,150,Europe,0 +idealista.com,idealista.com,inbound,150,Europe,0.001627 +ideascost.com,ramcorp.in,inbound,150,Europe,0 +inboxair.com,inboxair.com,inbound,150,Europe,0 +infoempleo.com,infoempleo.com,inbound,150,Europe,0 +infojobs.it,infojobs.it,inbound,150,Europe,0 +infojobs.net,infojobs.net,inbound,150,Europe,0 +infopraca.pl,careesma.com,inbound,150,Europe,0 +ingdirect.es,ingdirect.es,inbound,150,Europe,1 +inter-chat.com,inter-chat.com,inbound,150,Europe,0 +interdatesa.com,fagms.net,inbound,150,Europe,0 +internations.org,internations.org,inbound,150,Europe,1 +inx1and1.de,1and1.com,inbound,150,Europe,1 +inxserver.com,inxserver.de,inbound,150,Europe,0.952863 +inxserver.de,inxserver.de,inbound,150,Europe,0.980838 +itms.in.ua,itms.in.ua,inbound,150,Europe,0 +jiscmail.ac.uk,lsoft.se,inbound,150,Europe,0 +jobisjob.com,jobisjob.com,inbound,150,Europe,0 +jobrapidoalert.com,jobrapidoalert.com,inbound,150,Europe,0 +jobs2web.com,ondemand.com,inbound,150,Europe,1 +jobserve.com,jobserve.com,inbound,150,Europe,0 +joobmailer.com,joobmailer.com,inbound,150,Europe,0 +jumia.com.ng,fagms.de,inbound,150,Europe,0 +justclick.ru,justclick.ru,inbound,150,Europe,0 +kalunga.com.br,kalunga.com.br,inbound,150,Europe,0 +kiabi.com,dms-02.net,inbound,150,Europe,0 +kijiji.ca,kijiji.com,inbound,150,Europe,0 +kismia.com,kismia.com,inbound,150,Europe,1 +kiwari.com,kiwari.com,inbound,150,Europe,9e-06 +kundenserver.de,kundenserver.de,inbound,150,Europe,1 +laposte.net,laposte.net,inbound,150,Europe,0.271722 +laposte.net,laposte.net,outbound,150,Europe,0 +laredoute.fr,laredoute.fr,inbound,150,Europe,0 +laterooms.com,laterooms.com,inbound,150,Europe,0.014746 +leboncoin.fr,leboncoin.fr,inbound,150,Europe,0 +leparisien.fr,leparisien.fr,inbound,150,Europe,0 +lexpress.fr,bp06.net,inbound,150,Europe,0 +libero.it,libero.it,inbound,150,Europe,0.00024 +libero.it,libero.it,outbound,150,Europe,0 +lifecooler.com,1-hostingservice.com,inbound,150,Europe,0 +listadventure.com,adlabsinc.com,inbound,150,Europe,0 +listjoe.com,adlabsinc.com,inbound,150,Europe,0 +litres.ru,litres.ru,inbound,150,Europe,1 +loccitane.com,neolane.net,inbound,150,Europe,0 +logentries.com,logentries.com,inbound,150,Europe,0 +loveplanet.ru,pochta.ru,inbound,150,Europe,0.640035 +lowcostholidays.co.uk,communicatoremail.com,inbound,150,Europe,0 +lua.org,pepperfish.net,inbound,150,Europe,1 +ludokados.com,ludokado.com,inbound,150,Europe,0 +mail-cdiscount.com,mail-cdiscount.com,inbound,150,Europe,0 +mail-mbank.pl,mail-mbank.pl,inbound,150,Europe,0 +mail.ru,mail.ru,inbound,150,Europe,0.991269 +mail.ru,mail.ru,outbound,150,Europe,0.006918 +mailer-service.de,mailer-service.de,inbound,150,Europe,4.9e-05 +mailersend.com,mailersend.com,inbound,150,Europe,0 +mailing-list.it,mailing-list.it,inbound,150,Europe,0 +mailjet.com,mailjet.com,inbound,150,Europe,0 +mailplus.nl,brightbase.net,inbound,150,Europe,1 +mailpv.net,pvmailer.net,inbound,150,Europe,1 +maisonsdumonde.com,bp06.net,inbound,150,Europe,0 +makro.nl,srv2.de,inbound,150,Europe,0.888692 +mapfre.com,emv5.com,inbound,150,Europe,0 +marktplaats.nl,marktplaats.nl,inbound,150,Europe,0 +matchwereld.nl,matchwereld.nl,inbound,150,Europe,0 +maxpark.com,gidepark.ru,inbound,150,Europe,1 +mdirector.com,mdrctr.com,inbound,150,Europe,0 +mecumauction.com,mecumauction.com,inbound,150,Europe,0 +meetic.com,meetic.com,inbound,150,Europe,0 +mequedouno.com,mequedouno.com,inbound,150,Europe,0 +metrodeal.com,fagms.de,inbound,150,Europe,0 +mightydeals.co.uk,mightydeals.co.uk,inbound,150,Europe,0 +mirtesen.ru,mtml.ru,inbound,150,Europe,0 +mitula.net,mitula.org,inbound,150,Europe,0 +mitula.org,mitula.org,inbound,150,Europe,0 +mixcloudmail.com,mixcloudmail.com,inbound,150,Europe,0.995482 +mlgns.com,mlgns.com,inbound,150,Europe,0 +mlgnserv.com,mlgnserv.com,inbound,150,Europe,0 +mmks.it,mail-maker.it,inbound,150,Europe,0 +mmsecure.nl,donenad.nl,inbound,150,Europe,0 +modnakasta.ua,emv5.com,inbound,150,Europe,0 +mooply.co,mailendo.com,inbound,150,Europe,0 +moviestarplanet.com,moviestarplanet.com,inbound,150,Europe,0 +mrc.org,msgfocus.com,inbound,150,Europe,0 +msdp1.com,msdp1.com,inbound,150,Europe,0 +msgfocus.com,msgfocus.com,inbound,150,Europe,0.011658 +mymms.com,fagms.de,inbound,150,Europe,0 +namorico.me,namorico.me,inbound,150,Europe,1 +nasza-klasa.pl,nasza-klasa.pl,inbound,150,Europe,0 +nationalexpress.com,nationalexpress.com,inbound,150,Europe,0 +neolane.net,neolane.net,inbound,150,Europe,0 +netopia.pt,netopia.pt,inbound,150,Europe,0.028429 +nhs.jobs,nhscareersjobs.co.uk,inbound,150,Europe,0 +nieuwsblad.be,vummail.be,inbound,150,Europe,0 +nmp1.com,nmp1.net,inbound,150,Europe,0 +nos.pt,netcabo.pt,inbound,150,Europe,6.3e-05 +noticiasaominuto.com,ccmdcampaigns.net,inbound,150,Europe,0 +noticiasaominuto.com,noticiasaominuto.com,inbound,150,Europe,0 +nrholding.net,nrholding.net,inbound,150,Europe,0 +odisseias.com,emv4.net,inbound,150,Europe,0 +odnoklassniki.ru,odnoklassniki.ru,inbound,150,Europe,0 +offerum.com,cccampaigns.com,inbound,150,Europe,0 +offerum.com,ccemails.com,inbound,150,Europe,0 +olx.pt,fixeads.com,inbound,150,Europe,0 +orange.fr,orange.fr,inbound,150,Europe,0 +orange.fr,orange.fr,outbound,150,Europe,0 +oroscopofree.com,adsender.us,inbound,150,Europe,0 +oroscopofree.com,oroscopofree.com,inbound,150,Europe,0 +orsay.com,emp-mail.de,inbound,150,Europe,0 +ovh.net,ovh.net,inbound,150,Europe,0.165188 +oxfam.org.uk,msgfocus.com,inbound,150,Europe,0 +payback.de,artegic.net,inbound,150,Europe,0.999986 +pccomponentes.com,pccomponentes.com,inbound,150,Europe,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,150,Europe,0 +peperoni.de,peperoni.de,inbound,150,Europe,1 +peytz.dk,peytz.dk,inbound,150,Europe,4e-06 +photobox.com,photobox.com,inbound,150,Europe,0 +photoprintit.com,photoprintit.com,inbound,150,Europe,0 +pixum.com,pixum.com,inbound,150,Europe,0 +placedestendances.com,placedestendances.com,inbound,150,Europe,0 +plaisio.gr,fagms.de,inbound,150,Europe,0 +planeo.com,planeo.com,inbound,150,Europe,0 +planeo.pt,planeo.pt,inbound,150,Europe,0 +playtika.com,emv8.com,inbound,150,Europe,0 +poinx.com,poinx.com,inbound,150,Europe,0 +pokerstars.com,pokerstars.eu,inbound,150,Europe,0 +pokerstars.eu,pokerstars.eu,inbound,150,Europe,0 +pontofrio.com.br,emv8.com,inbound,150,Europe,0 +postgresql.org,postgresql.org,inbound,150,Europe,1 +praca.pl,praca.pl,inbound,150,Europe,1 +pracuj.pl,pracuj.pl,inbound,150,Europe,0 +printvenue.com,fagms.de,inbound,150,Europe,0 +privalia.com,privalia.com,inbound,150,Europe,3.94913202027326e-07 +promod-news.fr,promod-news.com,inbound,150,Europe,0 +protopmail.com,protopmail.com,inbound,150,Europe,0 +pur3.net,pur3.net,inbound,150,Europe,0.001123 +python.org,python.org,inbound,150,Europe,1 +quality.net.ua,quality.net.ua,inbound,150,Europe,0 +r-project.org,ethz.ch,inbound,150,Europe,0.99999 +r51.it,musvc.com,inbound,150,Europe,0.092498 +r52.it,musvc.com,inbound,150,Europe,0.000221 +r57.it,musvc.com,inbound,150,Europe,0.139425 +r67.it,musvc.com,inbound,150,Europe,0.199845 +r70.it,musvc.com,inbound,150,Europe,0.311983 +rakuten.co.jp,shareee.jp,inbound,150,Europe,0 +rambler.ru,rambler.ru,inbound,150,Europe,0.08173 +ratedpeople.com,ratedpeople.com,inbound,150,Europe,0.000979 +rax.ru,rax.ru,inbound,150,Europe,0.000258 +regie11.net,odiso.net,inbound,150,Europe,0 +relax7.hu,gruppi.hu,inbound,150,Europe,0 +richersoundsvip.com,ibwmail.com,inbound,150,Europe,0 +rightmove.com,rightmove.com,inbound,150,Europe,0 +roulartamail.be,roulartamail.be,inbound,150,Europe,0 +rueducommerce.com,groupe-rueducommerce.fr,inbound,150,Europe,0 +runtastic.com,runtastic.com,inbound,150,Europe,0 +ryanairmail.com,ryanairmail.com,inbound,150,Europe,0 +rzone.de,rzone.de,inbound,150,Europe,1 +saimails.in,infimail.com,inbound,150,Europe,0 +sainsburys.co.uk,emv5.com,inbound,150,Europe,0 +salesmanago.pl,salesmanago.pl,inbound,150,Europe,0 +samsung.ru,samsung.ru,inbound,150,Europe,0 +sapnetworkmail.com,sap-ag.de,inbound,150,Europe,1 +sapo.pt,sapo.pt,inbound,150,Europe,0.242839 +sapo.pt,sapo.pt,outbound,150,Europe,0 +savingdeals.in,infimail.com,inbound,150,Europe,0 +scmp.com,emarsys.net,inbound,150,Europe,0 +scoop.it,scoop.it,inbound,150,Europe,0 +scoopon.com.au,inxserver.de,inbound,150,Europe,1 +screwfix.info,fwdto.net,inbound,150,Europe,0 +secureserver.net,secureserver.net,inbound,150,Europe,0 +selection-priceminister.com,selection-priceminister.com,inbound,150,Europe,0 +sender.lt,sritis.lt,inbound,150,Europe,0.000339 +sendsmaily.info,sendsmaily.info,inbound,150,Europe,0 +seniorplanet.fr,seniorplanet.fr,inbound,150,Europe,0 +seznam.cz,seznam.cz,inbound,150,Europe,0.001781 +seznam.cz,seznam.cz,outbound,150,Europe,0.001741 +sfr.fr,sfr.fr,inbound,150,Europe,0.236539 +sfr.fr,sfr.fr,outbound,150,Europe,0.004753 +shopto.net,shopto.net,inbound,150,Europe,1 +skype.com,skype.com,inbound,150,Europe,0 +solveerrors.com,infimail.com,inbound,150,Europe,0 +spareroom.co.uk,spareroom.co.uk,inbound,150,Europe,1 +sqlservercentral.com,sqlservercentral.com,inbound,150,Europe,0 +staples-pt.com,1-hostingservice.com,inbound,150,Europe,0 +stepstone.de,stepstone.com,inbound,150,Europe,0.001689 +studentbeans.com,emv8.com,inbound,150,Europe,0 +subito.it,subito.it,inbound,150,Europe,0 +subscribe.ru,subscribe.ru,inbound,150,Europe,0 +superdrug.com,superdrug.com,inbound,150,Europe,0 +superjob.ru,superjob.ru,inbound,150,Europe,0 +support-love.com,support-love.com,inbound,150,Europe,0 +sut1.co.uk,sut1.co.uk,inbound,150,Europe,0.001789 +sut5.co.uk,sut5.co.uk,inbound,150,Europe,0 +swanson-vitamins.com,emv5.com,inbound,150,Europe,0 +t-online.de,t-online.de,inbound,150,Europe,1 +t-online.de,t-online.de,outbound,150,Europe,0.999939 +tatrabanka.sk,tatrabanka.sk,inbound,150,Europe,0 +tchibo.de,srv2.de,inbound,150,Europe,0.980447 +teamo.ru,teamo.ru,inbound,150,Europe,0 +teamviewer.com,teamviewer.com,inbound,150,Europe,0 +teapartyinfo.org,teapartyinfo.org,inbound,150,Europe,0 +telenet.be,telenet-ops.be,inbound,150,Europe,1.5e-05 +teleportmyjob.com,clara.net,inbound,150,Europe,0 +theleadmagnet.com,your-server.de,inbound,150,Europe,1 +timeweb.ru,timeweb.ru,inbound,150,Europe,1 +tiscali.it,tiscali.it,outbound,150,Europe,0 +totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 +transversal.net,transversal.net,inbound,150,Europe,1 +travisperkins.co.uk,travisperkins.co.uk,inbound,150,Europe,0 +trovit.com,trovit.com,inbound,150,Europe,0 +tucasa.com,grupodtm.com,inbound,150,Europe,0 +twoomail.com,twoomail.com,inbound,150,Europe,0 +ubuntu.com,canonical.com,inbound,150,Europe,0 +ukr.net,fwdcdn.com,inbound,150,Europe,1 +ukr.net,ukr.net,outbound,150,Europe,0.999992 +ulteem.com,ulteem.com,inbound,150,Europe,0 +usndr.com,usndr.com,inbound,150,Europe,1 +venere.com,kiwari.com,inbound,150,Europe,0 +venteprivee.com,venteprivee.com,inbound,150,Europe,0 +virgilio.it,virgilio.net,inbound,150,Europe,0 +visualsoft.co.uk,visualsoft.co.uk,inbound,150,Europe,0 +vouchercloud.com,vouchercloud.com,inbound,150,Europe,0 +voyageprive.com,cccampaigns.net,inbound,150,Europe,0 +voyageprive.es,ccmdcampaigns.net,inbound,150,Europe,0 +voyageprive.it,ccmdcampaigns.net,inbound,150,Europe,0 +voyages-sncf.com,neolane.net,inbound,150,Europe,0 +vueling.com,vueling.com,inbound,150,Europe,0 +wanadoo.fr,orange.fr,inbound,150,Europe,0 +wanadoo.fr,orange.fr,outbound,150,Europe,0 +waves-audio.com,emv8.com,inbound,150,Europe,0 +web.de,web.de,inbound,150,Europe,0.999986 +web.de,web.de,outbound,150,Europe,1 +whereareyounow.com,wayn.net,inbound,150,Europe,0 +wiggle.com,wiggle.com,inbound,150,Europe,0 +william-reed.com,neolane.net,inbound,150,Europe,0 +williamhill.com,williamhill.com,inbound,150,Europe,0 +wldemail.com,emarsys.net,inbound,150,Europe,0 +wmtransfer.com,wmtransfer.com,inbound,150,Europe,0 +wnd.com,emv4.net,inbound,150,Europe,0 +wnd.com,worldnetdaily.com,inbound,150,Europe,0 +work.ua,work.ua,inbound,150,Europe,1 +workcircle.com,workcircle.net,inbound,150,Europe,0 +wp.pl,wp.pl,inbound,150,Europe,0.998027 +wp.pl,wp.pl,outbound,150,Europe,1 +xmailix.com,xmailix.com,inbound,150,Europe,0 +yandex.ru,yandex.net,inbound,150,Europe,0.999658 +yandex.ru,yandex.ru,outbound,150,Europe,1 +ymlpserver.net,ymlpserver.net,inbound,150,Europe,0 +ymlpsrv.net,ymlpsrv.net,inbound,150,Europe,0 +zalando.be,fagms.de,inbound,150,Europe,0 +zalando.dk,fagms.de,inbound,150,Europe,0 +zalando.fi,fagms.de,inbound,150,Europe,0 +zalando.it,fagms.de,inbound,150,Europe,0 +zalando.nl,fagms.de,inbound,150,Europe,0 +zalando.pl,fagms.de,inbound,150,Europe,0 +zumzi.com,neogen.ro,inbound,150,Europe,0 +zumzi.com,zumzi.com,inbound,150,Europe,0 +0bz.biz,hmts.jp,inbound,ZZ,Unknown Region,0 +104.com.tw,104.com.tw,inbound,ZZ,Unknown Region,0.091133 +1105info.com,1105info.com,inbound,ZZ,Unknown Region,0 +1111.com.tw,1111.com.tw,inbound,ZZ,Unknown Region,0 +160by2inbox.com,160by2inbox.com,inbound,ZZ,Unknown Region,0 +160by2invite.com,160by2invite.com,inbound,ZZ,Unknown Region,0 +160by2mail.com,160by2mail.com,inbound,ZZ,Unknown Region,0 +163.com,163.com,inbound,ZZ,Unknown Region,0.996685 +1800flowersinc.com,1800flowersinc.com,inbound,ZZ,Unknown Region,0 +1sale.com,1sale.com,inbound,ZZ,Unknown Region,0 +1v1y.com,euromsg.net,inbound,ZZ,Unknown Region,0 +4shared.com,4shared.com,inbound,ZZ,Unknown Region,1 +6pm.com,6pm.com,inbound,ZZ,Unknown Region,1 +6pm.com,zappos.com,inbound,ZZ,Unknown Region,0.782581 +a8.net,a8.net,inbound,ZZ,Unknown Region,0 +aaa.com,nextjump.com,inbound,ZZ,Unknown Region,0 +aaas-science.org,aaas-science.org,inbound,ZZ,Unknown Region,0 +aarp.org,aarp.org,inbound,ZZ,Unknown Region,0 +about.com,about.com,inbound,ZZ,Unknown Region,8.2e-05 +academy-enews.com,academy-enews.com,inbound,ZZ,Unknown Region,0 +accenture.com,outlook.com,inbound,ZZ,Unknown Region,1 +accountonline.com,accountonline.com,inbound,ZZ,Unknown Region,0.348991 +acehelpfulemails.com,teradatadmc.com,inbound,ZZ,Unknown Region,0 +actorsaccess.com,nonfatmedia.com,inbound,ZZ,Unknown Region,0 +adminforfree.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +adminforfree.net,adminforfree.com,inbound,ZZ,Unknown Region,1 +administrativejobinsider.com,administrativejobinsider.com,inbound,ZZ,Unknown Region,0 +adobe.com,obsmtp.com,inbound,ZZ,Unknown Region,1 +adobesystems.com,adobesystems.com,inbound,ZZ,Unknown Region,0 +adoreme.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +adp.com,adp.com,inbound,ZZ,Unknown Region,1 +adsender.us,adsender.us,inbound,ZZ,Unknown Region,0 +adsolutionline.com,adsolutionline.com,inbound,ZZ,Unknown Region,0 +adultfriendfinder.com,friendfinder.com,inbound,ZZ,Unknown Region,0 +advanceauto.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,ZZ,Unknown Region,0 +af.mil,af.mil,inbound,ZZ,Unknown Region,0.997561 +agoda-emails.com,agoda-emails.com,inbound,ZZ,Unknown Region,0 +agrupemonos.cl,agrupemonos.cl,inbound,ZZ,Unknown Region,1 +albertsonsemail.com,email4-mywebgrocer.com,inbound,ZZ,Unknown Region,0 +alibaba.com,alibaba.com,inbound,ZZ,Unknown Region,0 +alice.it,alice.it,inbound,ZZ,Unknown Region,0 +alice.it,aliceposta.it,outbound,ZZ,Unknown Region,0 +aliexpress.com,alibaba.com,inbound,ZZ,Unknown Region,0 +allegro.pl,allegro.pl,inbound,ZZ,Unknown Region,0 +allegrogroup.ua,allegrogroup.ua,inbound,ZZ,Unknown Region,0 +allsaints.com,allsaints.com,inbound,ZZ,Unknown Region,0 +ama-assn.org,elabs10.com,inbound,ZZ,Unknown Region,0 +amadeus.com,amadeus.net,inbound,ZZ,Unknown Region,0 +amazon.{...},amazon.{...},inbound,ZZ,Unknown Region,0.021685 +amazon.{...},amazonses.com,inbound,ZZ,Unknown Region,0.999971 +amazonses.com,amazonses.com,inbound,ZZ,Unknown Region,0.997919 +amazonses.com,postini.com,inbound,ZZ,Unknown Region,0.924067 +amctheatres.com,amctheatres.com,inbound,ZZ,Unknown Region,0 +americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,ZZ,Unknown Region,0 +ancestry.com,ancestry.com,inbound,ZZ,Unknown Region,0 +angieslist.com,angieslist.com,inbound,ZZ,Unknown Region,0 +anntaylor.com,anntaylor.com,inbound,ZZ,Unknown Region,0 +anpdm.com,anpdm.com,inbound,ZZ,Unknown Region,4e-06 +aol.com,aol.com,outbound,ZZ,Unknown Region,1 +apple.com,apple.com,inbound,ZZ,Unknown Region,0.915839 +argos.co.uk,argos.co.uk,inbound,ZZ,Unknown Region,1 +argos.co.uk,exacttarget.com,inbound,ZZ,Unknown Region,1 +artists-hub.com,artists-hub.com,inbound,ZZ,Unknown Region,0 +asda.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +ask.fm,ask.fm,inbound,ZZ,Unknown Region,1e-05 +askmen.com,askmen.com,inbound,ZZ,Unknown Region,0 +astrocenter.com,center.com,inbound,ZZ,Unknown Region,0 +athleta.com,athleta.com,inbound,ZZ,Unknown Region,0 +atlassian.net,uc-inf.net,inbound,ZZ,Unknown Region,1 +att.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999956 +auctionzip-email.com,email-auctionholdings.com,inbound,ZZ,Unknown Region,0 +auinmeio.com.br,fnac.com.br,inbound,ZZ,Unknown Region,0 +authorize.net,authorize.net,inbound,ZZ,Unknown Region,0 +authorize.net,visa.com,inbound,ZZ,Unknown Region,0.993555 +autoreply.com,autoreply.com,inbound,ZZ,Unknown Region,0 +avomail.com,avomail.com,inbound,ZZ,Unknown Region,0 +avon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 +avon.com,postdirect.com,inbound,ZZ,Unknown Region,0 +aweber.com,aweber.com,inbound,ZZ,Unknown Region,3e-06 +ayi.com,ayi.com,inbound,ZZ,Unknown Region,0 +backcountry.com,backcountry.com,inbound,ZZ,Unknown Region,0 +backlog.jp,backlog.jp,inbound,ZZ,Unknown Region,0 +bagitgetitmailer.in,emce2.in,inbound,ZZ,Unknown Region,0 +banamex.com,citi.com,inbound,ZZ,Unknown Region,0.999958 +bananarepublic.com,bananarepublic.com,inbound,ZZ,Unknown Region,3.48869418874254e-07 +bancochile.cl,bancochile.cl,inbound,ZZ,Unknown Region,0.999504 +bancofalabella.com,bancofalabella.com,inbound,ZZ,Unknown Region,0 +banesco.com,banesco.com,inbound,ZZ,Unknown Region,0 +bankofamerica.com,bankofamerica.com,inbound,ZZ,Unknown Region,0.971142 +banorte.com,gfnorte.com.mx,inbound,ZZ,Unknown Region,0.994999 +barclaycardus.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +basecamp.com,basecamp.com,inbound,ZZ,Unknown Region,1 +basecamphq.com,basecamphq.com,inbound,ZZ,Unknown Region,1 +baskinrobbins.com,baskinrobbins.com,inbound,ZZ,Unknown Region,0 +bazarchic-invitations.com,bazarchic-emstech.com,inbound,ZZ,Unknown Region,0 +bcbg.com,bcbg.com,inbound,ZZ,Unknown Region,0 +beatport-email.com,beatport-email.com,inbound,ZZ,Unknown Region,0 +beautylish.com,beautylish.com,inbound,ZZ,Unknown Region,1 +bebe.com,ed10.com,inbound,ZZ,Unknown Region,0 +befrugal.com,befrugal.com,inbound,ZZ,Unknown Region,0 +belkemail.com,belkemail.com,inbound,ZZ,Unknown Region,0 +bellsouth.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999951 +benihana-news.com,benihana-news.com,inbound,ZZ,Unknown Region,0 +bestbuy.com,bestbuy.com,inbound,ZZ,Unknown Region,0.003289 +beta.lt,mailersend3.com,inbound,ZZ,Unknown Region,0 +beyondtherack.com,beyondtherack.com,inbound,ZZ,Unknown Region,0 +bigfishgames.com,bigfishgames.com,inbound,ZZ,Unknown Region,0 +biglots.com,biglots.com,inbound,ZZ,Unknown Region,0 +bjsrestaurants.com,bjsrestaurants.com,inbound,ZZ,Unknown Region,0 +blackberry.com,blackberry.com,inbound,ZZ,Unknown Region,0 +blackboard.com,notification.com,inbound,ZZ,Unknown Region,0 +blackpeoplemeet.com,blackpeoplemeet.com,inbound,ZZ,Unknown Region,0 +bloglovin.com,bloglovin.com,inbound,ZZ,Unknown Region,0 +blogtrottr.com,blogtrottr.com,inbound,ZZ,Unknown Region,0 +bloomberg.com,bloomberg.com,inbound,ZZ,Unknown Region,0.001463 +bloomberg.net,bloomberg.net,inbound,ZZ,Unknown Region,1 +bluediamondhost3.com,web-hosting.com,inbound,ZZ,Unknown Region,1 +bluehornet.com,bluehornet.com,inbound,ZZ,Unknown Region,0 +bluenile.com,bluenile.com,inbound,ZZ,Unknown Region,0 +bluestatedigital.com,bluestatedigital.com,inbound,ZZ,Unknown Region,0 +bm05.net,bm05.net,inbound,ZZ,Unknown Region,0 +bmnt.jp,bmnt.jp,inbound,ZZ,Unknown Region,0 +bmsend.com,bmsend.com,inbound,ZZ,Unknown Region,0 +bn.com,bn.com,inbound,ZZ,Unknown Region,0 +bncollegemail.com,bncollegemail.com,inbound,ZZ,Unknown Region,0 +booking.com,booking.com,inbound,ZZ,Unknown Region,1 +bookmyshow.com,eccluster.com,inbound,ZZ,Unknown Region,0 +boots.com,boots.com,inbound,ZZ,Unknown Region,0 +box.com,box.com,inbound,ZZ,Unknown Region,0.955607 +brierleycrm.com,brierleycrm.com,inbound,ZZ,Unknown Region,0 +brooksbrothers.com,brooksbrothers.com,inbound,ZZ,Unknown Region,0 +btinternet.com,cpcloud.co.uk,inbound,ZZ,Unknown Region,0 +btinternet.com,cpcloud.co.uk,outbound,ZZ,Unknown Region,0 +btinternet.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +budgettravel.com,email-budgettravel.com,inbound,ZZ,Unknown Region,0 +buyinvite.com.au,buyinvite.com.au,inbound,ZZ,Unknown Region,0 +bv.com.br,bv.com.br,inbound,ZZ,Unknown Region,0 +byway.it,byway.it,inbound,ZZ,Unknown Region,0 +cabelas.com,cabelas.com,inbound,ZZ,Unknown Region,0 +californiajobdepartment.com,californiajobdepartment.com,inbound,ZZ,Unknown Region,0 +callcommand.com,callcommand.com,inbound,ZZ,Unknown Region,0 +calottery.com,calottery.com,inbound,ZZ,Unknown Region,1 +canadiantire.ca,canadiantire.ca,inbound,ZZ,Unknown Region,0 +canalplus.es,canalplus.es,inbound,ZZ,Unknown Region,0 +capitalone.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +capitaloneemail.com,capitaloneemail.com,inbound,ZZ,Unknown Region,0 +career-hub.net,career-hub.net,inbound,ZZ,Unknown Region,1 +careerbuilder-email.com,careerbuilder-email.com,inbound,ZZ,Unknown Region,0 +careerbuilder.com,careerbuilder.com,inbound,ZZ,Unknown Region,0 +careerflash.net,careerflash.net,inbound,ZZ,Unknown Region,1 +carrefour.fr,carrefour.fr,inbound,ZZ,Unknown Region,0 +carters.com,carters.com,inbound,ZZ,Unknown Region,2e-06 +caseyresearch.com,caseyresearch.com,inbound,ZZ,Unknown Region,1 +catchyfreebies.net,mmsend53.com,inbound,ZZ,Unknown Region,0 +cccampaigns.net,emv9.net,inbound,ZZ,Unknown Region,0 +cecentertainment.com,cecentertainment.com,inbound,ZZ,Unknown Region,0 +celebritycruises.com,celebritycruises.com,inbound,ZZ,Unknown Region,0 +centaur.co.uk,centaur.co.uk,inbound,ZZ,Unknown Region,0 +centauro.com.br,centauro.com.br,inbound,ZZ,Unknown Region,0 +centerparcs.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +cfmailer.com,elabs11.com,inbound,ZZ,Unknown Region,0 +change.org,change.org,inbound,ZZ,Unknown Region,1 +channel4.com,channel4.com,inbound,ZZ,Unknown Region,0 +chase.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +cheaperthandirt.com,cheaperthandirt.com,inbound,ZZ,Unknown Region,0 +chefscatalog.com,chefscatalog.com,inbound,ZZ,Unknown Region,0 +chemistdirect.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +chemistry.com,chemistry.com,inbound,ZZ,Unknown Region,0 +chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,ZZ,Unknown Region,0 +chopra.com,chopra.com,inbound,ZZ,Unknown Region,1 +christianmingle.com,postdirect.com,inbound,ZZ,Unknown Region,0 +cir.ca,cir.ca,inbound,ZZ,Unknown Region,1 +citi.com,citi.com,inbound,ZZ,Unknown Region,0.999941 +citibank.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +citibank.com,citi.com,inbound,ZZ,Unknown Region,0.999997 +citicorp.com,citi.com,inbound,ZZ,Unknown Region,0.999999 +citruslane.com,citruslane.com,inbound,ZZ,Unknown Region,5e-06 +clarisonic.com,clarisonic.com,inbound,ZZ,Unknown Region,0 +classmates.com,classmates.com,inbound,ZZ,Unknown Region,0 +clickon.com.ar,clickon.com.ar,inbound,ZZ,Unknown Region,0 +cmail1.com,createsend.com,inbound,ZZ,Unknown Region,0 +cmail2.com,createsend.com,inbound,ZZ,Unknown Region,0 +cmrfalabella.com,cmrfalabella.com,inbound,ZZ,Unknown Region,0 +codebreak.info,codebreak.info,inbound,ZZ,Unknown Region,1 +comcast.net,comcast.net,outbound,ZZ,Unknown Region,1 +confirmsignup.com,mmsend53.com,inbound,ZZ,Unknown Region,0 +constantcontact.com,postini.com,inbound,ZZ,Unknown Region,0.139551 +constantcontact.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999541 +contact-darty.com,mm-send.com,inbound,ZZ,Unknown Region,0 +contactlab.it,contactlab.it,inbound,ZZ,Unknown Region,0 +converse.com,converse.com,inbound,ZZ,Unknown Region,1 +cookingchanneltv.com,cookingchanneltv.com,inbound,ZZ,Unknown Region,0 +copernica.nl,picsrv.net,inbound,ZZ,Unknown Region,0.011507 +copernica.nl,vicinity.nl,inbound,ZZ,Unknown Region,0.011753 +coppel.com,coppel.com,inbound,ZZ,Unknown Region,0 +corporateperks.com,nextjump.com,inbound,ZZ,Unknown Region,0 +countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,ZZ,Unknown Region,0 +coupondunia.in,coupondunia.in,inbound,ZZ,Unknown Region,1 +crabtree-evelyn.com,crabtree-evelyn.com,inbound,ZZ,Unknown Region,0 +crainnewsalerts.com,crainnewsalerts.com,inbound,ZZ,Unknown Region,0 +crashlytics.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +cricut.com,elabs12.com,inbound,ZZ,Unknown Region,0 +criticalimpactinc.com,criticalimpactinc.com,inbound,ZZ,Unknown Region,0 +critsend.com,critsend.com,inbound,ZZ,Unknown Region,0 +crocs-email.com,crocs-email.com,inbound,ZZ,Unknown Region,0 +crunchyroll.com,crunchyroll.com,inbound,ZZ,Unknown Region,0 +cudo.com.au,exacttarget.com,inbound,ZZ,Unknown Region,0 +cuenote.jp,cuenote.jp,inbound,ZZ,Unknown Region,0 +cupomturbinado.com.br,cupomnaweb.com.br,inbound,ZZ,Unknown Region,1 +cuppon.pl,cuppon.pl,inbound,ZZ,Unknown Region,0 +curriculum.com.br,curriculum.com.br,inbound,ZZ,Unknown Region,0 +currys.co.uk,currys.co.uk,inbound,ZZ,Unknown Region,0 +custom-emailing.com,elabs12.com,inbound,ZZ,Unknown Region,0 +cvent-planner.com,cvent-planner.com,inbound,ZZ,Unknown Region,0 +cyberdiet.com.br,allinmedia.com.br,inbound,ZZ,Unknown Region,0 +dabmail.com,iaires.com,inbound,ZZ,Unknown Region,0 +dailyom.com,dailyom.com,inbound,ZZ,Unknown Region,1 +dairyqueen.com,dairyqueen.com,inbound,ZZ,Unknown Region,0 +datadrivenemail.com,datadrivenemail.com,inbound,ZZ,Unknown Region,0 +daveramsey.com,daveramsey.com,inbound,ZZ,Unknown Region,0 +ddc-emails.com,ddc-emails.com,inbound,ZZ,Unknown Region,0 +dealchicken.com,dealchicken.com,inbound,ZZ,Unknown Region,0 +dealchicken.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +dealfind.com,dealfind.com,inbound,ZZ,Unknown Region,0 +dealnews.com,dealnews.com,inbound,ZZ,Unknown Region,0 +dealsdirect.com.au,dealsdirect.com.au,inbound,ZZ,Unknown Region,0 +dealspl.us,dealspl.us,inbound,ZZ,Unknown Region,0 +delta.com,delta.com,inbound,ZZ,Unknown Region,0.040177 +dermstore.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +dhl.com,dhl.com,inbound,ZZ,Unknown Region,0.994107 +dietaesaude.com.br,dietaesaude.com.br,inbound,ZZ,Unknown Region,0 +dinda.com.br,dinda.com.br,inbound,ZZ,Unknown Region,0 +directvla.com,directvla.com,inbound,ZZ,Unknown Region,0 +discover.com,discover.com,inbound,ZZ,Unknown Region,0 +disneydestinations.com,disneyparks.com,inbound,ZZ,Unknown Region,0 +disneydestinations.com,disneyworld.com,inbound,ZZ,Unknown Region,0 +diynetwork.com,diynetwork.com,inbound,ZZ,Unknown Region,0 +doctoroz.com,email-sharecare2.com,inbound,ZZ,Unknown Region,0 +dollartree.com,email-dollartree.com,inbound,ZZ,Unknown Region,0 +dominos.com,dominos.com,inbound,ZZ,Unknown Region,0.011861 +dominos.com.au,dominos.com.au,inbound,ZZ,Unknown Region,0 +donuts.ne.jp,dnuts.jp,inbound,ZZ,Unknown Region,0 +dotz.com.br,dotz.com.br,inbound,ZZ,Unknown Region,0 +doubletakeoffers.com,doubletakeoffers.com,inbound,ZZ,Unknown Region,0 +downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 +dptagent.net,dptagent.net,inbound,ZZ,Unknown Region,0 +draftkings.com,draftkings.com,inbound,ZZ,Unknown Region,0 +dreamhost.com,dreamhost.com,inbound,ZZ,Unknown Region,0 +driftem.com,emce2.in,inbound,ZZ,Unknown Region,0 +drjays-mail.com,drjays-mail.com,inbound,ZZ,Unknown Region,0 +dromadaire-news.com,ecmcluster.com,inbound,ZZ,Unknown Region,0 +dukecareers.com,dukecareers.com,inbound,ZZ,Unknown Region,1 +duluthtradingemail.com,email-duluthtrading.com,inbound,ZZ,Unknown Region,0 +dynect-mailer.net,dynect.net,inbound,ZZ,Unknown Region,0 +dynect-mailer.net,sendlabs.com,inbound,ZZ,Unknown Region,0 +e-bodyc.com,email-bodycentral.com,inbound,ZZ,Unknown Region,0 +e-jobs-ville.com,e-jobs-ville.com,inbound,ZZ,Unknown Region,1 +e-leclerc.com,e-leclerc.com,inbound,ZZ,Unknown Region,0 +easycanvasprints.com,easycanvasprints.com,inbound,ZZ,Unknown Region,0 +easyhits4u.com,easyhits4u.com,inbound,ZZ,Unknown Region,1 +easyhits4u.com,relmax.net,inbound,ZZ,Unknown Region,0 +ebags.com,ebags.com,inbound,ZZ,Unknown Region,0 +ebay-kleinanzeigen.de,mobile.de,inbound,ZZ,Unknown Region,1 +ebay.{...},ebay.{...},inbound,ZZ,Unknown Region,0.999648 +ebay.{...},emarsys.net,inbound,ZZ,Unknown Region,0 +ebay.{...},postdirect.com,inbound,ZZ,Unknown Region,0 +ebizac3.com,ebizac3.com,inbound,ZZ,Unknown Region,0 +ebuildabear.com,ebuildabear.com,inbound,ZZ,Unknown Region,8e-06 +ed10.net,ed10.com,inbound,ZZ,Unknown Region,0 +eddiebauer.com,eddiebauer.com,inbound,ZZ,Unknown Region,0 +eduk.com,eduk.com,inbound,ZZ,Unknown Region,0 +efamilydollar.com,efamilydollar.com,inbound,ZZ,Unknown Region,0 +elabs10.com,elabs10.com,inbound,ZZ,Unknown Region,0 +elabs12.com,elabs12.com,inbound,ZZ,Unknown Region,0 +elistas.net,elistas.net,inbound,ZZ,Unknown Region,0 +elkjop.no,ec-cluster.com,inbound,ZZ,Unknown Region,0 +elkjop.no,eccluster.com,inbound,ZZ,Unknown Region,0 +elo7.com.br,elo7.com.br,inbound,ZZ,Unknown Region,0 +email-1800contacts.com,email-1800contacts.com,inbound,ZZ,Unknown Region,0 +email-aaa.com,email-aaa.com,inbound,ZZ,Unknown Region,0 +email-aeriagames.com,email-aeriagames.com,inbound,ZZ,Unknown Region,0 +email-dressbarn.com,email-dressbarn.com,inbound,ZZ,Unknown Region,0 +email-firestone.com,reminder-firestone.com,inbound,ZZ,Unknown Region,0 +email-honest.com,email-honest.com,inbound,ZZ,Unknown Region,0 +email-od.com,email-od.com,inbound,ZZ,Unknown Region,0.999752 +email-petsmart.com,email-petsmart.com,inbound,ZZ,Unknown Region,0 +email-sportchalet.com,email-sportchalet.com,inbound,ZZ,Unknown Region,0 +email-telekom.de,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +email-totalwine.com,email-totalwine.com,inbound,ZZ,Unknown Region,0 +email-wildstar-online.com,email-carbine.com,inbound,ZZ,Unknown Region,0 +email2-beyond.com,messagebus.com,inbound,ZZ,Unknown Region,0 +email360api.com,email360api.com,inbound,ZZ,Unknown Region,0 +email365inc.com,email365inc.com,inbound,ZZ,Unknown Region,0 +email3m.com,email3m.com,inbound,ZZ,Unknown Region,0 +emailrestaurant.com,emailrestaurant.com,inbound,ZZ,Unknown Region,0 +emailtoryburch.com,emailtoryburch.com,inbound,ZZ,Unknown Region,0 +emarsys.net,emarsys.net,inbound,ZZ,Unknown Region,0.265197 +embarqmail.com,centurylink.net,inbound,ZZ,Unknown Region,0.999918 +emergencyemail.org,emergencyemail.org,inbound,ZZ,Unknown Region,0 +eminentinc.com,eminentinc.com,inbound,ZZ,Unknown Region,0 +emktviajarbarato.com.br,splio.com.br,inbound,ZZ,Unknown Region,0.875902 +employboard.com,employboard.com,inbound,ZZ,Unknown Region,1 +emsecure.net,emsecure.net,inbound,ZZ,Unknown Region,0 +emsmtp.com,emsmtp.com,inbound,ZZ,Unknown Region,0.44284 +en25.com,kqed.org,inbound,ZZ,Unknown Region,0 +enewscartes.net,bp06.net,inbound,ZZ,Unknown Region,0 +enewsletter.pl,mydeal.pl,inbound,ZZ,Unknown Region,0 +enewsletter.pl,sare25.com,inbound,ZZ,Unknown Region,0 +enplenitud.com,enplenitud.com,inbound,ZZ,Unknown Region,0 +entrepreneur.com,entrepreneur.com,inbound,ZZ,Unknown Region,0 +ethingsremembered.com,ethingsremembered.com,inbound,ZZ,Unknown Region,0 +etrade.com,etrade.com,inbound,ZZ,Unknown Region,0 +etransmail.com,etransmail.com,inbound,ZZ,Unknown Region,0 +etransmail.com,ptransmail.com,inbound,ZZ,Unknown Region,0 +etrmailbox.com,etrmailbox.com,inbound,ZZ,Unknown Region,0 +etsy.com,etsy.com,inbound,ZZ,Unknown Region,0.020849 +evernote.com,evernote.com,inbound,ZZ,Unknown Region,1 +everydayfamily.com,everydayfamily.com,inbound,ZZ,Unknown Region,0 +everytown.org,everytown.org,inbound,ZZ,Unknown Region,1 +exacttarget.com,bazaarvoice.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,booksamillion.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,exacttarget.com,inbound,ZZ,Unknown Region,0.000325 +exacttarget.com,msg.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,redboxinstant.com,inbound,ZZ,Unknown Region,0 +exacttarget.com,skylinetechnologies.com,inbound,ZZ,Unknown Region,0 +expediamail.com,airasiago.com,inbound,ZZ,Unknown Region,1 +expediamail.com,exacttarget.com,inbound,ZZ,Unknown Region,1 +expediamail.com,expediamail.com,inbound,ZZ,Unknown Region,0.839662 +expediamail.com,quotitmail.com,inbound,ZZ,Unknown Region,0 +experteer.com,experteer.com,inbound,ZZ,Unknown Region,0 +express.com,expressfashion.com,inbound,ZZ,Unknown Region,0 +facebook.com,facebook.com,inbound,ZZ,Unknown Region,0 +facebook.com,facebook.com,outbound,ZZ,Unknown Region,1 +facebookmail.com,postini.com,inbound,ZZ,Unknown Region,0.918705 +facebookmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999846 +falabella.com,falabella.com,inbound,ZZ,Unknown Region,0 +fanatics.com,fanatics.com,inbound,ZZ,Unknown Region,0 +fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,ZZ,Unknown Region,0 +fanbridge.com,fanbridge.com,inbound,ZZ,Unknown Region,0 +fanofannas.com,fanofannas.com,inbound,ZZ,Unknown Region,0 +fansedge.com,fansedge.com,inbound,ZZ,Unknown Region,0 +farmers.com,farmers.com,inbound,ZZ,Unknown Region,0.999984 +fastcompany.com,fastcompany.com,inbound,ZZ,Unknown Region,0 +fedex.com,fedex.com,inbound,ZZ,Unknown Region,0.919932 +feld-ent.com,postdirect.com,inbound,ZZ,Unknown Region,0 +fellowshiponemail.com,fellowshiponemail.com,inbound,ZZ,Unknown Region,0 +fetlifemail.com,fetlifemail.com,inbound,ZZ,Unknown Region,0 +fidelity.com,fidelity.com,inbound,ZZ,Unknown Region,1 +financialfreedommail.com,financialfreedommail.com,inbound,ZZ,Unknown Region,0 +fingerhut.com,fingerhut.com,inbound,ZZ,Unknown Region,0 +flets.com,flets.com,inbound,ZZ,Unknown Region,0 +flirtlocal.com,flirtlocal.com,inbound,ZZ,Unknown Region,1 +flmsecure.com,fling.com,inbound,ZZ,Unknown Region,0 +flmsecure.com,flmsecure.com,inbound,ZZ,Unknown Region,0 +floridajobdepartment.com,floridajobdepartment.com,inbound,ZZ,Unknown Region,0 +fnac.com,fnac.com,inbound,ZZ,Unknown Region,0.025375 +fnb.co.za,fnb.co.za,inbound,ZZ,Unknown Region,0.325259 +foodnetwork.com,foodnetwork.com,inbound,ZZ,Unknown Region,0 +forcemail.in,iaires.com,inbound,ZZ,Unknown Region,0 +foreseegame.com,iaires.com,inbound,ZZ,Unknown Region,0 +fotffamily.com,fotffamily.com,inbound,ZZ,Unknown Region,0 +fotolivro.com.br,fotolivro.com.br,inbound,ZZ,Unknown Region,0 +fpmailerbr.com,fpmailerbr.com,inbound,ZZ,Unknown Region,0 +freecycle.org,freecycle.org,inbound,ZZ,Unknown Region,1 +freelancer.com,getafreelancer.com,inbound,ZZ,Unknown Region,0 +freesafelistmailer.com,waters-advertising.com,inbound,ZZ,Unknown Region,0 +freshmail.pl,freshmail.pl,inbound,ZZ,Unknown Region,0 +fridays.com,fridays.com,inbound,ZZ,Unknown Region,0 +frk.com,frk.com,inbound,ZZ,Unknown Region,1 +frontdoor.com,frontdoor.com,inbound,ZZ,Unknown Region,0 +frontgate-email.com,frontgate-email.com,inbound,ZZ,Unknown Region,0 +fuckbooknet.net,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +fundplaza.co.in,arrowsignindia.com,inbound,ZZ,Unknown Region,0 +fundplaza.in,fundplaza.in,inbound,ZZ,Unknown Region,0 +funonthenet.in,funonthenet.in,inbound,ZZ,Unknown Region,1 +futurmailer.pt,futurmailer.pt,inbound,ZZ,Unknown Region,0 +gabbar.info,gabbar.info,inbound,ZZ,Unknown Region,1 +gamefly.com,gamefly.com,inbound,ZZ,Unknown Region,0.014317 +gamingmails.com,gamingmails.com,inbound,ZZ,Unknown Region,0 +gap.com,gap.com,inbound,ZZ,Unknown Region,0 +gap.eu,gap.eu,inbound,ZZ,Unknown Region,0 +gapcanada.ca,gapcanada.ca,inbound,ZZ,Unknown Region,0 +garanti.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 +garnethill-email.com,garnethill-email.com,inbound,ZZ,Unknown Region,0 +gaylordalert.com,gaylordalert.com,inbound,ZZ,Unknown Region,0 +gcast.com.au,systemsserver.net,inbound,ZZ,Unknown Region,0 +gdtsuccess.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 +geico.com,geico.com,inbound,ZZ,Unknown Region,0.096795 +gene.com,roche.com,inbound,ZZ,Unknown Region,1 +geocaching.com,groundspeak.com,inbound,ZZ,Unknown Region,1 +getinbox.net,getinbox.net,inbound,ZZ,Unknown Region,0 +getkeepsafe.com,getkeepsafe.com,inbound,ZZ,Unknown Region,1 +getmein.com,getmein.com,inbound,ZZ,Unknown Region,0 +getresponse.com,getresponse.com,inbound,ZZ,Unknown Region,0 +gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,ZZ,Unknown Region,0 +gilt.com,gilt.com,inbound,ZZ,Unknown Region,1e-06 +github.com,github.net,inbound,ZZ,Unknown Region,1 +glassdoor.com,glassdoor.com,inbound,ZZ,Unknown Region,0 +globalmembersupport.com,globalmembersupport.com,inbound,ZZ,Unknown Region,0 +globalspec.com,globalspec.com,inbound,ZZ,Unknown Region,0 +gmail.com,blackberry.com,inbound,ZZ,Unknown Region,0.998326 +gmail.com,postini.com,inbound,ZZ,Unknown Region,0.891175 +gmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.998661 +go.com,starwave.com,inbound,ZZ,Unknown Region,0.006276 +godtubemail.com,godtubemail.com,inbound,ZZ,Unknown Region,0 +godvinemail.com,godvinemail.com,inbound,ZZ,Unknown Region,0 +gohappy.com.tw,gohappy.com.tw,inbound,ZZ,Unknown Region,0 +goldenbrands.gr,goldenbrands.gr,inbound,ZZ,Unknown Region,1 +golfnow.com,email-golfnow.com,inbound,ZZ,Unknown Region,0 +google.com,postini.com,inbound,ZZ,Unknown Region,0.81059 +gop.com,gop.com,inbound,ZZ,Unknown Region,0 +grabone-mail-ie.com,grabone-mail-ie.com,inbound,ZZ,Unknown Region,0 +grabone-mail.com,grabone-mail.com,inbound,ZZ,Unknown Region,0 +gratka.pl,gratka.pl,inbound,ZZ,Unknown Region,0 +grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,ZZ,Unknown Region,0 +groopdealz.com,groopdealz.com,inbound,ZZ,Unknown Region,1 +groupalia.es,groupalia.es,inbound,ZZ,Unknown Region,0 +groupalia.it,groupalia.it,inbound,ZZ,Unknown Region,0 +groupon.jp,data-hotel.net,inbound,ZZ,Unknown Region,0 +groupon.{...},groupon.{...},inbound,ZZ,Unknown Region,0.990176 +grouponmail.{...},grouponmail.{...},inbound,ZZ,Unknown Region,0 +grubhubmail.com,grubhubmail.com,inbound,ZZ,Unknown Region,0 +grupanya.com,euromsg.net,inbound,ZZ,Unknown Region,0 +guruin.info,guru.net.in,inbound,ZZ,Unknown Region,1 +habitaclia.com,splio.es,inbound,ZZ,Unknown Region,0.951406 +harborfreightemail.com,harborfreightemail.com,inbound,ZZ,Unknown Region,0 +hautelook.com,hautelook.com,inbound,ZZ,Unknown Region,0 +hepsiburada.com,euromsg.net,inbound,ZZ,Unknown Region,0 +herbalifemail.com,herbalifemail.com,inbound,ZZ,Unknown Region,0 +herculist.com,herculist.com,inbound,ZZ,Unknown Region,0 +hgtv.com,hgtv.com,inbound,ZZ,Unknown Region,0 +hhgreggemail.com,hhgreggemail.com,inbound,ZZ,Unknown Region,0 +hipmunk.com,hipmunk.com,inbound,ZZ,Unknown Region,0 +hln.be,persgroep-ops.net,inbound,ZZ,Unknown Region,0 +hm-f.jp,hm-f.jp,inbound,ZZ,Unknown Region,0 +hobsonsmail.com,hobsonsmail.com,inbound,ZZ,Unknown Region,0 +homebaselife.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +homechoice.co.za,homechoice.co.za,inbound,ZZ,Unknown Region,0 +homedepotemail.com,homedepotemail.com,inbound,ZZ,Unknown Region,0 +honto.jp,honto.jp,inbound,ZZ,Unknown Region,0 +horoscope.com,center.com,inbound,ZZ,Unknown Region,0 +hotels.com,hotels.com,inbound,ZZ,Unknown Region,0 +hotelurbano.com.br,allin.com.br,inbound,ZZ,Unknown Region,0 +hotmail.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999944 +hotmail.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 +hotspotmailer.com,hotspotmailer.com,inbound,ZZ,Unknown Region,1 +hp.com,hp.com,inbound,ZZ,Unknown Region,0.198696 +hsn.com,hsn.com,inbound,ZZ,Unknown Region,0 +htcampusmailer.com,eccluster.com,inbound,ZZ,Unknown Region,0 +hubspot.com,hubspot.com,inbound,ZZ,Unknown Region,1 +huinforma.com.br,huinforma.com.br,inbound,ZZ,Unknown Region,0 +hulumail.com,hulumail.com,inbound,ZZ,Unknown Region,0 +hungryhouse.co.uk,mxmfb.com,inbound,ZZ,Unknown Region,0 +huntington.com,huntington.com,inbound,ZZ,Unknown Region,0.986937 +i-say.com,ipsos-interactive.com,inbound,ZZ,Unknown Region,1 +icloud.com,apple.com,inbound,ZZ,Unknown Region,1 +icloud.com,icloud.com,outbound,ZZ,Unknown Region,1 +icloud.com,mac.com,inbound,ZZ,Unknown Region,1 +icloud.com,me.com,inbound,ZZ,Unknown Region,0.999995 +ifttt.com,ifttt.com,inbound,ZZ,Unknown Region,1 +ign.com,ign.com,inbound,ZZ,Unknown Region,0 +ignitionsender.com,ignitionsender.com,inbound,ZZ,Unknown Region,0 +iheart.com,iheart.com,inbound,ZZ,Unknown Region,0 +immobilienscout24.de,immobilienscout24.de,inbound,ZZ,Unknown Region,1 +in-boxpays.com,in-boxpays.com,inbound,ZZ,Unknown Region,0 +indiamart.com,indiamart.com,inbound,ZZ,Unknown Region,1 +indiatimes.com,speakingtree.in,inbound,ZZ,Unknown Region,0 +indiatimeshop.com,sendpal.in,inbound,ZZ,Unknown Region,0 +indieroyale.com,desura.com,inbound,ZZ,Unknown Region,1 +infibeam.com,eccluster.com,inbound,ZZ,Unknown Region,0 +infopanel.jp,mailds.jp,inbound,ZZ,Unknown Region,0 +infos-micromania.com,infos-micromania.com,inbound,ZZ,Unknown Region,0 +infosephora.com,splio.com,inbound,ZZ,Unknown Region,0.962167 +infoworld.com,infoworld.com,inbound,ZZ,Unknown Region,0 +infusionmail.com,infusionmail.com,inbound,ZZ,Unknown Region,0 +inmotionhosting.com,inmotionhosting.com,inbound,ZZ,Unknown Region,1 +ino.com,ino.com,inbound,ZZ,Unknown Region,0.999981 +interwell.gr,interwell.gr,inbound,ZZ,Unknown Region,0 +ipcmedia.co.uk,ipcmedia.co.uk,inbound,ZZ,Unknown Region,0 +itunes.com,apple.com,inbound,ZZ,Unknown Region,0.043012 +jackwills.com,jackwills.com,inbound,ZZ,Unknown Region,0 +jalag.de,jalag.de,inbound,ZZ,Unknown Region,1 +jane.com,jane.com,inbound,ZZ,Unknown Region,1 +jango.com,jango.com,inbound,ZZ,Unknown Region,0 +jared.com,jared.com,inbound,ZZ,Unknown Region,0 +jdate.com,postdirect.com,inbound,ZZ,Unknown Region,0 +jetprivilege.com,jetprivilege.com,inbound,ZZ,Unknown Region,0 +jeuxvideo.com,jeuxvideo.com,inbound,ZZ,Unknown Region,0.148921 +jeweloscoemail.com,email-mywebgrocer2.com,inbound,ZZ,Unknown Region,0 +joann-mail.com,joann-mail.com,inbound,ZZ,Unknown Region,0 +jobinsider.com,jobinsider.com,inbound,ZZ,Unknown Region,0 +jobisjob.com,jobisjob.com,inbound,ZZ,Unknown Region,0 +jobomas.com,jobomas.com,inbound,ZZ,Unknown Region,1 +jobstreet.com,jobstreet.com,inbound,ZZ,Unknown Region,0 +jpcycles.com,jpcycles.com,inbound,ZZ,Unknown Region,0.001716 +jungleerummy.com,jungleerummy.com,inbound,ZZ,Unknown Region,1 +jusbrasil.com.br,jusbrasil.com.br,inbound,ZZ,Unknown Region,0 +just-eat.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +justclick.ru,justclick.ru,inbound,ZZ,Unknown Region,0 +justdial.com,iaires.com,inbound,ZZ,Unknown Region,0 +k1speed.com,k1speed.com,inbound,ZZ,Unknown Region,1 +kaskusnetworks.com,kaskus.com,inbound,ZZ,Unknown Region,0 +kay.com,kay.com,inbound,ZZ,Unknown Region,0 +keek.com,keek.com,inbound,ZZ,Unknown Region,1 +kgbdeals.co.uk,email1-kgbdeals.com,inbound,ZZ,Unknown Region,0 +kliksa.net,euromsg.net,inbound,ZZ,Unknown Region,0 +kliktoday.com,kliktoday.com,inbound,ZZ,Unknown Region,0 +klm-mail.com,klm-mail.com,inbound,ZZ,Unknown Region,0 +kohls.com,kohls.com,inbound,ZZ,Unknown Region,0 +kongregate.com,kongregate.com,inbound,ZZ,Unknown Region,0 +krs.bz,tricorn.net,inbound,ZZ,Unknown Region,0 +la-meteo-mail.fr,splio.com,inbound,ZZ,Unknown Region,1 +laaptuemail.com,laaptuemail.com,inbound,ZZ,Unknown Region,0 +lakewoodchurch.com,lakewoodchurch.com,inbound,ZZ,Unknown Region,0 +landsend.com,email-landsend.com,inbound,ZZ,Unknown Region,0 +landsend.com,postdirect.com,inbound,ZZ,Unknown Region,0 +laptuinvite.com,laptuinvite.com,inbound,ZZ,Unknown Region,0 +lastminute.com,lastminute.com,inbound,ZZ,Unknown Region,0 +lazerhits.com,lazerhits.com,inbound,ZZ,Unknown Region,1 +leadercontato.com.br,leadercontato.com.br,inbound,ZZ,Unknown Region,0 +lefigaro.fr,splio.com,inbound,ZZ,Unknown Region,1 +lemonde.fr,lemonde.fr,inbound,ZZ,Unknown Region,0 +life360.com,life360.com,inbound,ZZ,Unknown Region,1 +lifecare-news.com,email-lifecare.com,inbound,ZZ,Unknown Region,0 +lifemiles.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +lifescript.com,ilinkmd.com,inbound,ZZ,Unknown Region,0 +line6.com,line6.com,inbound,ZZ,Unknown Region,0 +linkedin.com,linkedin.com,inbound,ZZ,Unknown Region,0.995201 +linkedin.com,postini.com,inbound,ZZ,Unknown Region,0.940251 +liquidation.com,liquidation.com,inbound,ZZ,Unknown Region,0 +listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,ZZ,Unknown Region,0.00023 +live.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999921 +live.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 +livejournal.com,livejournal.com,inbound,ZZ,Unknown Region,0 +livemailservice.com,livemailservice.com,inbound,ZZ,Unknown Region,0 +livenation.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +lmlmgv.com.br,gvarev.com.br,inbound,ZZ,Unknown Region,0 +loccitane.com,neolane.net,inbound,ZZ,Unknown Region,0 +loft.com,anntaylor.com,inbound,ZZ,Unknown Region,0 +lombardipublishing.com,lombardipublishing.com,inbound,ZZ,Unknown Region,0 +lookout.com,lookout.com,inbound,ZZ,Unknown Region,1 +lsi.com,postini.com,inbound,ZZ,Unknown Region,0.981115 +lynxmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +lyris.net,lyris.net,inbound,ZZ,Unknown Region,0 +m1e.net,m1e.net,inbound,ZZ,Unknown Region,0.000342 +mac.com,icloud.com,outbound,ZZ,Unknown Region,1 +mac.com,mac.com,inbound,ZZ,Unknown Region,1 +macromill.com,macromill.com,inbound,ZZ,Unknown Region,0 +macys.com,macys.com,inbound,ZZ,Unknown Region,0 +magix.net,magix.net,inbound,ZZ,Unknown Region,0.673176 +mail-backcountry.com,email-bcmarketing.com,inbound,ZZ,Unknown Region,0 +mail.ru,mail.ru,inbound,ZZ,Unknown Region,0.986526 +mail.ru,mail.ru,outbound,ZZ,Unknown Region,0.006561 +maileclipse.com,emce2.in,inbound,ZZ,Unknown Region,0 +mailengine1.com,mailengine1.com,inbound,ZZ,Unknown Region,0 +mailer4u.in,elabs10.com,inbound,ZZ,Unknown Region,0 +mailersend.com,mailersend.com,inbound,ZZ,Unknown Region,0 +mailjet.com,mailjet.com,inbound,ZZ,Unknown Region,0.803083 +mailmailmail.net,mailmailmail.net,inbound,ZZ,Unknown Region,0 +mailoct.in,tcmailer14.in,inbound,ZZ,Unknown Region,0 +mailoct1.in,mailoct1.in,inbound,ZZ,Unknown Region,0 +mailoct1.in,myntramail2.in,inbound,ZZ,Unknown Region,0 +mailorama.fr,mailorama.fr,inbound,ZZ,Unknown Region,0 +mailpost.in,iaires.com,inbound,ZZ,Unknown Region,0 +mailquant.com,iaires.com,inbound,ZZ,Unknown Region,0 +mandrillapp.com,backpage.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,mcsignup.com,inbound,ZZ,Unknown Region,1 +mandrillapp.com,myjobhelperalerts.com,inbound,ZZ,Unknown Region,1 +mango.com,emstechnology2.net,inbound,ZZ,Unknown Region,0 +manipal.edu,iaires.com,inbound,ZZ,Unknown Region,0 +manta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +mar0.net,mar0.net,inbound,ZZ,Unknown Region,0.976409 +markavip.com,markavip.com,inbound,ZZ,Unknown Region,0 +marketingstudio.com,marketingstudio.com,inbound,ZZ,Unknown Region,0 +marksandspencer.com,marksandspencer.com,inbound,ZZ,Unknown Region,0 +maropost.com,mp2200.com,inbound,ZZ,Unknown Region,0 +maropost.com,survivallife.com,inbound,ZZ,Unknown Region,0 +marykay.com,marykay.com,inbound,ZZ,Unknown Region,0 +masivapp.com,masivapp.com,inbound,ZZ,Unknown Region,1 +match.com,match.com,inbound,ZZ,Unknown Region,0 +mbga.jp,mbga.jp,inbound,ZZ,Unknown Region,0 +mbna.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 +mcdlv.net,mcdlv.net,inbound,ZZ,Unknown Region,0 +mcsv.net,mcsv.net,inbound,ZZ,Unknown Region,0 +me.com,icloud.com,outbound,ZZ,Unknown Region,1 +me.com,mac.com,inbound,ZZ,Unknown Region,1 +medallia.com,medallia.com,inbound,ZZ,Unknown Region,1 +mediapost.com,mediapost.com,inbound,ZZ,Unknown Region,0 +medium.com,messagebus.com,inbound,ZZ,Unknown Region,0 +medscape.com,medscape.com,inbound,ZZ,Unknown Region,0 +meetup.com,meetup.com,inbound,ZZ,Unknown Region,0 +melaleuca.com,melaleuca.com,inbound,ZZ,Unknown Region,0 +mercadojobs.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +mercadolibre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 +mercadolivre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 +merceworld.com,merceworld.com,inbound,ZZ,Unknown Region,0.996737 +merodea.me,sendgrid.net,inbound,ZZ,Unknown Region,1 +metro.co.in,srv2.de,inbound,ZZ,Unknown Region,0.966947 +mgmresorts.com,mgmresorts.com,inbound,ZZ,Unknown Region,0 +microsoft.com,msn.com,inbound,ZZ,Unknown Region,1 +microsoftemail.com,microsoftemail.com,inbound,ZZ,Unknown Region,0 +microsoftemail.com,microsoftstoreemail.com,inbound,ZZ,Unknown Region,0 +mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,ZZ,Unknown Region,0 +minhavida.com.br,minhavida.com.br,inbound,ZZ,Unknown Region,0 +mixi.jp,mixi.jp,inbound,ZZ,Unknown Region,0 +mjinn.com,mailurja.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktdns.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktomail.com,inbound,ZZ,Unknown Region,0 +mktomail.com,mktroute.com,inbound,ZZ,Unknown Region,0 +mlsend.com,mlsend.com,inbound,ZZ,Unknown Region,0 +mlsend2.com,mlsend2.com,inbound,ZZ,Unknown Region,0 +mlssoccer.com,mlssoccer.com,inbound,ZZ,Unknown Region,0 +mmaco.net,mmaco.net,inbound,ZZ,Unknown Region,0.999998 +mmorpg.com,mmorpg.com,inbound,ZZ,Unknown Region,0.002979 +mocospace.com,mocospace.com,inbound,ZZ,Unknown Region,0 +moneyforward.com,moneyforward.com,inbound,ZZ,Unknown Region,0 +moneysupermarketmail.com,moneysupermarketmail.com,inbound,ZZ,Unknown Region,0 +monografias.com,elistas.net,inbound,ZZ,Unknown Region,0 +monster.com,monster.com,inbound,ZZ,Unknown Region,0.000503 +monsterindia.com,monster.co.in,inbound,ZZ,Unknown Region,0 +mooply.co,mailendo.com,inbound,ZZ,Unknown Region,0 +morningstar.net,morningstar.net,inbound,ZZ,Unknown Region,0 +mothercaregroup.com,neolane.net,inbound,ZZ,Unknown Region,0 +mozilla.org,mozilla.com,inbound,ZZ,Unknown Region,0.633241 +mpse.jp,mpme.jp,inbound,ZZ,Unknown Region,0 +ms00.net,ms00.net,inbound,ZZ,Unknown Region,0 +msdp1.com,msdp1.com,inbound,ZZ,Unknown Region,0 +msn.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999946 +msn.com,hotmail.{...},outbound,ZZ,Unknown Region,1 +musiciansfriend.com,musiciansfriend.com,inbound,ZZ,Unknown Region,0 +mxmfb.com,mxmfb.com,inbound,ZZ,Unknown Region,0 +mycolorscreen.com,mta4.net,inbound,ZZ,Unknown Region,0 +myfitnesspal.com,messagebus.com,inbound,ZZ,Unknown Region,0 +mygroupon.co.th,grouponmail.{...},inbound,ZZ,Unknown Region,0 +myheritage.com,myheritage.com,inbound,ZZ,Unknown Region,0 +myideeli.com,myideeli.com,inbound,ZZ,Unknown Region,0 +myntramail.com,iaires.com,inbound,ZZ,Unknown Region,0 +myntramail.com,myntramail.com,inbound,ZZ,Unknown Region,0 +myntramails.in,icubes.in,inbound,ZZ,Unknown Region,0 +myoutlets.in,trustmailer.com,inbound,ZZ,Unknown Region,0 +mypoints.com,mypoints.com,inbound,ZZ,Unknown Region,0 +mysale.my,mysale.my,inbound,ZZ,Unknown Region,0 +mysale.ph,mysale.ph,inbound,ZZ,Unknown Region,0 +mysupermarket.co.uk,mysupermarket.co.uk,inbound,ZZ,Unknown Region,0 +mysurvey.com,mysurvey.com,inbound,ZZ,Unknown Region,0 +mysurvey.eu,mysurvey.com,inbound,ZZ,Unknown Region,0 +myvegas.com,myvegas.com,inbound,ZZ,Unknown Region,1 +naaptoldeals.com,eccluster.com,inbound,ZZ,Unknown Region,0 +nanomail.com.br,araie.com.br,inbound,ZZ,Unknown Region,1 +nascar.com,nascar.com,inbound,ZZ,Unknown Region,0 +nationbuilder.com,nationbuilder.com,inbound,ZZ,Unknown Region,1 +nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,ZZ,Unknown Region,0 +naukri.com,naukri.com,inbound,ZZ,Unknown Region,0 +navy.mil,navy.mil,inbound,ZZ,Unknown Region,0.000243 +nend.net,postini.com,inbound,ZZ,Unknown Region,0 +netatlantic.com,netatlantic.com,inbound,ZZ,Unknown Region,0.001085 +netflix.com,amazonses.com,inbound,ZZ,Unknown Region,0.999999 +netflix.com,netflix.com,inbound,ZZ,Unknown Region,1 +netlogmail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 +netprosoftmail.com,netprosoftmail.com,inbound,ZZ,Unknown Region,0 +netshoes.com.br,netshoes.com.br,inbound,ZZ,Unknown Region,0 +newegg.com,newegg.com,inbound,ZZ,Unknown Region,2e-06 +newmarkethealth.com,newmarkethealth.com,inbound,ZZ,Unknown Region,0 +news-h5g.com,news-h5g.com,inbound,ZZ,Unknown Region,0 +newsletter-verychic.com,splio.es,inbound,ZZ,Unknown Region,0.9804 +newsmax.com,newsmax.com,inbound,ZZ,Unknown Region,0 +nflshop.com,nflshop.com,inbound,ZZ,Unknown Region,0 +nieuwsblad.be,vummail.be,inbound,ZZ,Unknown Region,0 +nike.com,nike.com,inbound,ZZ,Unknown Region,0 +ninewestmail.com,ninewestmail.com,inbound,ZZ,Unknown Region,0 +nokia.com,nokia.com,inbound,ZZ,Unknown Region,0.001256 +npr.org,npr.org,inbound,ZZ,Unknown Region,0 +ns.nl,tripolis.com,inbound,ZZ,Unknown Region,0 +nsandi.com,mxmfb.com,inbound,ZZ,Unknown Region,0 +nytimes.com,nytimes.com,inbound,ZZ,Unknown Region,0 +nzsale.co.nz,nzsale.co.nz,inbound,ZZ,Unknown Region,0 +oakley.com,oakley.com,inbound,ZZ,Unknown Region,0 +ocmail1.in,tcmail.in,inbound,ZZ,Unknown Region,0 +ocmail14.in,tcmailer5.in,inbound,ZZ,Unknown Region,0 +ocmail22.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 +ocmail22.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 +ocmail40.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 +ocmail40.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 +ocnmail.in,ocmail6.in,inbound,ZZ,Unknown Region,0 +ocnmail.in,tcmail3.in,inbound,ZZ,Unknown Region,0 +ofertasbmc.com.br,ofertasbmc.com.br,inbound,ZZ,Unknown Region,0 +ofertix.com,ofertix.com,inbound,ZZ,Unknown Region,0 +offers.com,offers.com,inbound,ZZ,Unknown Region,1 +officedepot.com,officedepot.com,inbound,ZZ,Unknown Region,0.019269 +officemax.com,officemax.com,inbound,ZZ,Unknown Region,0 +officemax.com,officemaxworkplace.com,inbound,ZZ,Unknown Region,0 +ofsys.com,bulletin-metro.ca,inbound,ZZ,Unknown Region,0 +oknotify2.com,oknotify2.com,inbound,ZZ,Unknown Region,0 +oldnavy.ca,oldnavy.ca,inbound,ZZ,Unknown Region,0 +oldnavy.com,oldnavy.com,inbound,ZZ,Unknown Region,0 +omahasteaks.com,omahasteaks.com,inbound,ZZ,Unknown Region,0 +oneindia.in,mailurja.com,inbound,ZZ,Unknown Region,0 +onekingslane.com,onekingslane.com,inbound,ZZ,Unknown Region,0 +onlive.com,ipost.com,inbound,ZZ,Unknown Region,0 +onmicrosoft.com,outlook.com,inbound,ZZ,Unknown Region,1 +optimusmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +orderscatalog.com,orderscatalog.com,inbound,ZZ,Unknown Region,0 +oroscopofree.com,adsender.us,inbound,ZZ,Unknown Region,0 +os-email.com,os-email.com,inbound,ZZ,Unknown Region,0 +oshkoshbgosh.com,oshkoshbgosh.com,inbound,ZZ,Unknown Region,6e-06 +osu.edu,outlook.com,inbound,ZZ,Unknown Region,1 +otto.de,eccluster.com,inbound,ZZ,Unknown Region,1 +ouffer.com,ouffer.com,inbound,ZZ,Unknown Region,0 +ourtime.com,seniorpeoplemeet.com,inbound,ZZ,Unknown Region,0 +outback.com,outback.com,inbound,ZZ,Unknown Region,0 +outlook.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999843 +outlook.com,hotmail.{...},outbound,ZZ,Unknown Region,1 +outspot.be,teneo.be,inbound,ZZ,Unknown Region,0 +outspot.nl,teneo.be,inbound,ZZ,Unknown Region,0 +ovenmail.com,iaires.com,inbound,ZZ,Unknown Region,0 +overstock.com,overstock.com,inbound,ZZ,Unknown Region,0 +ovh.net,ovh.net,inbound,ZZ,Unknown Region,0.352531 +ozsale.com.au,ozsale.com.au,inbound,ZZ,Unknown Region,0.000192 +panelplace.com,smtp.com,inbound,ZZ,Unknown Region,0 +panerabreadnews.com,panerabreadnews.com,inbound,ZZ,Unknown Region,0 +pantaloondirect.net,iaires.com,inbound,ZZ,Unknown Region,0 +papajohns-specials.com,papajohns-specials.com,inbound,ZZ,Unknown Region,0 +paradisepublishers.com,paradisepublishers.com,inbound,ZZ,Unknown Region,0.99957 +path.com,path.com,inbound,ZZ,Unknown Region,1 +payback.in,eccluster.com,inbound,ZZ,Unknown Region,0 +payback.in,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +payback.in,ecmcluster.com,inbound,ZZ,Unknown Region,0 +paypal.co.uk,paypal.com,inbound,ZZ,Unknown Region,1 +paypal.com,paypal.com,inbound,ZZ,Unknown Region,0.608834 +paypal.com.au,paypal.com,inbound,ZZ,Unknown Region,1 +paypal.de,paypal.com,inbound,ZZ,Unknown Region,1 +pd25.com,pd25.com,inbound,ZZ,Unknown Region,1 +peanuthome.info,adopterc.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,aguitytr.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bevest.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bluester.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,burror.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,bursion.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cantexi.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,caserhi.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,celect.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,chintone.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,citery.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cleathal.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,colicom.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,complec.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,deciarc.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,declaws.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,dewest.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,epconce.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,fnotec.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,folkswor.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,forepert.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,gurgaro.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,heallyps.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,holeph.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,homewor.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,hydroni.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,ingenbu.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 +peanuthome.info,unglibshudder.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,adcrent.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,addmiel.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,agilhe.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,allegap.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,andvore.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,angogl.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,animass.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,arettery.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,aribank.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,arkefoc.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,avenog.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,barrave.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bindowmo.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bitravit.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,borsand.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,branti.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,breaserp.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bredogly.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,briantra.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,bridea.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,carial.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,castac.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,chedoner.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,chiquent.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cinchoi.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cliate.info,inbound,ZZ,Unknown Region,0 +peanutwebmaster.info,cognn.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,abjibbin.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,audiette.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,blancer.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,bluester.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,burror.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,bursion.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,caserhi.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,celect.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,cleathal.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,complec.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,condost.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,deciarc.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,declaws.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,epconce.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ferrayer.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,fnotec.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,folkswor.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,forepert.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,gurgaro.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,holeph.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,homewor.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,hydroni.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ingenbu.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 +peanutwebsite.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 +peixeurbano.com.br,peixeurbano.com.br,inbound,ZZ,Unknown Region,0 +pennwell.com,pennwell.com,inbound,ZZ,Unknown Region,0 +perfectworld.com,perfectworld.com,inbound,ZZ,Unknown Region,0 +personare.com.br,personare.com.br,inbound,ZZ,Unknown Region,0 +pge.com,pge.com,inbound,ZZ,Unknown Region,0.149792 +philosophy.com,philosophy.com,inbound,ZZ,Unknown Region,0 +phpclasses.org,phpclasses.org,inbound,ZZ,Unknown Region,0 +pinger.com,pinger.com,inbound,ZZ,Unknown Region,0 +pinterest.com,pinterest.com,inbound,ZZ,Unknown Region,1 +pizzahutoffers.com,pizzahutoffers.com,inbound,ZZ,Unknown Region,0 +playstationmail.net,playstationmail.net,inbound,ZZ,Unknown Region,0 +plumdistrict.com,plumdistrict.com,inbound,ZZ,Unknown Region,1 +politicoemail.com,politicoemail.com,inbound,ZZ,Unknown Region,0 +polyvore.com,polyvore.com,inbound,ZZ,Unknown Region,1 +popsugar.com,popsugar.com,inbound,ZZ,Unknown Region,0 +priceline.com,priceline.com,inbound,ZZ,Unknown Region,1 +princess.com,princess.com,inbound,ZZ,Unknown Region,0 +privoscite.si,privoscite.si,inbound,ZZ,Unknown Region,0 +profitcenteronline.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 +progressive.com,progressive.com,inbound,ZZ,Unknown Region,0.814003 +publix.com,publix.com,inbound,ZZ,Unknown Region,0 +puritan.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 +qoinpro.com,qoinpro.com,inbound,ZZ,Unknown Region,1 +qoo10.jp,qoo10.jp,inbound,ZZ,Unknown Region,4e-06 +qoo10.sg,qoo10.co.id,inbound,ZZ,Unknown Region,1.9e-05 +qoo10.sg,qoo10.com,inbound,ZZ,Unknown Region,0 +qoo10.sg,qoo10.my,inbound,ZZ,Unknown Region,2.2e-05 +qoo10.sg,qoo10.sg,inbound,ZZ,Unknown Region,8e-06 +quinstreet.com,neoquin.com,inbound,ZZ,Unknown Region,0 +quora.com,quora.com,inbound,ZZ,Unknown Region,1 +rabota.ua,rabota.ua,inbound,ZZ,Unknown Region,0 +rackroom-email.com,rackroom-email.com,inbound,ZZ,Unknown Region,0 +railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,ZZ,Unknown Region,0 +rakuten.co.jp,rakuten.co.jp,inbound,ZZ,Unknown Region,0 +rakuten.com,rakuten.com,inbound,ZZ,Unknown Region,0 +reactiveadz.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 +realage-mail.com,postdirect.com,inbound,ZZ,Unknown Region,0 +redbox.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +redcross.org.uk,redcross.org.uk,inbound,ZZ,Unknown Region,0 +rediffmail.com,rediffmail.com,inbound,ZZ,Unknown Region,0 +reebonz.com,reebonz.com,inbound,ZZ,Unknown Region,0.759458 +reed.co.uk,reed.co.uk,inbound,ZZ,Unknown Region,0 +regie11.net,odiso.net,inbound,ZZ,Unknown Region,0 +regionalhelpwanted.com,regionalhelpwanted.com,inbound,ZZ,Unknown Region,1 +registrar-servers.com,registrar-servers.com,inbound,ZZ,Unknown Region,0.05163 +repica.jp,kakaku.com,inbound,ZZ,Unknown Region,0 +repica.jp,repica.jp,inbound,ZZ,Unknown Region,0 +republicwireless.com,republicwireless.com,inbound,ZZ,Unknown Region,0.033564 +retailjobinsider.com,retailjobinsider.com,inbound,ZZ,Unknown Region,0 +retailmenot.com,retailmenot.com,inbound,ZZ,Unknown Region,4.68294583517529e-07 +reverbnation.com,reverbnation.com,inbound,ZZ,Unknown Region,0 +revolutiongolf.com,revolutiongolf.com,inbound,ZZ,Unknown Region,0 +reyrey.net,reyrey.net,inbound,ZZ,Unknown Region,0.999904 +richyrichmailer.com,maddog-productions.info,inbound,ZZ,Unknown Region,1 +rikunabi.com,rikunabi.com,inbound,ZZ,Unknown Region,0 +ringcentral.com,ringcentral.com,inbound,ZZ,Unknown Region,1 +rmtr.de,rapidmail.de,inbound,ZZ,Unknown Region,0 +rnmk.com,rnmk.com,inbound,ZZ,Unknown Region,0 +rockpath.info,rockpath.info,inbound,ZZ,Unknown Region,1 +rogers.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +rogers.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +rookiestewsemails.com,rookiestewsemails.com,inbound,ZZ,Unknown Region,0 +royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,ZZ,Unknown Region,0 +rpo9usa.email,rpo9usa.email,inbound,ZZ,Unknown Region,0 +rr.com,rr.com,inbound,ZZ,Unknown Region,2e-06 +rr.com,rr.com,outbound,ZZ,Unknown Region,0 +rsgsv.net,rsgsv.net,inbound,ZZ,Unknown Region,0 +rummycirclemails.com,eccluster.com,inbound,ZZ,Unknown Region,0 +runkeeper.com,runkeeper.com,inbound,ZZ,Unknown Region,0 +s3s-br1.net,splio.com.br,inbound,ZZ,Unknown Region,0.908187 +s3s-main.net,splio.com,inbound,ZZ,Unknown Region,0.970428 +s4s-pl1.pl,splio.com,inbound,ZZ,Unknown Region,0.939032 +safe-sender.net,safe-sender.net,inbound,ZZ,Unknown Region,0 +safeway.com,safeway.com,inbound,ZZ,Unknown Region,0.000382 +salesforce.com,postini.com,inbound,ZZ,Unknown Region,0.915635 +salesforce.com,salesforce.com,inbound,ZZ,Unknown Region,0.952721 +samsungusa.com,samsungusa.com,inbound,ZZ,Unknown Region,0 +sanmina-sci.com,postini.com,inbound,ZZ,Unknown Region,0.999991 +sanmina.com,postini.com,inbound,ZZ,Unknown Region,0.999828 +saturday.com,saturday.com,inbound,ZZ,Unknown Region,0 +sbcglobal.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999985 +sears.ca,sears.ca,inbound,ZZ,Unknown Region,0.005447 +searscard.com,searscard.com,inbound,ZZ,Unknown Region,1 +secretescapes.com,secretescapes.com,inbound,ZZ,Unknown Region,0 +secure.ne.jp,secure.ne.jp,inbound,ZZ,Unknown Region,0.000618 +secureserver.net,secureserver.net,inbound,ZZ,Unknown Region,0 +seek.com.au,seek.com.au,inbound,ZZ,Unknown Region,0 +selectacast.net,selectacast.net,inbound,ZZ,Unknown Region,0.000561 +semana.com,semana.com,inbound,ZZ,Unknown Region,0 +sendgrid.info,sendgrid.net,inbound,ZZ,Unknown Region,0.999896 +sendgrid.me,sendgrid.net,inbound,ZZ,Unknown Region,1 +sendpal.in,sendpal.in,inbound,ZZ,Unknown Region,0 +serviciobancomer.com,serviciobancomer.com,inbound,ZZ,Unknown Region,0 +sfid01.com,sfid01.com,inbound,ZZ,Unknown Region,0 +shaadi.com,shaadi.com,inbound,ZZ,Unknown Region,0 +shop2gether.com.br,shop2gether.com.br,inbound,ZZ,Unknown Region,0 +shopjustice.com,shopjustice.com,inbound,ZZ,Unknown Region,0 +shopnineteenmails.in,iaires.com,inbound,ZZ,Unknown Region,0 +shoppersstop.com,shoppersstop.com,inbound,ZZ,Unknown Region,0 +shoprite-email.com,email-mywebgrocer.com,inbound,ZZ,Unknown Region,0 +showingtime.com,showingtime.com,inbound,ZZ,Unknown Region,0 +showroomprive.com,showroomprive.be,inbound,ZZ,Unknown Region,0 +showroomprive.com,showroomprive.nl,inbound,ZZ,Unknown Region,0 +showroomprive.es,showroomprive.es,inbound,ZZ,Unknown Region,0 +showroomprive.it,showroomprive.pt,inbound,ZZ,Unknown Region,0 +showroomprive.pt,showroomprive.co.uk,inbound,ZZ,Unknown Region,0 +shtyle.fm,shtyle.fm,inbound,ZZ,Unknown Region,0 +simplesafelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +simpletextadz.com,web-hosting.com,inbound,ZZ,Unknown Region,1 +singsale.com.sg,singsale.com.sg,inbound,ZZ,Unknown Region,0 +skillpages-mailer.com,dynect.net,inbound,ZZ,Unknown Region,0 +skymall.com,skymall.com,inbound,ZZ,Unknown Region,0 +skynet.be,belgacom.be,inbound,ZZ,Unknown Region,0 +skype.com,skype.com,inbound,ZZ,Unknown Region,0 +skyscanner.net,skyscanner.net,inbound,ZZ,Unknown Region,1 +slickdeals.net,slickdeals.net,inbound,ZZ,Unknown Region,0 +slidesharemail.com,slideshare.net,inbound,ZZ,Unknown Region,1 +smartresponder.ru,smartresponder.ru,inbound,ZZ,Unknown Region,1 +snagajob-email.com,snagajob-email.com,inbound,ZZ,Unknown Region,0 +snapdeal.com,snapdeal.com,inbound,ZZ,Unknown Region,0 +socialsex.biz,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +softbank.jp,softbank.jp,outbound,ZZ,Unknown Region,0 +softbank.ne.jp,softbank.ne.jp,inbound,ZZ,Unknown Region,0 +softbank.ne.jp,softbank.ne.jp,outbound,ZZ,Unknown Region,0 +sony.com,sony.com,inbound,ZZ,Unknown Region,0 +sonyrewards.com,sonyrewards.com,inbound,ZZ,Unknown Region,0 +soundcloudmail.com,soundcloudmail.com,inbound,ZZ,Unknown Region,0.999996 +spanishdict.com,spanishdict.com,inbound,ZZ,Unknown Region,0 +sparklist.com,sparklist.com,inbound,ZZ,Unknown Region,0 +spartoo.com,spartoo.com,inbound,ZZ,Unknown Region,0 +speedyrewards-email.com,speedyrewards-email.com,inbound,ZZ,Unknown Region,0 +spotifymail.com,spotifymail.com,inbound,ZZ,Unknown Region,1 +ssgadm.com,ssg.com,inbound,ZZ,Unknown Region,0 +staffeazymailers.com,iaires.com,inbound,ZZ,Unknown Region,0 +stakemail.com,iaires.com,inbound,ZZ,Unknown Region,0 +stampmail.in,iaires.com,inbound,ZZ,Unknown Region,0 +standaard.be,vummail.be,inbound,ZZ,Unknown Region,0 +stansberryresearch.com,stansberry-re.net,inbound,ZZ,Unknown Region,0 +stansberryresearch.com,stansberryresearch.com,inbound,ZZ,Unknown Region,0 +staples.co.uk,ncrwebhost.de,inbound,ZZ,Unknown Region,0 +starsports.com,eccluster.com,inbound,ZZ,Unknown Region,0 +startwire.com,jobsreport.com,inbound,ZZ,Unknown Region,1 +starwoodhotels.com,outlook.com,inbound,ZZ,Unknown Region,1 +state-of-the-art-mailer.com,futurebanners.net,inbound,ZZ,Unknown Region,0 +stayfriends.de,stayfriends.de,inbound,ZZ,Unknown Region,0 +steampowered.com,steampowered.com,inbound,ZZ,Unknown Region,1 +stelladot.com,stelladot.com,inbound,ZZ,Unknown Region,0 +stjobs.sg,st701.com,inbound,ZZ,Unknown Region,1 +stjude.org,stjude.org,inbound,ZZ,Unknown Region,0.006723 +strava.com,strava.com,inbound,ZZ,Unknown Region,1 +streeteasy.com,streeteasy.com,inbound,ZZ,Unknown Region,0 +stylecareers.com,stylecareers.com,inbound,ZZ,Unknown Region,0 +subtend.info,subtend.info,inbound,ZZ,Unknown Region,1 +subway.com,subway.com,inbound,ZZ,Unknown Region,0 +surveyspot.com,ssisurveys.com,inbound,ZZ,Unknown Region,0 +sweepstakesalerts.com,sweepstakesalerts.com,inbound,ZZ,Unknown Region,0 +swimoutlet.com,isport.com,inbound,ZZ,Unknown Region,0 +sympatico.ca,hotmail.{...},inbound,ZZ,Unknown Region,1 +synchronyfinancial.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tadtopmails.com,tadtopmails.com,inbound,ZZ,Unknown Region,0 +taishinbank.com.tw,taishinbank.com.tw,inbound,ZZ,Unknown Region,0 +take2games.com,take2games.com,inbound,ZZ,Unknown Region,0 +talkmatch.com,talkmatch.com,inbound,ZZ,Unknown Region,0 +tanga.com,tanga.com,inbound,ZZ,Unknown Region,0 +tanningmail.com,tanningmail.com,inbound,ZZ,Unknown Region,0 +tappingsolutionemail.com,tappingsolutionemail.com,inbound,ZZ,Unknown Region,0 +target.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tasteofhome.com,tasteofhome.com,inbound,ZZ,Unknown Region,0 +tchibo.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 +teambuymail.com,teambuymail.com,inbound,ZZ,Unknown Region,0 +teamsnap.com,teamsnap.com,inbound,ZZ,Unknown Region,1 +technolutions.net,technolutions.net,inbound,ZZ,Unknown Region,1 +telegraph.co.uk,telegraph.co.uk,inbound,ZZ,Unknown Region,0 +telus.com,telus.com,inbound,ZZ,Unknown Region,0 +templeandwebster.com.au,templeandwebster.com.au,inbound,ZZ,Unknown Region,0 +texasjobdepartment.com,texasjobdepartment.com,inbound,ZZ,Unknown Region,0 +thebodyshop-usa.com,email-bodyshop.com,inbound,ZZ,Unknown Region,0 +thebodyshop-usa.com,postdirect.com,inbound,ZZ,Unknown Region,0 +thecarousell.com,thecarousell.com,inbound,ZZ,Unknown Region,1 +theguardian.com,theguardian.com,inbound,ZZ,Unknown Region,0 +thepamperedchef.com,thepamperedchef.com,inbound,ZZ,Unknown Region,0 +thephonehouse.es,splio.com,inbound,ZZ,Unknown Region,0.983578 +thephonehouse.es,splio.es,inbound,ZZ,Unknown Region,0.983266 +therealreal.com,email-realreal.com,inbound,ZZ,Unknown Region,0 +thesource.ca,thesource.ca,inbound,ZZ,Unknown Region,0 +thewarehouse.co.nz,thewarehouse.co.nz,inbound,ZZ,Unknown Region,0 +thinkgeek.com,thinkgeek.com,inbound,ZZ,Unknown Region,1e-06 +thirtyonegifts.com,thirtyonegifts.com,inbound,ZZ,Unknown Region,0 +thomascook.com,eccluster.com,inbound,ZZ,Unknown Region,0 +ticketmaster.com,ticketmaster.com,inbound,ZZ,Unknown Region,0.001001 +ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,ZZ,Unknown Region,0 +timehop.com,timehop.com,inbound,ZZ,Unknown Region,1 +timeout.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 +timewarnercable.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 +tinyletterapp.com,tinyletterapp.com,inbound,ZZ,Unknown Region,0 +tobi.com,messagebus.com,inbound,ZZ,Unknown Region,0 +topface.com,topface.com,inbound,ZZ,Unknown Region,1e-06 +topica.com,topica-silver-y.com,inbound,ZZ,Unknown Region,0 +topqpon.si,topqpon.si,inbound,ZZ,Unknown Region,0 +touchbase2.com,mailurja.com,inbound,ZZ,Unknown Region,0 +townnews-mail.com,townnews-mail.com,inbound,ZZ,Unknown Region,0 +trabajar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999997 +trabalhar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999994 +tradeloop.com,tradeloop.com,inbound,ZZ,Unknown Region,1 +trafficwave.net,trafficwave.net,inbound,ZZ,Unknown Region,0 +transittraveljobinsider.com,transittraveljobinsider.com,inbound,ZZ,Unknown Region,0 +transportexchangegroup.com,transportexchangegroup.com,inbound,ZZ,Unknown Region,0.998825 +travelchannel.com,travelchannel.com,inbound,ZZ,Unknown Region,0 +travelocity.com,travelocity.com,inbound,ZZ,Unknown Region,0.000195 +travelzoo.com,travelzoo.com,inbound,ZZ,Unknown Region,0 +trclient.com,trclient.com,inbound,ZZ,Unknown Region,0 +trello.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 +triongames.com,triongames.com,inbound,ZZ,Unknown Region,0.085718 +tripadvisor.com,tripadvisor.com,inbound,ZZ,Unknown Region,0.000588 +tripolis.com,tripolis.com,inbound,ZZ,Unknown Region,0 +trulia.com,trulia.com,inbound,ZZ,Unknown Region,0 +tsmmail.com,tsmmail.com,inbound,ZZ,Unknown Region,0 +tumblr.com,tumblr.com,inbound,ZZ,Unknown Region,1 +turbine.com,turbine.com,inbound,ZZ,Unknown Region,0 +turner.com,cnn.com,inbound,ZZ,Unknown Region,0 +twe-safelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 +twitter.com,twitter.com,inbound,ZZ,Unknown Region,0.999969 +twoomail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 +ubivox.com,ubivox.com,inbound,ZZ,Unknown Region,0.964085 +uga.edu,outlook.com,inbound,ZZ,Unknown Region,1 +uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,ZZ,Unknown Region,0 +ulta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 +ulta.com,ulta.com,inbound,ZZ,Unknown Region,0 +uniqlo-usa.com,uniqlo-usa.com,inbound,ZZ,Unknown Region,0 +unitedrepublic.org,unitedrepublic.org,inbound,ZZ,Unknown Region,1 +unosinsidersclub.com,unosinsidersclub.com,inbound,ZZ,Unknown Region,0 +urx.com.br,urx.com.br,inbound,ZZ,Unknown Region,0 +usaa.com,usaa.com,inbound,ZZ,Unknown Region,0.999997 +usahockey-email.com,usahockey-email.com,inbound,ZZ,Unknown Region,0 +usndr.com,usndr.com,inbound,ZZ,Unknown Region,1 +usx.com.br,uqx.com.br,inbound,ZZ,Unknown Region,0 +usx.com.br,usx.com.br,inbound,ZZ,Unknown Region,0 +usx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 +utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,ZZ,Unknown Region,0 +utx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 +uvarosa.com.br,uvarosa.com.br,inbound,ZZ,Unknown Region,0 +vakifbank.com.tr,vakifbank.com.tr,inbound,ZZ,Unknown Region,1 +vd.nl,emsecure.net,inbound,ZZ,Unknown Region,0 +venca.es,eccluster.com,inbound,ZZ,Unknown Region,0 +verizon.com,verizon.com,inbound,ZZ,Unknown Region,0.999816 +vfoutletvip.com,vfoutletvip.com,inbound,ZZ,Unknown Region,0 +vicinity.nl,picsrv.net,inbound,ZZ,Unknown Region,0.022107 +vietcombank.com.vn,vietcombank.com.vn,inbound,ZZ,Unknown Region,1 +viralsender.com,viralsender.com,inbound,ZZ,Unknown Region,0 +vistaprint.com,vistaprint.com,inbound,ZZ,Unknown Region,0 +vistaprint.com.au,vistaprint.com.au,inbound,ZZ,Unknown Region,0 +vitaminworld.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 +vk.com,vkontakte.ru,inbound,ZZ,Unknown Region,0 +vocus.com,vocus.com,inbound,ZZ,Unknown Region,0 +vovici.com,vovici.com,inbound,ZZ,Unknown Region,0 +vresp.com,verticalresponse.com,inbound,ZZ,Unknown Region,0 +vudu.com,vudu.com,inbound,ZZ,Unknown Region,0 +walmart.ca,walmart.ca,inbound,ZZ,Unknown Region,0 +walmart.com,walmart.com,inbound,ZZ,Unknown Region,0.315059 +warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,ZZ,Unknown Region,0 +way2sms.biz,way2sms.biz,inbound,ZZ,Unknown Region,0 +way2sms.in,way2sms.in,inbound,ZZ,Unknown Region,0 +way2smsemail.com,way2smsemail.com,inbound,ZZ,Unknown Region,0 +way2smsemails.com,way2smsemails.com,inbound,ZZ,Unknown Region,0 +way2smsmail.in,way2smsmail.in,inbound,ZZ,Unknown Region,0 +way2smsmails.com,way2smsmails.com,inbound,ZZ,Unknown Region,0 +wealthyaffiliate.com,wealthyaffiliate.com,inbound,ZZ,Unknown Region,1 +webmd.com,webmd.com,inbound,ZZ,Unknown Region,0 +wegottickets.com,wegottickets.com,inbound,ZZ,Unknown Region,0 +weheartit.com,weheartit.com,inbound,ZZ,Unknown Region,1 +wehkamp.nl,wehkamp.nl,inbound,ZZ,Unknown Region,0 +wellsfargo.com,wellsfargo.com,inbound,ZZ,Unknown Region,1 +wemakeprice.com,wemakeprice.com,inbound,ZZ,Unknown Region,0 +westwing.com.br,cust-cluster.com,inbound,ZZ,Unknown Region,0 +westwing.es,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +westwing.ru,ecm-cluster.com,inbound,ZZ,Unknown Region,0 +wgbh.org,wgbh.org,inbound,ZZ,Unknown Region,0 +whaakky.com,whaakky.com,inbound,ZZ,Unknown Region,0 +whereareyounow.com,wayn.net,inbound,ZZ,Unknown Region,0 +whitehouse.gov,whitehouse.gov,inbound,ZZ,Unknown Region,0 +whitelabelpros.com,whitelabelpros.com,inbound,ZZ,Unknown Region,0 +wikia.com,wikia.com,inbound,ZZ,Unknown Region,0.289222 +wisdomitservices.com,infimail.com,inbound,ZZ,Unknown Region,0 +wolfmedia.us,wolfmedia.us,inbound,ZZ,Unknown Region,0 +wordfly.com,wordfly.com,inbound,ZZ,Unknown Region,0 +workhunter.net,workhunter.net,inbound,ZZ,Unknown Region,1 +worldwinner.com,worldwinner.com,inbound,ZZ,Unknown Region,0 +wowcher.co.uk,wowcher.co.uk,inbound,ZZ,Unknown Region,0 +wp.com,wordpress.com,inbound,ZZ,Unknown Region,0 +wpengine.com,wpengine.com,inbound,ZZ,Unknown Region,1 +writers-community.com,writers-community.com,inbound,ZZ,Unknown Region,0 +writersstore.com,writersstore.com,inbound,ZZ,Unknown Region,0 +wsjemail.com,wsjemail.com,inbound,ZZ,Unknown Region,0 +wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,ZZ,Unknown Region,0 +xbox.com,xbox.com,inbound,ZZ,Unknown Region,0 +xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,ZZ,Unknown Region,0 +xing.com,xing.com,inbound,ZZ,Unknown Region,0 +xxxconnect.com,infinitypersonals.com,inbound,ZZ,Unknown Region,0 +yahoo-inc.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +yahoo.{...},yahoo.{...},inbound,ZZ,Unknown Region,0.999989 +yahoo.{...},yahoodns.net,outbound,ZZ,Unknown Region,1 +yahoogroups.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +yammer.com,yammer.com,inbound,ZZ,Unknown Region,1 +yapikredi.com.tr,yapikredi.com.tr,inbound,ZZ,Unknown Region,1 +yapstone.com,yapstone.com,inbound,ZZ,Unknown Region,0 +yesbank.in,yesbank.in,inbound,ZZ,Unknown Region,0 +yipit.com,yipit.com,inbound,ZZ,Unknown Region,1 +ymail.com,yahoo.{...},inbound,ZZ,Unknown Region,1 +ymail.com,yahoodns.net,outbound,ZZ,Unknown Region,1 +youravon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 +yournewsletters.net,everydayhealth.com,inbound,ZZ,Unknown Region,0 +youversion.com,youversion.com,inbound,ZZ,Unknown Region,1 +zappos.com,zappos.com,inbound,ZZ,Unknown Region,0.625377 +zattoo.com,sendnode.com,inbound,ZZ,Unknown Region,0 +zelonews.com.br,zelonews.com.br,inbound,ZZ,Unknown Region,0 +zendesk.com,zdsys.com,inbound,ZZ,Unknown Region,1 +zibmail.info,zibmail.info,inbound,ZZ,Unknown Region,0 +zillow.com,zillow.com,inbound,ZZ,Unknown Region,2.82543102656668e-07 +zinio.net,zinio.com,inbound,ZZ,Unknown Region,1 +zipalerts.com,sendgrid.net,inbound,ZZ,Unknown Region,1 +zipalerts.com,zipalerts.com,inbound,ZZ,Unknown Region,1 +zlavadna.sk,zlavadna.sk,inbound,ZZ,Unknown Region,0 +zoom.com.br,zoom.com.br,inbound,ZZ,Unknown Region,0 +zoominternet.net,synacor.com,inbound,ZZ,Unknown Region,0 +zoosk.com,zoosk.com,inbound,ZZ,Unknown Region,0 +zorpia.com,zorpia.com,inbound,ZZ,Unknown Region,0.520147 +zovifashion.com,eccluster.com,inbound,ZZ,Unknown Region,0 +zyngamail.com,zyngamail.com,inbound,ZZ,Unknown Region,0 \ No newline at end of file From 6a1aa8e6b640e0c6504cdbb9635f88ec9929cd7d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 17:34:21 -0400 Subject: [PATCH 057/639] Update to latest config format. --- CheckSTARTTLS.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 205a4a779..9e5117b84 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -7,6 +7,7 @@ import socket import subprocess import re import json +import collections import dns.resolver from M2Crypto import X509 @@ -152,12 +153,8 @@ if __name__ == '__main__': if len(sys.argv) == 1: print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") - config = { - "address-domains": { - }, - "mx-domains": { - } - } + config = collections.defaultdict(dict) + for domain in open(sys.argv[1]).readlines(): domain = domain.strip() if not os.path.exists(domain): @@ -168,10 +165,10 @@ if __name__ == '__main__': min_version = min_tls_version(domain) if suffix != "": suffix_match = "." + suffix - config["address-domains"][domain] = { + config["acceptable-mxs"][domain] = { "accept-mx-domains": [suffix_match] } - config["mx-domains"][suffix_match] = { + config["tls-policies"][suffix_match] = { "require-tls": True, "min-tls-version": min_version } From 749c4e39e0fa539a36cfa717a07554153ff97406 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 7 Aug 2014 17:34:40 -0400 Subject: [PATCH 058/639] Update meta-config with latest domains. --- starttls-everywhere.json | 93 +++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/starttls-everywhere.json b/starttls-everywhere.json index 5dd487f9a..db76745ef 100644 --- a/starttls-everywhere.json +++ b/starttls-everywhere.json @@ -1,5 +1,15 @@ { - "address-domains": { + "acceptable-mxs": { + "163.com": { + "accept-mx-domains": [ + ".163.com" + ] + }, + "aol.com": { + "accept-mx-domains": [ + ".aol.com" + ] + }, "craigslist.org": { "accept-mx-domains": [ ".craigslist.org" @@ -10,19 +20,49 @@ ".google.com" ] }, - "interia.pl": { + "hotmail.com": { "accept-mx-domains": [ - ".interia.pl" + ".outlook.com" ] }, - "marktplaats.nl": { + "icloud.com": { "accept-mx-domains": [ - ".marktplaats.nl" + ".icloud.com" ] }, - "rambler.ru": { + "live.com": { "accept-mx-domains": [ - ".rambler.ru" + ".outlook.com" + ] + }, + "mac.com": { + "accept-mx-domains": [ + ".icloud.com" + ] + }, + "me.com": { + "accept-mx-domains": [ + ".icloud.com" + ] + }, + "msn.com": { + "accept-mx-domains": [ + ".outlook.com" + ] + }, + "naver.com": { + "accept-mx-domains": [ + ".naver.com" + ] + }, + "outlook.com": { + "accept-mx-domains": [ + ".outlook.com" + ] + }, + "qq.com": { + "accept-mx-domains": [ + ".qq.com" ] }, "rocketmail.com": { @@ -45,9 +85,14 @@ ".yahoo.com" ] }, - "sompo-japan.co.jp": { + "shaw.ca": { "accept-mx-domains": [ - ".psmtp.com" + ".shaw.ca" + ] + }, + "sympatico.ca": { + "accept-mx-domains": [ + ".outlook.com" ] }, "t-online.de": { @@ -60,12 +105,12 @@ ".wp.pl" ] }, - "yahoo.co.uk": { + "yahoo.com": { "accept-mx-domains": [ ".yahoo.com" ] }, - "yahoo.com": { + "yahoogroups.com": { "accept-mx-domains": [ ".yahoo.com" ] @@ -81,7 +126,15 @@ ] } }, - "mx-domains": { + "tls-policies": { + ".163.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".aol.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, ".craigslist.org": { "min-tls-version": "TLSv1.1", "require-tls": true @@ -90,11 +143,15 @@ "min-tls-version": "TLSv1.1", "require-tls": true }, - ".interia.pl": { + ".icloud.com": { "min-tls-version": "TLSv1", "require-tls": true }, - ".marktplaats.nl": { + ".naver.com": { + "min-tls-version": "TLSv1.1", + "require-tls": true + }, + ".outlook.com": { "min-tls-version": "TLSv1.1", "require-tls": true }, @@ -102,8 +159,12 @@ "min-tls-version": "TLSv1", "require-tls": true }, - ".rambler.ru": { - "min-tls-version": "TLSv1.1", + ".qq.com": { + "min-tls-version": "TLSv1", + "require-tls": true + }, + ".shaw.ca": { + "min-tls-version": "TLSv1", "require-tls": true }, ".t-online.de": { From cebc6f9a205696d7f849bba6ebbd6af549849043 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 8 Aug 2014 13:28:01 -0400 Subject: [PATCH 059/639] ProcessGoogleSTARTTLSDomains -> latest CSV format. Also output in a more useful format, require >= 99% encrypted output in the CSV, hande .{...} domains, and manually add gmail.com. --- ProcessGoogleSTARTTLSDomains.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ProcessGoogleSTARTTLSDomains.py b/ProcessGoogleSTARTTLSDomains.py index abb2b3495..3078bd93a 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/ProcessGoogleSTARTTLSDomains.py @@ -15,8 +15,15 @@ from collections import defaultdict csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') d = defaultdict(set) -for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in csvreader: +# Google's report doesn't include gmail.com because it's local delivery, but we +# know they support STARTTLS, so manually include them. +d["gmail.com"] = set([1]) +for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader: if direction == "outbound": + # Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}. + # We're tryingto get a solid list of the relevant TLDs, but in the meantime + # just use .com. + address_suffix = address_suffix.replace("{...}", "com") try: d[address_suffix].add(float(fraction_encrypted)) except ValueError: @@ -24,4 +31,4 @@ for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in for address_suffix, fraction_encrypted in d.iteritems(): if min(fraction_encrypted) >= 0.99: - print min(fraction_encrypted), address_suffix + print address_suffix From 21ff3acf932c74455e2d50ac4679afbd5cfb27a4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:26:59 -0700 Subject: [PATCH 060/639] Set/list comprehensions are a bit more readable than lambdas --- CheckSTARTTLS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 9e5117b84..ffff05dce 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -24,7 +24,7 @@ def mkdirp(path): else: raise def extract_names(pem): - """Return a list of DNS subject names from PEM-encoded leaf cert.""" + """Return a set of DNS subject names from PEM-encoded leaf cert.""" leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) subj = leaf.get_subject() @@ -93,7 +93,7 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) - new_names = map(lambda n: public_suffix_list.get_public_suffix(n), new_names) + new_names = set(public_suffix_list.get_public_suffix(n) for in new_names) names.update(new_names) if len(names) >= 1: # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. From ff5810d78f0cdc4e46de03470aa685744b05fa91 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:36:04 -0700 Subject: [PATCH 061/639] Don't accept files on the command line that don't do anything --- CheckSTARTTLS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index ffff05dce..eda1761f0 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -150,7 +150,7 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) == 1: + if len(sys.argv) != 2: # XXX or accept multiple files as input print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = collections.defaultdict(dict) From 0bd8134e5fde7930de557c041adf69199ef2cd5a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 11:57:01 -0700 Subject: [PATCH 062/639] Comments (and code review in comment form) --- CheckSTARTTLS.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index eda1761f0..a9d8de9de 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -86,6 +86,8 @@ def valid_cert(filename): return False def check_certs(mail_domain): + # Return "" if any certs for any mx domains pointed to by mail_domain + # were invalid, and a public suffix for one if they were all valid names = set() for mx_hostname in os.listdir(mail_domain): filename = os.path.join(mail_domain, mx_hostname) @@ -141,6 +143,8 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): + # XXX comment this function and explain why we're using the + # filesystem rather than internal data structures for plumbing here print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') From 30ba7e930587ac40104c9bf9d6173e2f8706eb34 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:01:04 -0700 Subject: [PATCH 063/639] Deduplicate stray comment line --- MTAConfigGenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 4d2f3432a..061b9445f 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -6,7 +6,6 @@ import os, os.path def parse_line(line_data): """ - Return the left and right hand sides of stripped, non-comment postfix Return the (line number, left hand side, right hand side) of a stripped postfix config line. From 8c6d28ce9583a56fa07d342fdbaf9b4c72eddf0f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:01:33 -0700 Subject: [PATCH 064/639] Comment with some sample postfix log lines (both those we support already, and those we may want to in the future...) --- PostfixLogSummary.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index daec93db0..566651cc2 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -5,10 +5,26 @@ import collections import ConfigParser +# XXX There's more to be learned from postfix logs! Here's one sample +# observed during failures from the sender vagrant vm: + +# Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] +# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1] +# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1] +# Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix +# Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from= +# Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com> +# Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=, size=576, nrcpt=1 (queue active) +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: +# Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) +# def get_counts(input, config): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) + # Typical line looks like: + # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) From 9cafcf1caf14ca8bb81bf61f9597014f6c060b71 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:15:15 -0700 Subject: [PATCH 065/639] Comment regexp --- PostfixLogSummary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 566651cc2..db3653a6f 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -25,6 +25,7 @@ def get_counts(input, config): counts = collections.defaultdict(lambda: collections.defaultdict(int)) # Typical line looks like: # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) + # ([^[]*) <--- any group of characters that is not "[" r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: result = r.search(line) From 78a55c3823b1794c9ff6d7e268e78e785d817b93 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Aug 2014 13:16:40 -0700 Subject: [PATCH 066/639] Question about postfix logparsing output --- PostfixLogSummary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index db3653a6f..1c9d64610 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -40,6 +40,7 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 if not seen_trusted: + # XXX aren't these outbound? How can the admin install certs? print "Didn't see any trusted connections. Need to install some certs?" return counts From e0edc1b7ec0867d2624ef56d33da79765d776554 Mon Sep 17 00:00:00 2001 From: jsha Date: Tue, 12 Aug 2014 12:18:32 -0400 Subject: [PATCH 067/639] Add link to mailing list. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e50630ab8..3f5818255 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ Jacob Hoffman-Andrews , Peter Eckersley +## Mailing List + +starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere + ## Background Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. From ad40618897ee3905a7e651443ba0fb624eebbfdf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 13 Aug 2014 10:49:50 -0400 Subject: [PATCH 068/639] Respond to pde's comments --- CheckSTARTTLS.py | 52 ++++++++++++++++++++++++-------------------- PostfixLogSummary.py | 7 +++--- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index a9d8de9de..7c9c94e0f 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -86,8 +86,10 @@ def valid_cert(filename): return False def check_certs(mail_domain): - # Return "" if any certs for any mx domains pointed to by mail_domain - # were invalid, and a public suffix for one if they were all valid + """ + Return "" if any certs for any mx domains pointed to by mail_domain + were invalid, and a public suffix for one if they were all valid + """ names = set() for mx_hostname in os.listdir(mail_domain): filename = os.path.join(mail_domain, mx_hostname) @@ -95,7 +97,7 @@ def check_certs(mail_domain): return "" else: new_names = extract_names_from_openssl_output(filename) - new_names = set(public_suffix_list.get_public_suffix(n) for in new_names) + new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names) names.update(new_names) if len(names) >= 1: # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. @@ -143,8 +145,11 @@ def min_tls_version(mail_domain): return min(protocols) def collect(mail_domain): - # XXX comment this function and explain why we're using the - # filesystem rather than internal data structures for plumbing here + """ + Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS. + Store the output in a directory with the same name as mail_domain to make + subsequent analysis faster. + """ print "Checking domain %s" % mail_domain mkdirp(mail_domain) answers = dns.resolver.query(mail_domain, 'MX') @@ -154,27 +159,28 @@ def collect(mail_domain): if __name__ == '__main__': """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) != 2: # XXX or accept multiple files as input + if len(sys.argv) < 2: print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") config = collections.defaultdict(dict) - for domain in open(sys.argv[1]).readlines(): - domain = domain.strip() - if not os.path.exists(domain): - collect(domain) - if len(os.listdir(domain)) == 0: - continue - suffix = check_certs(domain) - min_version = min_tls_version(domain) - if suffix != "": - suffix_match = "." + suffix - config["acceptable-mxs"][domain] = { - "accept-mx-domains": [suffix_match] - } - config["tls-policies"][suffix_match] = { - "require-tls": True, - "min-tls-version": min_version - } + for input in sys.argv[1:]: + for domain in open(input).readlines(): + domain = domain.strip() + if not os.path.exists(domain): + collect(domain) + if len(os.listdir(domain)) == 0: + continue + suffix = check_certs(domain) + min_version = min_tls_version(domain) + if suffix != "": + suffix_match = "." + suffix + config["acceptable-mxs"][domain] = { + "accept-mx-domains": [suffix_match] + } + config["tls-policies"][suffix_match] = { + "require-tls": True, + "min-tls-version": min_version + } print json.dumps(config, indent=2, sort_keys=True) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 1c9d64610..b68c7f6c6 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -5,7 +5,7 @@ import collections import ConfigParser -# XXX There's more to be learned from postfix logs! Here's one sample +# TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: # Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] @@ -40,8 +40,9 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 if not seen_trusted: - # XXX aren't these outbound? How can the admin install certs? - print "Didn't see any trusted connections. Need to install some certs?" + # Postfix will only emit 'Trusted' if the certificate validates according to + # the set of trust roots (CA certs) configured in smtp_tls_CAfile. + print "Didn't see any trusted connections. Need to install some trust roots?" return counts def print_summary(counts): From 31e320d0a7832ca8d25032eeb321cce75f48b51b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 13 Aug 2014 15:59:50 -0400 Subject: [PATCH 069/639] Collect certs in a subdir. --- CheckSTARTTLS.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index 7c9c94e0f..e8adc010e 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -14,6 +14,7 @@ from M2Crypto import X509 from publicsuffix import PublicSuffixList public_suffix_list = PublicSuffixList() +CERTS_OBSERVED = 'certs-observed' def mkdirp(path): try: @@ -63,7 +64,7 @@ def tls_connect(mx_host, mail_domain): return # Save a copy of the certificate for later analysis - with open(os.path.join(mail_domain, mx_host), "w") as f: + with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f: f.write(output) def valid_cert(filename): @@ -90,9 +91,12 @@ def check_certs(mail_domain): Return "" if any certs for any mx domains pointed to by mail_domain were invalid, and a public suffix for one if they were all valid """ + dir = os.path.join(CERTS_OBSERVED, mail_domain) + if not os.path.exists(dir): + collect(mail_domain) names = set() - for mx_hostname in os.listdir(mail_domain): - filename = os.path.join(mail_domain, mx_hostname) + for mx_hostname in os.listdir(dir): + filename = os.path.join(dir, mx_hostname) if not valid_cert(filename): return "" else: @@ -137,8 +141,8 @@ def supports_starttls(mx_host): def min_tls_version(mail_domain): protocols = [] - for mx_hostname in os.listdir(mail_domain): - filename = os.path.join(mail_domain, mx_hostname) + for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)): + filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname) contents = open(filename).read() protocol = re.findall("Protocol : (.*)", contents)[0] protocols.append(protocol) @@ -151,7 +155,7 @@ def collect(mail_domain): subsequent analysis faster. """ print "Checking domain %s" % mail_domain - mkdirp(mail_domain) + mkdirp(os.path.join(CERTS_OBSERVED, mail_domain)) answers = dns.resolver.query(mail_domain, 'MX') for rdata in answers: mx_host = str(rdata.exchange).rstrip(".") @@ -167,10 +171,6 @@ if __name__ == '__main__': for input in sys.argv[1:]: for domain in open(input).readlines(): domain = domain.strip() - if not os.path.exists(domain): - collect(domain) - if len(os.listdir(domain)) == 0: - continue suffix = check_certs(domain) min_version = min_tls_version(domain) if suffix != "": From 42c63cb6ddb4864f155a3a844f512a1add61b50f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 8 Sep 2014 16:25:40 -0400 Subject: [PATCH 070/639] More informative message for RDNS fail. --- CheckSTARTTLS.py | 11 ++++++++--- requirements.txt | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py index e8adc010e..e5c5a4323 100755 --- a/CheckSTARTTLS.py +++ b/CheckSTARTTLS.py @@ -135,8 +135,13 @@ def supports_starttls(mx_host): except socket.error as e: print "Connection to %s failed: %s" % (mx_host, e.strerror) return False - except smtplib.SMTPException: - print "No STARTTLS support on %s" % mx_host + except smtplib.SMTPException, e: + # In order to talk to some hosts, you need to run this from a host that has a + # reverse DNS entry. AWS instances all have reverse DNS, as an example. + if e[0] == 554: + print e[1] + else: + print "No STARTTLS support on %s" % mx_host, e[0] return False def min_tls_version(mail_domain): @@ -172,8 +177,8 @@ if __name__ == '__main__': for domain in open(input).readlines(): domain = domain.strip() suffix = check_certs(domain) - min_version = min_tls_version(domain) if suffix != "": + min_version = min_tls_version(domain) suffix_match = "." + suffix config["acceptable-mxs"][domain] = { "accept-mx-domains": [suffix_match] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..891e5809d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dnspython +publicsuffix +m2crypto From 622fc72dc13d4d9de2c95875e4ef98fff6895b05 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 10 Sep 2014 17:36:46 -0400 Subject: [PATCH 071/639] Treat min-tls-version as a minimum. Fixes #5. --- MTAConfigGenerator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 061b9445f..a733ab27a 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -128,7 +128,14 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.tls_policies[mx_domain] entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: - entry += " protocols=" + mx_policy["min-tls-version"] + if mx_policy["min-tls-version"].lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy["min-tls-version"].lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy["min-tls-version"].lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy["min-tls-version"] self.policy_lines.append(entry) f = open(self.policy_file, "w") From a1d016d0312c6f4a9e97ed772c0df90031bda7b5 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 13 Oct 2014 15:57:46 -0400 Subject: [PATCH 072/639] Add motivating examples to README --- PostfixLogSummary.py | 25 +++++++++++++++++++------ README.md | 8 ++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index b68c7f6c6..0348432b0 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -18,18 +18,27 @@ import ConfigParser # Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 # Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: # Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) -# +# +# Also: +# Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) def get_counts(input, config): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) + tls_deferred = collections.defaultdict(int) # Typical line looks like: # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) + # indicate a problem that should be alerted on. # ([^[]*) <--- any group of characters that is not "[" - r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + # Log lines for when a message is deferred for a TLS-related reason. These + deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS") + # Log lines for when a TLS connection was successfully established. These can + # indicate the difference between Untrusted, Trusted, and Verified certs. + connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") for line in sys.stdin: - result = r.search(line) - if result: + deferred = deferred_re.search(line) + connected = connected_re.search(line) + if connected: validation = result.group(1) mx_hostname = result.group(2).lower() if validation == "Trusted" or validation == "Verified": @@ -39,11 +48,14 @@ def get_counts(input, config): for d in address_domains: counts[d][validation] += 1 counts[d]["all"] += 1 + elif deferred: + mx_hostname = result.group(1).lower() + tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to # the set of trust roots (CA certs) configured in smtp_tls_CAfile. print "Didn't see any trusted connections. Need to install some trust roots?" - return counts + return (counts, tls_deferred) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -54,5 +66,6 @@ def print_summary(counts): if __name__ == "__main__": config = ConfigParser.Config("starttls-everywhere.json") - counts = get_counts(sys.stdin, config) + (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) + print tls_deferred diff --git a/README.md b/README.md index 3f5818255..8d6160c9a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d * Develop a fully-decentralized solution. * Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. +## Motivating examples + +* [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to + prevent negotiating an encrypted connection] + (https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) +* [Unknown party removes STARTTLS flag from all SMTP connections leaving + Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) + ## Threat model Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. From 726afb8b95f7759724c487f1163d582119333332 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:32 -0400 Subject: [PATCH 073/639] Install CAfile on config generation. --- MTAConfigGenerator.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index a733ab27a..051d9676d 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,11 +28,16 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() + if not os.access(self.postfix_cf_file, os.W_OK): + raise Exception("Can't write to %s, please re-run as root." + % self.postfix_cf_file) + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") + MTAConfigGenerator.__init__(self, policy_config) self.wrangle_existing_config() self.set_domainwise_tls_policies() + self.update_CAfile() os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): @@ -49,7 +54,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): values = map(parse_line, l) if len(set(values)) > 1: if self.fixup: - #print "Scheduling deletions:" + `values` conflicting_lines = [num for num,_var,val in values] self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) @@ -57,7 +61,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): raise ExistingConfigError, "Conflicting existing config values " + `l` val = values[0][2] if val not in acceptable: - #print "Scheduling deletions:" + `values` if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) @@ -86,6 +89,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): policy_cf_entry = "texthash:" + self.policy_file self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) self.maybe_add_config_lines() @@ -109,7 +113,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.new_cf += line self.new_cf += sep + new_cf_lines - #print self.new_cf f = open(self.fn, "w") f.write(self.new_cf) f.close() @@ -142,6 +145,10 @@ class PostfixConfigGenerator(MTAConfigGenerator): f.write("\n".join(self.policy_lines) + "\n") f.close() + def update_CAfile(self): + os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + + self.ca_file) + if __name__ == "__main__": import ConfigParser if len(sys.argv) != 3: @@ -150,3 +157,4 @@ if __name__ == "__main__": c = ConfigParser.Config(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + print "Done." From 1d47acddfdffe93f75c0d07ae0f127dd1693d984 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:44 -0400 Subject: [PATCH 074/639] Remove extraneous print of config. --- ConfigParser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ConfigParser.py b/ConfigParser.py index 2d7c88ada..d1c413f74 100755 --- a/ConfigParser.py +++ b/ConfigParser.py @@ -90,7 +90,6 @@ class Config: # XXX is it ever permissible to have a domain with an acceptable-mx # that does not point to a TLS security policy? If not, check/warn/fail # here - print self.tls_policies def get_address_domains(self, mx_hostname): labels = mx_hostname.split(".") From 828c00b758063172990073bd4b667b61c1a30699 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Oct 2014 15:28:49 -0400 Subject: [PATCH 075/639] Find deferred lines in log summary. Also add a cron mode that only emits output if there's something wrong. --- PostfixLogSummary.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 0348432b0..9b2f9c3b1 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -2,6 +2,7 @@ import re import sys import collections +import argparse import ConfigParser @@ -39,8 +40,8 @@ def get_counts(input, config): deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: - validation = result.group(1) - mx_hostname = result.group(2).lower() + validation = connected.group(1) + mx_hostname = connected.group(2).lower() if validation == "Trusted" or validation == "Verified": seen_trusted = True address_domains = config.get_address_domains(mx_hostname) @@ -49,13 +50,13 @@ def get_counts(input, config): counts[d][validation] += 1 counts[d]["all"] += 1 elif deferred: - mx_hostname = result.group(1).lower() + mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to # the set of trust roots (CA certs) configured in smtp_tls_CAfile. print "Didn't see any trusted connections. Need to install some trust roots?" - return (counts, tls_deferred) + return (counts, tls_deferred, seen_trusted) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -65,7 +66,19 @@ def print_summary(counts): print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": + arg_parser = argparse.ArgumentParser(description='This is a PyMOTW sample program') + arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) + args = arg_parser.parse_args() + config = ConfigParser.Config("starttls-everywhere.json") - (counts, tls_deferred) = get_counts(sys.stdin, config) - print_summary(counts) - print tls_deferred + (counts, tls_deferred, seen_trusted) = get_counts(sys.stdin, config) + + # If not running in cron, print an overall summary of log lines seen from known hosts. + if not args.cron: + print_summary(counts) + if not seen_trusted: + print 'No Trusted connections seen! Probably need to install a CAfile.' + + if len(tls_deferred) > 0: + print "Some mail was deferred due to TLS problems:" + print tls_deferred From f04e8259a9aef0e6547fe22884268581461512e2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 20 Oct 2014 09:29:22 -0400 Subject: [PATCH 076/639] Protocols separated by colons, not commas. --- MTAConfigGenerator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 051d9676d..5ca8f5aee 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -132,11 +132,11 @@ class PostfixConfigGenerator(MTAConfigGenerator): entry = address_domain + " encrypt" if "min-tls-version" in mx_policy: if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" + entry += " protocols=!SSLv2:!SSLv3" elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1" elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: print mx_policy["min-tls-version"] self.policy_lines.append(entry) From 613a8f5e8877878f3d0b169a045b0f6b2b31adec Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 20 Oct 2014 16:15:13 -0400 Subject: [PATCH 077/639] Improve cronjob operation to only process diffs. --- PostfixLogSummary.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 9b2f9c3b1..be130d958 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -1,11 +1,15 @@ #!/usr/bin/python2.7 +import argparse +import collections +import os import re import sys -import collections -import argparse +import time import ConfigParser +TIME_FORMAT = "%b %d %H:%M:%S" + # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -22,7 +26,7 @@ import ConfigParser # # Also: # Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) -def get_counts(input, config): +def get_counts(input, config, earliest_timestamp): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) @@ -32,11 +36,15 @@ def get_counts(input, config): # indicate a problem that should be alerted on. # ([^[]*) <--- any group of characters that is not "[" # Log lines for when a message is deferred for a TLS-related reason. These - deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS") + deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") # Log lines for when a TLS connection was successfully established. These can # indicate the difference between Untrusted, Trusted, and Verified certs. connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + timestamp = 0 for line in sys.stdin: + timestamp = time.strptime(line[0:15], TIME_FORMAT) + if timestamp < earliest_timestamp: + continue deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: @@ -52,11 +60,7 @@ def get_counts(input, config): elif deferred: mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 - if not seen_trusted: - # Postfix will only emit 'Trusted' if the certificate validates according to - # the set of trust roots (CA certs) configured in smtp_tls_CAfile. - print "Didn't see any trusted connections. Need to install some trust roots?" - return (counts, tls_deferred, seen_trusted) + return (counts, tls_deferred, seen_trusted, timestamp) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -71,7 +75,14 @@ if __name__ == "__main__": args = arg_parser.parse_args() config = ConfigParser.Config("starttls-everywhere.json") - (counts, tls_deferred, seen_trusted) = get_counts(sys.stdin, config) + + last_timestamp_processed = 0 + timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' + if os.path.isfile(timestamp_file): + last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) + (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) + with open(timestamp_file, "w") as f: + f.write(time.strftime(TIME_FORMAT, latest_timestamp)) # If not running in cron, print an overall summary of log lines seen from known hosts. if not args.cron: @@ -81,4 +92,5 @@ if __name__ == "__main__": if len(tls_deferred) > 0: print "Some mail was deferred due to TLS problems:" - print tls_deferred + for (k, v) in tls_deferred.iteritems(): + print "%s: %s" % (k, v) From fe17c873c0e7bd0f1e6debdb5cc1e6df11bcb0a3 Mon Sep 17 00:00:00 2001 From: pypoet Date: Tue, 13 Oct 2015 17:09:03 -0400 Subject: [PATCH 078/639] Initial re-vamp of the Config object to centralize validation and lay the basis for making compositions of configs and overrides. Lots of TODOs, be warned. --- Config.py | 315 ++++++++++++++++++++++++++++++++++++++++ bigger_test_config.json | 36 +++++ 2 files changed, 351 insertions(+) create mode 100644 Config.py create mode 100644 bigger_test_config.json diff --git a/Config.py b/Config.py new file mode 100644 index 000000000..157c2c2e4 --- /dev/null +++ b/Config.py @@ -0,0 +1,315 @@ +from datetime import datetime +import json + + +"""Idea here being to start with something that is decomposed so it's easier to +make do json in *and* out, differences between configs and config extension. +""" + +def parse_bool_from_json(value, attr_name): + if value in ('true', '1', 1, 'yes'): + bool_value = True + elif value in ('false', '0', 0, 'no'): + bool_value = False + elif value in (True, False): + bool_value = value + else: + raise ValueError('Config value %s is an invalid boolean value.' % attr_name) + return bool_value + + +def parse_timestamp(value, attr_name): + #TODO support full extended timestamp "2014-06-06T14:30:16+00:00" as well + if isinstance(value, datetime): + dt = value + else: + try: + ts = int(value) + dt = datetime.fromtimestamp(ts) + except: + raise ValueError('Config value %s is an invalid timestamp integer.' % attr_name) + return dt + + +def verify_member_of(value, member_list, attr_name): + if value not in member_list: + raise ValueError('Config value "%s" must be one of (%s)' % ( + attr_name, ', '.join(member_list)) + ) + return value + + +def verify_string(value, attr_name, max_length=200): + if not isinstance(value, (str, unicode)): + raise TypeError('Config value %s must be a string.' % attr_name) + if len(value) > max_length: + raise ValueError('Config value %s is too long.' % attr_name) + return value + + +class Config(object): + """Config container for StartTLS Everywhere configuration. + + Intended as a simple container that unifies where validatation occurs, + and is capable of comparing configs to warn of things like changing + certificate fingerprints from one scan to the next. + + There is a one to one mapping of the object attributes to the JSON + object keys, albeit with dashes replaced with underscores. + """ + + def __init__(self): + # container for validated properties with JSON names + self._data = {} + + self.tls_policies = [] + self.acceptable_mxs = [] + + def __add__(self, other_config): + """Allow addition but not really of *full* configs, need to flesh that out.""" + #TODO add this + raise NotImplemented + + def __repr__(self): + #TODO fix this generically, and maybe put it in the inheritence tree + s = '' % (self._data.iteritems()) + return s + + def update(self, other_config): + """Update properties of config from a 'newer' config and force verification.""" + #TODO add this + raise NotImplemented + + def load_from_json_file(self, json_filename, f_open=open): + #TODO add robust catching and checking + # try: + with f_open(json_filename, 'r') as f: + json_str = f.read() + json_dict = json.loads(json_str) + # except oserr + # except json parse err + self.from_json_dict(json_dict) + + def from_json_dict(self, json_dict): + """Assign JSON data to Config properties and declare sub-objects. + + Let's property verification methods do the heavy lifting and mostly + maps between the JSON config names and attributes. Keeps track of + unused variables and warns about them. + """ + for key, val in json_dict.iteritems(): + if key == 'author': + self.author = val + elif key == 'comment': + self.comment = val + elif key == 'expires': + self.expires = val + elif key == 'timestamp': + self.timestamp = val + elif key == 'tls-policies': + self.tls_policies = self.make_tls_policy_dict(val) + elif key == 'acceptable-mxs': + self.acceptable_mxs = self.make_acceptable_mxs_dict(val) + else: + #TODO log warning + print 'Unknown attribute "%s", skipping' % key + + def to_json(self): + #TODO implement output and make sure it can be re-input with identical results + raise NotImplemented + + @property + def author(self): + return self._data.get('author') + + @author.setter + def author(self, value): + self._data['author'] = verify_string(value, 'author') + + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + + @property + def expires(self): + return self._data.get('expires') + + @expires.setter + def expires(self, value): + self._data['expires'] = parse_timestamp(value, 'expires') + + @property + def timestamp(self): + return self._data.get('timestamp') + + @timestamp.setter + def timestamp(self, value): + self._data['timestamp'] = parse_timestamp(value, 'timestamp') + + def make_tls_policy_dict(self, policy_dict): + tls_policy_dict = {} + for domain_suffix, settings in policy_dict.iteritems(): + new_domain_policy = TLSPolicy(domain_suffix) + #TODO define config errs and use + #try + new_domain_policy.from_json_dict(settings) + #except config err + tls_policy_dict[domain_suffix] = new_domain_policy + return tls_policy_dict + + def make_acceptable_mxs_dict(self, mxs_dict): + acceptable_mxs_dict = {} + for domain, settings in mxs_dict.iteritems(): + new_domain_policy = AcceptableMX(domain) + #TODO define config errs and use + #try + new_domain_policy.from_json_dict(settings) + #except config err + acceptable_mxs_dict[domain] = new_domain_policy + return acceptable_mxs_dict + + def is_valid(self): + #TODO implement with checks to make sure domains don't overlap + # and every acceptable mx has a tls policy, etc. + raise NotImplemented + + +class TLSPolicy(object): + + ENFORCE_MODES = ('enforce', 'log-only') + TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') + + def __init__(self, domain_suffix): + # container for validated properties with JSON names + self._data = {} + self.domain_suffix = domain_suffix + + #TODO add me + self.accept_spki_hashs = None + #TODO add me + self.error_notification = None + + def from_json_dict(self, json_dict): + for key, val in json_dict.iteritems(): + if key == 'comment': + self.comment = val + elif key == 'enforce-mode': + self.enforce_mode = val + elif key == 'min-tls-version': + self.min_tls_version = val + elif key == 'require-tls': + self.require_tls = val + elif key == 'require-valid-certificate': + self.require_valid_certificate = val + else: + #TODO wat, log this instead + print 'Unknown key %s' % key + + def is_valid(self): + """Do simple check that config contains all required values. + + Should find a way to expose easily which config values + are required, at least place in error messages such that + incomplete configs will expose it. + """ + required_attrs = ('enforce-mode', 'min-tls-version', + 'require-tls') + values_set = [self._data.get(attr) for attr in required_attrs] + if not all(values_set): + return False + else: + return True + + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + + @property + def enforce_mode(self): + return self._data.get('enforce-mode') + + @enforce_mode.setter + def enforce_mode(self, value): + self._data['enforce-mode'] = verify_member_of(value, self.ENFORCE_MODES, 'enforce-mode') + + @property + def min_tls_version(self): + return self._data.get('min-tls-version') + + @min_tls_version.setter + def min_tls_version(self, value): + """Should this be dealing only with strings processed by map ... lower()?""" + tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] + tls_versions.extend(self.TLS_VERSIONS) + self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') + + @property + def require_tls(self): + return self._data.get('require-tls') + + @require_tls.setter + def require_tls(self, value): + self._data['require-tls'] = parse_bool_from_json(value, 'require-tls') + + @property + def require_valid_certificate(self): + return self._data.get('require-valid-certificate') + + @require_valid_certificate.setter + def require_valid_certificate(self, value): + self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') + + +class AcceptableMX(object): + """Holds acceptable MX domain suffixes for a single mail serving domain. + + Such as for gmail.com that single mail serving suffix domain is: + gmail-smtp-in.l.google.com. + + Configuration of the acceptable MX suffix domains must match up with TLS policies + for the suffix domains. + """ + def __init__(self, domain): + self.domain = domain + # container for validated properties with JSON names + self._data = {} + self._data['accept-mx-domains'] = [] + + def add_acceptable_mx(self, domain_suffix): + unique_domain_suffixes = set(self._data['accept-mx-domains']) + unique_domain_suffixes.add(domain_suffix) + self._data['accept-mx-domains'] = list(unique_domain_suffixes) + + def is_valid(self): + """Check to make sure there is one acceptable domain suffix. + + This will need to be updated once we can actually test and support + for more than one acceptable domain suffix. + + TODO: could make this object double check the data it is given with + DNS queries. + """ + if len(self._data['accept-mx-domains']) != 1: + return False + else: + return True + + def from_json_dict(self, json_dict): + for key, val in json_dict.iteritems(): + if key == 'accept-mx-domains': + if isinstance(val, list): + for domain_suffix in val: + self.add_acceptable_mx(domain_suffix) + else: + self.add_acceptable_mx(val) + else: + #TODO add logging for this + print 'warning: unknown key %s' % key diff --git a/bigger_test_config.json b/bigger_test_config.json new file mode 100644 index 000000000..e0697fc85 --- /dev/null +++ b/bigger_test_config.json @@ -0,0 +1,36 @@ +{ + "timestamp": 1401414363, + "author": "Electronic Frontier Foundation https://eff.org", + "expires": 1404242424, + "tls-policies": { + ".yahoodns.net": { + "require-valid-certificate": true + }, + ".eff.org": { + "require-tls": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "enforce", + "accept-spki-hashes": [ + "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", + "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" + ] + }, + ".google.com": { + "require-valid-certificate": true, + "min-tls-version": "TLSv1.1", + "enforce-mode": "log-only", + "error-notification": "https://google.com/post/reports/here" + } + }, + "acceptable-mxs": { + "yahoo.com": { + "accept-mx-domains": [".yahoodns.net"] + }, + "gmail.com": { + "accept-mx-domains": [".google.com"] + }, + "eff.org": { + "accept-mx-domains": [".eff.org"] + } + } +} From 147f58bdbc5cc08c7abe43d304df6efd2ad86fe2 Mon Sep 17 00:00:00 2001 From: pypoet Date: Wed, 14 Oct 2015 02:49:34 -0400 Subject: [PATCH 079/639] Rounds out missing features and is now on par with ConfigParser.py. Still missing logging, composibility and a couple of attributes. --- Config.py | 165 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 64 deletions(-) diff --git a/Config.py b/Config.py index 157c2c2e4..93504dd5a 100644 --- a/Config.py +++ b/Config.py @@ -1,11 +1,14 @@ from datetime import datetime +from dateutil import parser import json +import pprint """Idea here being to start with something that is decomposed so it's easier to make do json in *and* out, differences between configs and config extension. """ + def parse_bool_from_json(value, attr_name): if value in ('true', '1', 1, 'yes'): bool_value = True @@ -14,26 +17,27 @@ def parse_bool_from_json(value, attr_name): elif value in (True, False): bool_value = value else: - raise ValueError('Config value %s is an invalid boolean value.' % attr_name) + raise ConfigError('Config value %s is an invalid boolean value.' % attr_name) return bool_value def parse_timestamp(value, attr_name): - #TODO support full extended timestamp "2014-06-06T14:30:16+00:00" as well if isinstance(value, datetime): - dt = value - else: - try: - ts = int(value) - dt = datetime.fromtimestamp(ts) - except: - raise ValueError('Config value %s is an invalid timestamp integer.' % attr_name) - return dt + return value + try: + ts = int(value) + return datetime.fromtimestamp(ts) + except (TypeError, ValueError): + pass + try: + return parser.parse(value) + except (TypeError, ValueError): + raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) def verify_member_of(value, member_list, attr_name): if value not in member_list: - raise ValueError('Config value "%s" must be one of (%s)' % ( + raise ConfigError('Config value "%s" must be one of (%s)' % ( attr_name, ', '.join(member_list)) ) return value @@ -41,13 +45,67 @@ def verify_member_of(value, member_list, attr_name): def verify_string(value, attr_name, max_length=200): if not isinstance(value, (str, unicode)): - raise TypeError('Config value %s must be a string.' % attr_name) + raise ConfigError('Config value %s must be a string.' % attr_name) if len(value) > max_length: - raise ValueError('Config value %s is too long.' % attr_name) + raise ConfigError('Config value %s is too long.' % attr_name) return value -class Config(object): +def to_dict(config_dict): + """Cleans up BaseConfig children to be serialized.""" + d = {} + for key, val in config_dict.iteritems(): + if isinstance(val, BaseConfig): + d[key] = to_dict(val._data) + elif isinstance(val, datetime): + d[key] = val.strftime('%Y-%m-%dT%H:%M:%S%z') + elif isinstance(val, dict): + d[key] = to_dict(val) + else: + d[key] = val + return d + + +class BaseConfig(object): + """Top level config class for common methods.""" + + def __init__(self): + # container for validated properties with JSON names + self._data = {} + + def __repr__(self): + s = '< %s %s >' % (self.__class__.__name__, + pprint.pformat(self._data)) + return s + + def to_json(self): + d = to_dict(self._data) + return json.dumps(d) + + def write_to_json_file(self, json_filename, f_open=open): + data = self.to_json() + try: + with f_open(json_filename, 'w') as f: + f.write(data) + except IOError: + raise + + def load_from_json_file(self, json_filename, f_open=open): + try: + with f_open(json_filename, 'r') as f: + json_str = f.read() + json_dict = json.loads(json_str) + except IOError: + raise + except ValueError: + raise ConfigError('No valid JSON found in file: %s' % json_filename) + self.from_json_dict(json_dict) + + def from_json_dict(self, json_dict): + raise NotImplmented('BaseConfig should not be populated.') + + +class Config(BaseConfig): """Config container for StartTLS Everywhere configuration. Intended as a simple container that unifies where validatation occurs, @@ -59,37 +117,20 @@ class Config(object): """ def __init__(self): - # container for validated properties with JSON names - self._data = {} - - self.tls_policies = [] - self.acceptable_mxs = [] + super(self.__class__, self).__init__() + self._data['tls-policies'] = {} + self._data['acceptable-mxs'] = {} def __add__(self, other_config): """Allow addition but not really of *full* configs, need to flesh that out.""" #TODO add this raise NotImplemented - def __repr__(self): - #TODO fix this generically, and maybe put it in the inheritence tree - s = '' % (self._data.iteritems()) - return s - def update(self, other_config): """Update properties of config from a 'newer' config and force verification.""" #TODO add this raise NotImplemented - def load_from_json_file(self, json_filename, f_open=open): - #TODO add robust catching and checking - # try: - with f_open(json_filename, 'r') as f: - json_str = f.read() - json_dict = json.loads(json_str) - # except oserr - # except json parse err - self.from_json_dict(json_dict) - def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -107,17 +148,13 @@ class Config(object): elif key == 'timestamp': self.timestamp = val elif key == 'tls-policies': - self.tls_policies = self.make_tls_policy_dict(val) + self.make_tls_policy_dict(val) elif key == 'acceptable-mxs': - self.acceptable_mxs = self.make_acceptable_mxs_dict(val) + self.make_acceptable_mxs_dict(val) else: #TODO log warning print 'Unknown attribute "%s", skipping' % key - def to_json(self): - #TODO implement output and make sure it can be re-input with identical results - raise NotImplemented - @property def author(self): return self._data.get('author') @@ -151,47 +188,42 @@ class Config(object): self._data['timestamp'] = parse_timestamp(value, 'timestamp') def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = {} + tls_policy_dict = self._data['tls-policies'] for domain_suffix, settings in policy_dict.iteritems(): new_domain_policy = TLSPolicy(domain_suffix) - #TODO define config errs and use - #try - new_domain_policy.from_json_dict(settings) - #except config err + try: + new_domain_policy.from_json_dict(settings) + except ConfigError as e: + raise tls_policy_dict[domain_suffix] = new_domain_policy - return tls_policy_dict def make_acceptable_mxs_dict(self, mxs_dict): - acceptable_mxs_dict = {} + acceptable_mxs_dict = self._data['acceptable-mxs'] for domain, settings in mxs_dict.iteritems(): new_domain_policy = AcceptableMX(domain) - #TODO define config errs and use - #try - new_domain_policy.from_json_dict(settings) - #except config err + try: + new_domain_policy.from_json_dict(settings) + except ConfigError as e: + raise acceptable_mxs_dict[domain] = new_domain_policy - return acceptable_mxs_dict def is_valid(self): #TODO implement with checks to make sure domains don't overlap # and every acceptable mx has a tls policy, etc. raise NotImplemented + - -class TLSPolicy(object): +class TLSPolicy(BaseConfig): ENFORCE_MODES = ('enforce', 'log-only') TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') def __init__(self, domain_suffix): - # container for validated properties with JSON names - self._data = {} + super(self.__class__, self).__init__() self.domain_suffix = domain_suffix - - #TODO add me - self.accept_spki_hashs = None - #TODO add me - self.error_notification = None + #TODO add support for two designed but yet unsupported attrs + # self._data['accept-spki-hashs'] = None + # self._data['error-notification'] = None def from_json_dict(self, json_dict): for key, val in json_dict.iteritems(): @@ -268,7 +300,7 @@ class TLSPolicy(object): self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') -class AcceptableMX(object): +class AcceptableMX(BaseConfig): """Holds acceptable MX domain suffixes for a single mail serving domain. Such as for gmail.com that single mail serving suffix domain is: @@ -278,9 +310,8 @@ class AcceptableMX(object): for the suffix domains. """ def __init__(self, domain): + super(self.__class__, self).__init__() self.domain = domain - # container for validated properties with JSON names - self._data = {} self._data['accept-mx-domains'] = [] def add_acceptable_mx(self, domain_suffix): @@ -313,3 +344,9 @@ class AcceptableMX(object): else: #TODO add logging for this print 'warning: unknown key %s' % key + + +class ConfigError(ValueError): + def __init__(self, message): + super(self.__class__, self).__init__(message) + From 6da5de6b19d3247c49290631dd4557a05f1e230b Mon Sep 17 00:00:00 2001 From: pypoet Date: Fri, 16 Oct 2015 00:57:42 -0400 Subject: [PATCH 080/639] Beginnings of generic config composibility in place. --- Config.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/Config.py b/Config.py index 93504dd5a..740de35c0 100644 --- a/Config.py +++ b/Config.py @@ -67,7 +67,18 @@ def to_dict(config_dict): class BaseConfig(object): - """Top level config class for common methods.""" + """Top level config class for common methods. + + Requirements for using class: + - list all properties with getters *and* setters in class + variable 'config_properties' + - __init__ of child classes must be callable with *only* + keyword arguments to allow method calls to update to create + a new config + ... more ... + """ + + config_properties = [] def __init__(self): # container for validated properties with JSON names @@ -78,6 +89,27 @@ class BaseConfig(object): pprint.pformat(self._data)) return s + def update(self, newer_config, merge=False, **kwargs): + fresh_config = self.__class__(**kwargs) + if not isinstance(newer_config, self.__class__): + raise ConfigError('Attempting to update a %s with a %s' % ( + self.__class__, + newer_config.__class__)) + for prop_name in self.config_properties: + prop = self.__class__.__dict__.get(prop_name) + assert prop + new_value = prop.fget(newer_config) + old_value = prop.fget(self) + if new_value is not None: + prop.fset(fresh_config, new_value) + elif merge and old_value is not None: + prop.fset(fresh_config, old_value) + return fresh_config + + def merge(self, newer_config, **kwargs): + kwargs['merge'] = True + return self.update(newer_config, **kwargs) + def to_json(self): d = to_dict(self._data) return json.dumps(d) @@ -129,7 +161,10 @@ class Config(BaseConfig): def update(self, other_config): """Update properties of config from a 'newer' config and force verification.""" #TODO add this + new_config = Config() raise NotImplemented + + def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -187,6 +222,14 @@ class Config(BaseConfig): def timestamp(self, value): self._data['timestamp'] = parse_timestamp(value, 'timestamp') + @property + def tls_policies(self): + return self._data.get('tls-policies') + + @property + def acceptable_mxs(self): + return self._data.get('acceptable-mxs') + def make_tls_policy_dict(self, policy_dict): tls_policy_dict = self._data['tls-policies'] for domain_suffix, settings in policy_dict.iteritems(): @@ -208,9 +251,19 @@ class Config(BaseConfig): acceptable_mxs_dict[domain] = new_domain_policy def is_valid(self): - #TODO implement with checks to make sure domains don't overlap - # and every acceptable mx has a tls policy, etc. - raise NotImplemented + #TODO implement checks to make sure domains don't overlap + #TODO add debug logging for troubleshooting stake + for mx_config in self.acceptable_mxs.values(): + if not mx_config.is_valid(): + return False + for domain_suffix in mx_config.accept_mx_domains: + # check to make sure every accepted MX has a TLS policy + if not domain_suffix in self.tls_policies: + return False + for tls_config in self.tls_policies.values(): + if not tls_config.is_valid(): + return False + return True class TLSPolicy(BaseConfig): @@ -218,7 +271,10 @@ class TLSPolicy(BaseConfig): ENFORCE_MODES = ('enforce', 'log-only') TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') - def __init__(self, domain_suffix): + config_properties = ['comment', 'enforce_mode', 'min_tls_version', + 'require_tls', 'require_valid_certificate'] + + def __init__(self, domain_suffix=None): super(self.__class__, self).__init__() self.domain_suffix = domain_suffix #TODO add support for two designed but yet unsupported attrs @@ -256,6 +312,12 @@ class TLSPolicy(BaseConfig): else: return True + def update(self, newer_policy, **kwargs): + fresh_policy = super(self.__class__, self).update(newer_policy, + domain_suffix=self.domain_suffix) + fresh_policy.domain_suffix = self.domain_suffix + return fresh_policy + @property def comment(self): return self._data.get('comment') @@ -278,7 +340,7 @@ class TLSPolicy(BaseConfig): @min_tls_version.setter def min_tls_version(self, value): - """Should this be dealing only with strings processed by map ... lower()?""" + """TODO: Should this be dealing only with strings processed by map ... lower()?""" tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] tls_versions.extend(self.TLS_VERSIONS) self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') @@ -309,11 +371,15 @@ class AcceptableMX(BaseConfig): Configuration of the acceptable MX suffix domains must match up with TLS policies for the suffix domains. """ - def __init__(self, domain): + def __init__(self, domain=None): super(self.__class__, self).__init__() self.domain = domain self._data['accept-mx-domains'] = [] + @property + def accept_mx_domains(self): + return self._data.get('accept-mx-domains') + def add_acceptable_mx(self, domain_suffix): unique_domain_suffixes = set(self._data['accept-mx-domains']) unique_domain_suffixes.add(domain_suffix) From 9a71b18b851c58bd7cb7c6a3fdfdaee23bdc3d94 Mon Sep 17 00:00:00 2001 From: pypoet Date: Fri, 23 Oct 2015 18:26:26 -0700 Subject: [PATCH 081/639] Fix updates and merges, add testing to make sure they stay fixed. --- Config.py | 65 ++++++++++++++++++++++++++++++++++++++------- TestConfig.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 TestConfig.py diff --git a/Config.py b/Config.py index 740de35c0..9cfbad3f0 100644 --- a/Config.py +++ b/Config.py @@ -1,6 +1,7 @@ from datetime import datetime from dateutil import parser import json +import logging import pprint @@ -8,6 +9,10 @@ import pprint make do json in *and* out, differences between configs and config extension. """ +#TODO scope logging and handlers better, control verbosity by command line flags +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + def parse_bool_from_json(value, attr_name): if value in ('true', '1', 1, 'yes'): @@ -90,7 +95,11 @@ class BaseConfig(object): return s def update(self, newer_config, merge=False, **kwargs): + # removed 'merge' kw arg - and it was passed to constructor + # make a note to not do that, consume it on the param list fresh_config = self.__class__(**kwargs) + logger.debug('from parent update kwargs %s' % kwargs) + logger.debug('from parent update merge %s' % merge) if not isinstance(newer_config, self.__class__): raise ConfigError('Attempting to update a %s with a %s' % ( self.__class__, @@ -108,6 +117,7 @@ class BaseConfig(object): def merge(self, newer_config, **kwargs): kwargs['merge'] = True + logger.debug('from parent merge: %s' % kwargs) return self.update(newer_config, **kwargs) def to_json(self): @@ -163,8 +173,6 @@ class Config(BaseConfig): #TODO add this new_config = Config() raise NotImplemented - - def from_json_dict(self, json_dict): """Assign JSON data to Config properties and declare sub-objects. @@ -187,8 +195,7 @@ class Config(BaseConfig): elif key == 'acceptable-mxs': self.make_acceptable_mxs_dict(val) else: - #TODO log warning - print 'Unknown attribute "%s", skipping' % key + logger.warn('Unknown attribute "%s", skipping' % key) @property def author(self): @@ -294,8 +301,7 @@ class TLSPolicy(BaseConfig): elif key == 'require-valid-certificate': self.require_valid_certificate = val else: - #TODO wat, log this instead - print 'Unknown key %s' % key + logger.warn('Unknown key %s' % key) def is_valid(self): """Do simple check that config contains all required values. @@ -313,9 +319,17 @@ class TLSPolicy(BaseConfig): return True def update(self, newer_policy, **kwargs): + if not kwargs.get('domain_suffix'): + kwargs['domain_suffix'] = self.domain_suffix fresh_policy = super(self.__class__, self).update(newer_policy, - domain_suffix=self.domain_suffix) - fresh_policy.domain_suffix = self.domain_suffix + **kwargs) + logger.debug('from TLS child update %s' % kwargs) + return fresh_policy + + def merge(self, newer_policy, **kwargs): + logger.debug('from TLS child merge: %s' % kwargs) + fresh_policy = super(self.__class__, self).merge(newer_policy, + domain_suffix=self.domain_suffix) return fresh_policy @property @@ -385,6 +399,14 @@ class AcceptableMX(BaseConfig): unique_domain_suffixes.add(domain_suffix) self._data['accept-mx-domains'] = list(unique_domain_suffixes) + @property + def comment(self): + return self._data.get('comment') + + @comment.setter + def comment(self, value): + self._data['comment'] = verify_string(value, 'comment') + def is_valid(self): """Check to make sure there is one acceptable domain suffix. @@ -407,9 +429,32 @@ class AcceptableMX(BaseConfig): self.add_acceptable_mx(domain_suffix) else: self.add_acceptable_mx(val) + elif key == 'comment': + self.comment = val else: - #TODO add logging for this - print 'warning: unknown key %s' % key + logger.warn('warning: unknown key %s' % key) + + def update(self, newer_policy, **kwargs): + logger.debug('from MX child update got %s' % kwargs) + if not kwargs.get('domain'): + kwargs['domain'] = self.domain + fresh_policy = super(self.__class__, self).update(newer_policy, + **kwargs) + if kwargs.get('merge'): + new_accepted_mxs = set(self.accept_mx_domains) + new_accepted_mxs = new_accepted_mxs.union(newer_policy.accept_mx_domains) + else: + new_accepted_mxs = newer_policy.accept_mx_domains + for domain in new_accepted_mxs: + fresh_policy.add_acceptable_mx(domain) + + return fresh_policy + + def merge(self, newer_policy, **kwargs): + logger.debug('from MX child merge: %s' % kwargs) + fresh_policy = super(self.__class__, self).merge(newer_policy, + **kwargs) + return fresh_policy class ConfigError(ValueError): diff --git a/TestConfig.py b/TestConfig.py new file mode 100644 index 000000000..043eb2dca --- /dev/null +++ b/TestConfig.py @@ -0,0 +1,73 @@ +import copy +import logging +import unittest + +import Config + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + + +class TestTLSPolicy(unittest.TestCase): + + def setUp(self): + self.old_config = Config.TLSPolicy(domain_suffix='.eff.org') + self.old_config.comment = 'Testing EFF.org TLS policy' + self.old_config.require_tls = True + self.old_config.require_valid_certificate = False + self.old_config.min_tls_version = 'TLSv1' + self.old_config.enforce_mode = 'log-only' + + self.new_config = Config.TLSPolicy(domain_suffix='.eff.org') + self.new_config.require_valid_certificate = True + self.new_config.min_tls_version = 'TLSv1.2' + self.new_config.enforce_mode = 'enforce' + + def testUpdateDropsOldSettings(self): + logger.debug('old: %s' % self.old_config) + logger.debug('new: %s' % self.new_config) + tls_policy = self.old_config.update(self.new_config) + logger.debug('just generated: %s' % tls_policy) + self.assertFalse(any([tls_policy.require_tls, tls_policy.comment])) + + def testMergeKeepsOldSettings(self): + logger.debug('old: %s' % self.old_config) + logger.debug('new: %s' % self.new_config) + tls_policy = self.old_config.merge(self.new_config, merge=True) + logger.debug('just generated: %s' % tls_policy) + self.assertTrue(all([tls_policy.require_tls, tls_policy.comment])) + + def testUpdateGetsNameSet(self): + tls_policy = self.old_config.update(self.new_config) + self.assertEquals(tls_policy.domain_suffix, self.old_config.domain_suffix) + + +class TestAcceptableMX(unittest.TestCase): + + def setUp(self): + self.old_config = Config.AcceptableMX(domain='eff.org') + self.old_config.add_acceptable_mx('.eff.org') + + def testUpdateDropsOldMXs(self): + new_bogus_mx = '.testing.eff.org' + new_config = Config.AcceptableMX(domain='eff.org') + new_config.add_acceptable_mx(new_bogus_mx) + updated_config = self.old_config.update(new_config) + self.assertNotIn('.eff.org', updated_config.accept_mx_domains) + + def testMergeKeepsOldMXs(self): + new_bogus_mx = '.testing.eff.org' + new_config = Config.AcceptableMX(domain='eff.org') + new_config.add_acceptable_mx(new_bogus_mx) + updated_config = self.old_config.merge(new_config) + self.assertListEqual(sorted(['.eff.org', '.testing.eff.org']), + sorted(updated_config.accept_mx_domains)) + + def testUpdateGetsNameSet(self): + new_policy = Config.AcceptableMX(domain=self.old_config.domain) + mx_policy = self.old_config.update(new_policy) + self.assertEquals(mx_policy.domain, self.old_config.domain) + + +if __name__ == '__main__': + unittest.main() From c87b5d6a7834553379b8f1a105adb35685499b96 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Thu, 21 Jan 2016 00:56:29 -0800 Subject: [PATCH 082/639] Hook the MTA config generation into the new config container. --- Config.py | 77 ++++++++++++++++++++++++++++++++++++++--- MTAConfigGenerator.py | 31 +++++++++-------- PostfixLogSummary.py | 24 ++++++++----- TestConfig.py | 58 +++++++++++++++++++++++++++++++ bigger_test_config.json | 2 +- 5 files changed, 164 insertions(+), 28 deletions(-) diff --git a/Config.py b/Config.py index 9cfbad3f0..eb4f31dba 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,6 @@ from datetime import datetime from dateutil import parser +import collections import json import logging import pprint @@ -238,7 +239,7 @@ class Config(BaseConfig): return self._data.get('acceptable-mxs') def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = self._data['tls-policies'] + tls_policy_dict = self.tls_policies for domain_suffix, settings in policy_dict.iteritems(): new_domain_policy = TLSPolicy(domain_suffix) try: @@ -247,6 +248,9 @@ class Config(BaseConfig): raise tls_policy_dict[domain_suffix] = new_domain_policy + def get_tls_policy(self, mx_domain): + return self.tls_policies.get(mx_domain) + def make_acceptable_mxs_dict(self, mxs_dict): acceptable_mxs_dict = self._data['acceptable-mxs'] for domain, settings in mxs_dict.iteritems(): @@ -257,9 +261,71 @@ class Config(BaseConfig): raise acceptable_mxs_dict[domain] = new_domain_policy + def get_address_domains(self, mx_hostname, mx_to_domain_map): + """Do a fuzzy DNS host match on provided map to get lists of policies. + + Args: + mx_hostname (string): The hostname from an MX record. + mx_to_domain_map: Mapping from MX hosts to AcceptableMX + policies, the same AcceptableMX policy may occur more + than once. e.g. {'mx_host3': set(AcceptableMX, ...)} + The map can be generated by Config.get_mx_to_domain_policy_map. + + Returns: + The set containing all AcceptableMX policies that list the + provided MX host as viable. + """ + labels = mx_hostname.split(".") + for n in range(1, len(labels)): + parent = "." + ".".join(labels[n:]) + if parent in mx_to_domains_map: + return mx_to_domain_map[parent] + return None + + def get_mx_to_domain_policy_map(self): + """Create mapping of MX hostnames to sets of AcceptableMX policies. + + Generate a dictionary that is typically used in log analysis + (e.g. if your MTA logs interact with beta.innotech.com you use + this mapping to tell you it used the innotech.com AcceptableMX + policy or policies). There are of course complications. + """ + # create reverse mapping dictionary as well for auditing + # and reviewing logs + mx_to_domain_policy = collections.defaultdict(set) + + for mx_host, domain_policy in self.get_all_mx_items(): + existing_mx_policies = mx_to_domain_policy.get(mx_host) + if existing_mx_policies: + existing_domains = [ e.domain for e in existing_mx_policies ] + if domain_policy.domain not in existing_domains: + #TODO plenty of room to enforce a security policy here + # this is also the case of google apps personal domains + msg = ('Attempting to add domain policy (%s) for MX host but MX' + ' host already has a domain policy (%s), appending...') + logger.debug(msg % (domain_policy.domain, + ', '.join(existing_domains))) + mx_to_domain_policy[mx_host].add(domain_policy) + return mx_to_domain_policy + + def get_all_mx_items(self): + """Iterate over (mx_host, mx_policy) - be sure to dedup!""" + all_mx_items = [] + for policy in self.acceptable_mxs.values(): + accepted_mxs = policy.accept_mx_domains + all_mx_items.extend([(mx_host, policy) + for mx_host in accepted_mxs]) + return all_mx_items + + def get_all_mx_hosts(self): + all_mx_hosts = [] + [ all_mx_hosts.extend(domain_policy.acceptable_mxs) + for domain_policy in self.acceptable_mxs.values() ] + return all_mx_hosts + def is_valid(self): #TODO implement checks to make sure domains don't overlap - #TODO add debug logging for troubleshooting stake + #TODO add debug logging for troubleshooting sake for mx_config in self.acceptable_mxs.values(): if not mx_config.is_valid(): return False @@ -267,9 +333,13 @@ class Config(BaseConfig): # check to make sure every accepted MX has a TLS policy if not domain_suffix in self.tls_policies: return False - for tls_config in self.tls_policies.values(): + all_mx_hosts = self.get_all_mx_hosts() + for domain_suffix, tls_config in self.tls_policies.iteritems(): if not tls_config.is_valid(): return False + # make sure no unclaimed TLS policies have made their way in + if domain_suffix not in all_mx_hosts: + return False return True @@ -460,4 +530,3 @@ class AcceptableMX(BaseConfig): class ConfigError(ValueError): def __init__(self, message): super(self.__class__, self).__init__(message) - diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index a733ab27a..f83d98d93 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -33,6 +33,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.postfix_cf_file = self.find_postfix_cf() self.wrangle_existing_config() self.set_domainwise_tls_policies() + #TODO make this optional for testing, etc. os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): @@ -120,33 +121,35 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - for address_domain, properties in self.policy_config.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] + all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + for address_domain, properties in all_acceptable_mxs.items(): + mx_list = properties.accept_mx_domains if len(mx_list) > 1: print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain mx_domain = mx_list[0] - mx_policy = self.policy_config.tls_policies[mx_domain] + mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" - if "min-tls-version" in mx_policy: - if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" - elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" - elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" - else: - print mx_policy["min-tls-version"] + if mx_policy.min_tls_version.lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy.min_tls_version.lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy.min_tls_version.lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy.min_tls_version self.policy_lines.append(entry) f = open(self.policy_file, "w") f.write("\n".join(self.policy_lines) + "\n") f.close() + if __name__ == "__main__": - import ConfigParser + import Config as config if len(sys.argv) != 3: print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" sys.exit(1) - c = ConfigParser.Config(sys.argv[1]) + c = config.Config() + c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index 0348432b0..c08e953ae 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -3,7 +3,7 @@ import re import sys import collections -import ConfigParser +import Config as config # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -35,21 +35,23 @@ def get_counts(input, config): # Log lines for when a TLS connection was successfully established. These can # indicate the difference between Untrusted, Trusted, and Verified certs. connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") + mx_to_domain_mapping = config.get_mx_to_domain_policy_map() + for line in sys.stdin: deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: - validation = result.group(1) - mx_hostname = result.group(2).lower() + validation = connected.group(1) + mx_hostname = connected.group(2).lower() if validation == "Trusted" or validation == "Verified": seen_trusted = True - address_domains = config.get_address_domains(mx_hostname) + address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) if address_domains: - for d in address_domains: - counts[d][validation] += 1 - counts[d]["all"] += 1 + d = ', '.join(address_domains) + counts[d][validation] += 1 + counts[d]["all"] += 1 elif deferred: - mx_hostname = result.group(1).lower() + mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 if not seen_trusted: # Postfix will only emit 'Trusted' if the certificate validates according to @@ -65,7 +67,11 @@ def print_summary(counts): print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": - config = ConfigParser.Config("starttls-everywhere.json") + if len(sys.argv) != 2: + print "Usage: %s starttls-everywhere.json" % sys.argv[0] + sys.exit(1) + c = config.Config() + c.load_from_json_file(sys.argv[1]) (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) print tls_deferred diff --git a/TestConfig.py b/TestConfig.py index 043eb2dca..f323554c0 100644 --- a/TestConfig.py +++ b/TestConfig.py @@ -1,4 +1,5 @@ import copy +import itertools import logging import unittest @@ -69,5 +70,62 @@ class TestAcceptableMX(unittest.TestCase): self.assertEquals(mx_policy.domain, self.old_config.domain) +class TestConfig(unittest.TestCase): + """Test entire configuration. + + Currently lower coverage is being obtained since string sets are + being compared rather than returned objects. Comparison logic for + the config objects isn't clear yet and proof that they function is enough. + """ + + def setUp(self): + self.config = Config.Config() + domain_policies = self.config._data['acceptable-mxs'] + self.mail_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', '123.cn', 'qq.com'] + for domain in self.mail_domains: + new = Config.AcceptableMX(domain=domain) + new.add_acceptable_mx('.' + domain) + domain_policies[domain] = new + + def testGetAllMxItems(self): + """Make sure the basic use case of get_all_mx_items functions.""" + # [ ('.gmail.com', 'gmail.com'), ('.yahoo.com', 'yahoo.com'), ... ] + control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] + test_data = [ (mx, p.domain) for mx, p in self.config.get_all_mx_items() ] + self.assertListEqual(sorted(test_data), sorted(control_data)) + + def testGetAllMxItemsMultiMX(self): + config = copy.deepcopy(self.config) + domain_policy = config.acceptable_mxs.get('gmail.com') + # deal with reality, mail.google.com + domain_policy.add_acceptable_mx('.mail.google.com') + control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] + control_data.append(('.mail.google.com', 'gmail.com')) + test_data = [ (mx, p.domain) for mx, p in config.get_all_mx_items() ] + self.assertListEqual(sorted(test_data), sorted(control_data)) + + def testGetMXtoDomainPolicy(self): + control_data = dict([ ('.' + domain, set([domain])) + for domain in self.mail_domains ]) + test_data = {} + for mx, pset in self.config.get_mx_to_domain_policy_map().items(): + policy_list = [ p.domain for p in pset ] + test_data[mx] = set(policy_list) + self.assertDictEqual(test_data, control_data) + + def testGetMXtoDomainPolicyMultiMX(self): + config = copy.deepcopy(self.config) + domain_policy = config.acceptable_mxs.get('gmail.com') + domain_policy.add_acceptable_mx('.mail.google.com') + control_data = dict([ ('.' + domain, set([domain])) + for domain in self.mail_domains ]) + control_data['.mail.google.com'] = set(['gmail.com']) + test_data = {} + for mx, pset in config.get_mx_to_domain_policy_map().items(): + policy_list = [ p.domain for p in pset ] + test_data[mx] = set(policy_list) + self.assertDictEqual(test_data, control_data) + + if __name__ == '__main__': unittest.main() diff --git a/bigger_test_config.json b/bigger_test_config.json index e0697fc85..c3c23c455 100644 --- a/bigger_test_config.json +++ b/bigger_test_config.json @@ -1,7 +1,7 @@ { "timestamp": 1401414363, "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404242424, + "expires": "2015-08-01T12:00:00+08:00", "tls-policies": { ".yahoodns.net": { "require-valid-certificate": true From 7c6c3efb0f94a6954c99cc8e90b9f0821c1222a8 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Thu, 21 Jan 2016 01:46:30 -0800 Subject: [PATCH 083/639] Confirmed Postfix log parsing is working again. --- Config.py | 2 +- PostfixLogSummary.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Config.py b/Config.py index eb4f31dba..256a17b04 100644 --- a/Config.py +++ b/Config.py @@ -278,7 +278,7 @@ class Config(BaseConfig): labels = mx_hostname.split(".") for n in range(1, len(labels)): parent = "." + ".".join(labels[n:]) - if parent in mx_to_domains_map: + if parent in mx_to_domain_map: return mx_to_domain_map[parent] return None diff --git a/PostfixLogSummary.py b/PostfixLogSummary.py index c08e953ae..f9e717f66 100755 --- a/PostfixLogSummary.py +++ b/PostfixLogSummary.py @@ -3,7 +3,7 @@ import re import sys import collections -import Config as config +import Config # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -47,7 +47,8 @@ def get_counts(input, config): seen_trusted = True address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) if address_domains: - d = ', '.join(address_domains) + domains_str = [ a.domain for a in address_domains ] + d = ', '.join(domains_str) counts[d][validation] += 1 counts[d]["all"] += 1 elif deferred: @@ -70,8 +71,8 @@ if __name__ == "__main__": if len(sys.argv) != 2: print "Usage: %s starttls-everywhere.json" % sys.argv[0] sys.exit(1) - c = config.Config() - c.load_from_json_file(sys.argv[1]) + config = Config.Config() + config.load_from_json_file(sys.argv[1]) (counts, tls_deferred) = get_counts(sys.stdin, config) print_summary(counts) print tls_deferred From 904dc11b03a9883952572b6b3294edcd040fc64a Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 09:45:37 -0800 Subject: [PATCH 084/639] Add docstrings for Config objects update/merge methods. --- Config.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Config.py b/Config.py index 256a17b04..dc402529a 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,5 @@ from datetime import datetime -from dateutil import parser +from dateutil import dateutil_parser import collections import json import logging @@ -36,7 +36,7 @@ def parse_timestamp(value, attr_name): except (TypeError, ValueError): pass try: - return parser.parse(value) + return dateutil_parser.parse(value) except (TypeError, ValueError): raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) @@ -96,6 +96,30 @@ class BaseConfig(object): return s def update(self, newer_config, merge=False, **kwargs): + """Create a fresh config combining the new and old configs. + + It does this by iterating over the 'config_properties' class + attribute which contains names of property attributes for the config. + + Two methods of combining configs are possible, an 'update' and + a 'merge', the latter set by the keyword argument 'merge=True'. + + An update overrides older values with new values -- even if those + new values are None. Update will remove values that are present in + the old config if they are not present in the new config. + + A merge by comparison will allow old values to persist if they are + not specified in the new config. This can be used for end-user + customizations to override specific settings without having to re-create + large portions of a config to override it. + + Arguments: + newer_config: A config object to combine with the current config. + merge: Allows old values not overridden to survive into the fresh config. + + Returns: + A config object of the same sort as called upon. + """ # removed 'merge' kw arg - and it was passed to constructor # make a note to not do that, consume it on the param list fresh_config = self.__class__(**kwargs) @@ -117,6 +141,17 @@ class BaseConfig(object): return fresh_config def merge(self, newer_config, **kwargs): + """Combines configs and keeps old values if they are not overridden. + + See docstring for 'update' method for more details. + + Arguments: + newer_config: A config object to combine with the current config. + merge: Allows old values not overridden to survive into the fresh config. + + Returns: + A config object of the same sort as called upon. + """ kwargs['merge'] = True logger.debug('from parent merge: %s' % kwargs) return self.update(newer_config, **kwargs) From 146fce3878f7b75ae3e7a9733f79cfae666de448 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 09:51:37 -0800 Subject: [PATCH 085/639] Add comment about magic hat trick with class properties. --- Config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Config.py b/Config.py index dc402529a..402fb4953 100644 --- a/Config.py +++ b/Config.py @@ -130,6 +130,7 @@ class BaseConfig(object): self.__class__, newer_config.__class__)) for prop_name in self.config_properties: + # get the specified property off of the current class prop = self.__class__.__dict__.get(prop_name) assert prop new_value = prop.fget(newer_config) From 9abef4c0bdf33fd4e2c874103def6c9bcac74462 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 10:20:56 -0800 Subject: [PATCH 086/639] Log MX records that will not be configured. --- MTAConfigGenerator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index f83d98d93..1c273cf94 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -125,7 +125,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + print "Lists of multiple accept-mx-domains not yet supported." + print "Using MX %s for %s" % (mx_list[0], address_domain) + print "Ignoring: %s" % (', '.join(mx_list[1:])) mx_domain = mx_list[0] mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" From 39a01190d50514666a9bc6f147fc9ba6cce20227 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:19:02 -0800 Subject: [PATCH 087/639] Use 4 space indents For greater compatibility with LE codebases --- MTAConfigGenerator.py | 262 +++++++++++++++++++++--------------------- 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 1c273cf94..bb3d5e5db 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -5,153 +5,153 @@ import string import os, os.path def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. + """ + Return the (line number, left hand side, right hand side) of a stripped + postfix config line. - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + num,line = line_data + left, sep, right = line.partition("=") + if not sep: + return None + return (num, left.strip(), right.strip()) class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config + def __init__(self, policy_config): + self.policy_config = policy_config class ExistingConfigError(ValueError): pass class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) - self.postfix_cf_file = self.find_postfix_cf() - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - #TODO make this optional for testing, etc. - os.system("sudo service postfix reload") + def __init__(self, policy_config, postfix_dir, fixup=False): + self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) + self.postfix_cf_file = self.find_postfix_cf() + self.wrangle_existing_config() + self.set_domainwise_tls_policies() + #TODO make this optional for testing, etc. + os.system("sudo service postfix reload") - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - """ - acceptable = [ideal] + also_acceptable + def ensure_cf_var(self, var, ideal, also_acceptable): + """ + Ensure that existing postfix config @var is in the list of @acceptable + values; if not, set it to the ideal value. + """ + acceptable = [ideal] + also_acceptable - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - #print "Scheduling deletions:" + `values` - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) + l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] + if not any(l): + self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Conflicting existing config values " + `l` - val = values[0][2] - if val not in acceptable: - #print "Scheduling deletions:" + `values` + values = map(parse_line, l) + if len(set(values)) > 1: + if self.fixup: + #print "Scheduling deletions:" + `values` + conflicting_lines = [num for num,_var,val in values] + self.deletions.extend(conflicting_lines) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Conflicting existing config values " + `l` + val = values[0][2] + if val not in acceptable: + #print "Scheduling deletions:" + `values` + if self.fixup: + self.deletions.append(values[0][0]) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + + def wrangle_existing_config(self): + """ + Try to ensure/mutate that the config file is in a sane state. + Fixup means we'll delete existing lines if necessary to get there. + """ + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + + # Check we're currently accepting inbound STARTTLS sensibly + self.ensure_cf_var("smtpd_use_tls", "yes", []) + # Ideally we use it opportunistically in the outbound direction + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) + # Maximum verbosity lets us collect failure information + self.ensure_cf_var("smtp_tls_loglevel", "1", []) + # Inject a reference to our per-domain policy map + policy_cf_entry = "texthash:" + self.policy_file + + self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + + self.maybe_add_config_lines() + + def maybe_add_config_lines(self): + if not self.additions: + return if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + print "Deleting lines:", self.deletions + self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + new_cf_lines = "\n".join(self.additions) + "\n" + print "Adding to %s:" % self.fn + print new_cf_lines + if self.raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] + self.new_cf = "" + for num, line in enumerate(self.raw_cf): + if self.fixup and num in self.deletions: + self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line + else: + self.new_cf += line + self.new_cf += sep + new_cf_lines - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file + #print self.new_cf + f = open(self.fn, "w") + f.write(self.new_cf) + f.close() - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.postfix_dir, "main.cf") - self.maybe_add_config_lines() + def set_domainwise_tls_policies(self): + self.policy_lines = [] + all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + for address_domain, properties in all_acceptable_mxs.items(): + mx_list = properties.accept_mx_domains + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported." + print "Using MX %s for %s" % (mx_list[0], address_domain) + print "Ignoring: %s" % (', '.join(mx_list[1:])) + mx_domain = mx_list[0] + mx_policy = self.policy_config.get_tls_policy(mx_domain) + entry = address_domain + " encrypt" + if mx_policy.min_tls_version.lower() == "tlsv1": + entry += " protocols=!SSLv2,!SSLv3" + elif mx_policy.min_tls_version.lower() == "tlsv1.1": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + elif mx_policy.min_tls_version.lower() == "tlsv1.2": + entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + else: + print mx_policy.min_tls_version + self.policy_lines.append(entry) - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - self.new_cf = "" - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - #print self.new_cf - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() - - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - - def set_domainwise_tls_policies(self): - self.policy_lines = [] - all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() - for address_domain, properties in all_acceptable_mxs.items(): - mx_list = properties.accept_mx_domains - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported." - print "Using MX %s for %s" % (mx_list[0], address_domain) - print "Ignoring: %s" % (', '.join(mx_list[1:])) - mx_domain = mx_list[0] - mx_policy = self.policy_config.get_tls_policy(mx_domain) - entry = address_domain + " encrypt" - if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" - elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" - elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" - else: - print mx_policy.min_tls_version - self.policy_lines.append(entry) - - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() + f = open(self.policy_file, "w") + f.write("\n".join(self.policy_lines) + "\n") + f.close() if __name__ == "__main__": - import Config as config - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) - c = config.Config() - c.load_from_json_file(sys.argv[1]) - postfix_dir = sys.argv[2] - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + import Config as config + if len(sys.argv) != 3: + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" + sys.exit(1) + c = config.Config() + c.load_from_json_file(sys.argv[1]) + postfix_dir = sys.argv[2] + pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) From 965027ce528358cd838cffd6f3a8228630643637 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:36:38 -0800 Subject: [PATCH 088/639] Start metamorphising to use LE's IInstaller interface --- MTAConfigGenerator.py | 137 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index bb3d5e5db..ae74892eb 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,13 +28,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) - self.postfix_cf_file = self.find_postfix_cf() - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - #TODO make this optional for testing, etc. - os.system("sudo service postfix reload") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -145,6 +138,133 @@ class PostfixConfigGenerator(MTAConfigGenerator): f.write("\n".join(self.policy_lines) + "\n") f.close() + ### Let's Encrypt client IPlugin ### + + def prepare(): + """Prepare the plugin. + Finish up any additional initialization. + :raises .PluginError: + when full initialization cannot be completed. + :raises .MisconfigurationError: + when full initialization cannot be completed. Plugin will + be displayed on a list of available plugins. + :raises .NoInstallationError: + when the necessary programs/files cannot be located. Plugin + will NOT be displayed on a list of available plugins. + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + """ + # XXX ensure we raise the right kinds of exceptions + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) + self.postfix_cf_file = self.find_postfix_cf() + + + def more_info(): + """Human-readable string to help the user. + Should describe the steps taken and any relevant info to help the user + decide which plugin to use. + :rtype str: + """ + + + ### Let's Encrypt client IInstaller ### + + def get_all_names(): + """Returns all names that may be authenticated. + :rtype: `list` of `str` + """ + + def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path): + """Deploy certificate. + :param str domain: domain to deploy certificate file + :param str cert_path: absolute path to the certificate file + :param str key_path: absolute path to the private key file + :param str chain_path: absolute path to the certificate chain file + :param str fullchain_path: absolute path to the certificate fullchain + file (cert plus chain) + :raises .PluginError: when cert cannot be deployed + """ + + self.wrangle_existing_config() + self.set_domainwise_tls_policies() + + def enhance(domain, enhancement, options=None): + """Perform a configuration enhancement. + :param str domain: domain for which to provide enhancement + :param str enhancement: An enhancement as defined in + :const:`~letsencrypt.constants.ENHANCEMENTS` + :param options: Flexible options parameter for enhancement. + Check documentation of + :const:`~letsencrypt.constants.ENHANCEMENTS` + for expected options for each enhancement. + :raises .PluginError: If Enhancement is not supported, or if + an error occurs during the enhancement. + """ + + def supported_enhancements(): + """Returns a list of supported enhancements. + :returns: supported enhancements which should be a subset of + :const:`~letsencrypt.constants.ENHANCEMENTS` + :rtype: :class:`list` of :class:`str` + """ + + def get_all_certs_keys(): + """Retrieve all certs and keys set in configuration. + :returns: tuples with form `[(cert, key, path)]`, where: + - `cert` - str path to certificate file + - `key` - str path to associated key file + - `path` - file path to configuration file + :rtype: list + """ + + def save(title=None, temporary=False): + """Saves all changes to the configuration files. + Both title and temporary are needed because a save may be + intended to be permanent, but the save is not ready to be a full + checkpoint. If an exception is raised, it is assumed a new + checkpoint was not created. + :param str title: The title of the save. If a title is given, the + configuration will be saved as a new checkpoint and put in a + timestamped directory. `title` has no effect if temporary is true. + :param bool temporary: Indicates whether the changes made will + be quickly reversed in the future (challenges) + :raises .PluginError: when save is unsuccessful + """ + + def rollback_checkpoints(rollback=1): + """Revert `rollback` number of configuration checkpoints. + :raises .PluginError: when configuration cannot be fully reverted + """ + + def recovery_routine(): + """Revert configuration to most recent finalized checkpoint. + Remove all changes (temporary and permanent) that have not been + finalized. This is useful to protect against crashes and other + execution interruptions. + :raises .errors.PluginError: If unable to recover the configuration + """ + + def view_config_changes(): + """Display all of the LE config changes. + :raises .PluginError: when config changes cannot be parsed + """ + + def config_test(): + """Make sure the configuration is valid. + :raises .MisconfigurationError: when the config is not in a usable state + """ + + def restart(): + """Restart or refresh the server content. + :raises .PluginError: when server cannot be restarted + """ + if os.geteuid() != 0: + os.system("sudo service postfix reload") + else: + os.system("service postfix reload") + if __name__ == "__main__": import Config as config @@ -155,3 +275,6 @@ if __name__ == "__main__": c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + pcgen.prepare() + pcgen.deploy_cert() # XXX add cert args! + pcgen.restart() From fedf97028427203ff8ea9e3b2e63e24826982c71 Mon Sep 17 00:00:00 2001 From: dmwilcox Date: Wed, 17 Feb 2016 11:37:28 -0800 Subject: [PATCH 089/639] Fix bad import to be import *as*... as it should be. --- Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Config.py b/Config.py index 402fb4953..ac5202d0c 100644 --- a/Config.py +++ b/Config.py @@ -1,5 +1,5 @@ from datetime import datetime -from dateutil import dateutil_parser +from dateutil import parser as dateutil_parser import collections import json import logging From 47a5b7e3ba6fd6708e2a776f5730602029f12c00 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 11:56:41 -0800 Subject: [PATCH 090/639] Start implementing cert installation --- MTAConfigGenerator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index ae74892eb..74a53a345 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -81,7 +81,6 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.maybe_add_config_lines() def maybe_add_config_lines(self): if not self.additions: @@ -186,8 +185,9 @@ class PostfixConfigGenerator(MTAConfigGenerator): file (cert plus chain) :raises .PluginError: when cert cannot be deployed """ - self.wrangle_existing_config() + self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) + self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() def enhance(domain, enhancement, options=None): @@ -233,6 +233,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): :raises .PluginError: when save is unsuccessful """ + self.maybe_add_config_lines() + def rollback_checkpoints(rollback=1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted @@ -277,4 +279,5 @@ if __name__ == "__main__": pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() pcgen.deploy_cert() # XXX add cert args! + pcgen.save() pcgen.restart() From 8f5b8558d28f817077d663854d89b5128d1b8cbe Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:09:25 -0800 Subject: [PATCH 091/639] Actually deploy a cert? - Also add missing selves to interface methods --- MTAConfigGenerator.py | 48 ++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 74a53a345..3a2a653d6 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -28,6 +28,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def __init__(self, policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + MTAConfigGenerator.__init__(self, policy_config) def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -139,7 +141,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): ### Let's Encrypt client IPlugin ### - def prepare(): + def prepare(self): """Prepare the plugin. Finish up any additional initialization. :raises .PluginError: @@ -155,12 +157,10 @@ class PostfixConfigGenerator(MTAConfigGenerator): currently supported. """ # XXX ensure we raise the right kinds of exceptions - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) self.postfix_cf_file = self.find_postfix_cf() - def more_info(): + def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user decide which plugin to use. @@ -170,12 +170,12 @@ class PostfixConfigGenerator(MTAConfigGenerator): ### Let's Encrypt client IInstaller ### - def get_all_names(): + def get_all_names(self): """Returns all names that may be authenticated. :rtype: `list` of `str` """ - def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path): + def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. :param str domain: domain to deploy certificate file :param str cert_path: absolute path to the certificate file @@ -190,7 +190,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() - def enhance(domain, enhancement, options=None): + def enhance(self, domain, enhancement, options=None): """Perform a configuration enhancement. :param str domain: domain for which to provide enhancement :param str enhancement: An enhancement as defined in @@ -203,14 +203,14 @@ class PostfixConfigGenerator(MTAConfigGenerator): an error occurs during the enhancement. """ - def supported_enhancements(): + def supported_enhancements(self): """Returns a list of supported enhancements. :returns: supported enhancements which should be a subset of :const:`~letsencrypt.constants.ENHANCEMENTS` :rtype: :class:`list` of :class:`str` """ - def get_all_certs_keys(): + def get_all_certs_keys(self): """Retrieve all certs and keys set in configuration. :returns: tuples with form `[(cert, key, path)]`, where: - `cert` - str path to certificate file @@ -219,7 +219,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): :rtype: list """ - def save(title=None, temporary=False): + def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be intended to be permanent, but the save is not ready to be a full @@ -235,12 +235,12 @@ class PostfixConfigGenerator(MTAConfigGenerator): self.maybe_add_config_lines() - def rollback_checkpoints(rollback=1): + def rollback_checkpoints(self, rollback=1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted """ - def recovery_routine(): + def recovery_routine(self): """Revert configuration to most recent finalized checkpoint. Remove all changes (temporary and permanent) that have not been finalized. This is useful to protect against crashes and other @@ -248,17 +248,17 @@ class PostfixConfigGenerator(MTAConfigGenerator): :raises .errors.PluginError: If unable to recover the configuration """ - def view_config_changes(): + def view_config_changes(self): """Display all of the LE config changes. :raises .PluginError: when config changes cannot be parsed """ - def config_test(): + def config_test(self): """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ - def restart(): + def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ @@ -268,16 +268,26 @@ class PostfixConfigGenerator(MTAConfigGenerator): os.system("service postfix reload") +def usage(): + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" + sys.exit(1) + if __name__ == "__main__": import Config as config - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) + if len(sys.argv) != 4: + usage() c = config.Config() c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] + le_lineage = sys.argv[3] + pieces = [os.path.join(le_lineage, f) for f in ("cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] + if not os.isdir(le_lineage) or not all(os.isfile(p) for p in pieces) : + print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" + print + usage() + cert, key, chain, fullchain = pieces pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() - pcgen.deploy_cert() # XXX add cert args! + pcgen.deploy_cert(cert, key, chain, fullchain) pcgen.save() pcgen.restart() From 3aeb62cf7e3d0eacedaafeca386afddf4a85a39f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:13:28 -0800 Subject: [PATCH 092/639] bugfixes, cleanups --- MTAConfigGenerator.py | 8 +++++--- requirements.txt | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3a2a653d6..3c676700e 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -269,7 +269,8 @@ class PostfixConfigGenerator(MTAConfigGenerator): def usage(): - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" + print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % + sys.argv[0]) sys.exit(1) if __name__ == "__main__": @@ -280,8 +281,9 @@ if __name__ == "__main__": c.load_from_json_file(sys.argv[1]) postfix_dir = sys.argv[2] le_lineage = sys.argv[3] - pieces = [os.path.join(le_lineage, f) for f in ("cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] - if not os.isdir(le_lineage) or not all(os.isfile(p) for p in pieces) : + pieces = [os.path.join(le_lineage, f) for f in ( + "cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] + if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) : print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" print usage() diff --git a/requirements.txt b/requirements.txt index 891e5809d..5334ba03a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ dnspython publicsuffix m2crypto +dateutils From 074fef773bcf3097a6c6532740b12535e3f3334d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:14:24 -0800 Subject: [PATCH 093/639] Make up a domain --- MTAConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 3c676700e..9de48ca26 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -290,6 +290,6 @@ if __name__ == "__main__": cert, key, chain, fullchain = pieces pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) pcgen.prepare() - pcgen.deploy_cert(cert, key, chain, fullchain) + pcgen.deploy_cert("example.com", cert, key, chain, fullchain) pcgen.save() pcgen.restart() From 28bb0eb6acf7c63c763189963803a93f0b2691fd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 12:20:26 -0800 Subject: [PATCH 094/639] Obtain acceptable_mxs the right way --- MTAConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 9de48ca26..43dad5f37 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -115,7 +115,7 @@ class PostfixConfigGenerator(MTAConfigGenerator): def set_domainwise_tls_policies(self): self.policy_lines = [] - all_acceptable_mxs = self.policy_config.get_acceptable_mxs_dict() + all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: From 9e42f6ed08e27f022568a4d651d24ddcf38aeffd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 18 Feb 2016 18:57:41 -0800 Subject: [PATCH 095/639] Clarify project status --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e50630ab8..49db2aeda 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # STARTTLS Everywhere +NOTE: this is a pre-alpha codebase. Do not run it on non-experimental systems +yet! + ## Authors -Jacob Hoffman-Andrews , Peter Eckersley +Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox ## Background From 4d14423a2146bc39f6d4f24c0266e26e262d71f2 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Wed, 24 Feb 2016 21:38:31 +0100 Subject: [PATCH 096/639] re-structured project folder.. * Removed `ConfigParser.py` (ACK by Daniel). * Removed MTAConfigGenerator-stub and renamed to `PostfixConfigGenerator.py` * Moved all text/csv processing scripts to `tools/`. * Moved all configuration files into dedicated `examples/` directory. * unified all shebangs to `#!/usr/bin/env python` (default system python). * Moved domain CSV and text-files to `share/`. --- ConfigParser.py | 115 ------------------ .../bigger_test_config.json | 0 config.json => examples/config.json | 0 .../starttls-everywhere.json | 0 Config.py => letsencrypt-postfix/Config.py | 1 + .../PostfixConfigGenerator.py | 15 +-- .../PostfixLogSummary.py | 2 +- .../TestConfig.py | 1 + .../golden-domains.txt | 0 .../google-starttls-domains.csv | 0 CheckSTARTTLS.py => tools/CheckSTARTTLS.py | 2 +- .../ProcessGoogleSTARTTLSDomains.py | 2 +- 12 files changed, 10 insertions(+), 128 deletions(-) delete mode 100755 ConfigParser.py rename bigger_test_config.json => examples/bigger_test_config.json (100%) rename config.json => examples/config.json (100%) rename starttls-everywhere.json => examples/starttls-everywhere.json (100%) rename Config.py => letsencrypt-postfix/Config.py (99%) rename MTAConfigGenerator.py => letsencrypt-postfix/PostfixConfigGenerator.py (96%) rename PostfixLogSummary.py => letsencrypt-postfix/PostfixLogSummary.py (99%) rename TestConfig.py => letsencrypt-postfix/TestConfig.py (99%) mode change 100644 => 100755 rename golden-domains.txt => share/golden-domains.txt (100%) rename google-starttls-domains.csv => share/google-starttls-domains.csv (100%) rename CheckSTARTTLS.py => tools/CheckSTARTTLS.py (99%) rename ProcessGoogleSTARTTLSDomains.py => tools/ProcessGoogleSTARTTLSDomains.py (98%) diff --git a/ConfigParser.py b/ConfigParser.py deleted file mode 100755 index 2d7c88ada..000000000 --- a/ConfigParser.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python - -import sys -import json -from datetime import datetime -import string -import collections - -def parse_timestamp(ts): - try: - int(ts) - dt = datetime.fromtimestamp(ts) - return dt - except: - raise ValueError, "Invalid timestamp integer: " + `ts` - -legal = string.letters + string.digits + ".-" -known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list -def looks_like_a_domain(s): - "Return true if string looks like a domain, as best we can tell..." - global known_tlds - try: - domain = s.lower() - assert domain[0].islower() - assert all([c in legal for c in domain]) - tld = s.split(".")[-1] - if tld not in known_tlds: - # XXX perform DNS query to determine that this TLD exists - pass - return True - except: - return False - -class Config: - def __init__(self, cfg_file_name = "config.json"): - f = open(cfg_file_name) - self.cfg = json.loads(f.read()) - self.tls_policies = {} - self.mx_map = {} - for atr, val in self.cfg.items(): - # Verify each attribute of the structure - if atr.startswith("comment"): - continue - if atr == "author": - if type(val) not in [str, unicode]: - raise TypeError, "Author must be a string: " + `val` - elif atr == "timestamp": - self.timestamp = parse_timestamp(val) - elif atr == "expires": - self.expires = parse_timestamp(val) - elif atr == "tls-policies": - for domain, policies in self.check_tls_policy_domains(val): - if type(policies) != dict: - raise TypeError, domain + "'s policies should be a dict: " + `policies` - self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, v in policies.items(): - value = str(v).lower() - if policy == "require-tls": - if value in ("true", "1", "yes"): - self.tls_policies[domain]["required"] = True - elif value in ("false", "0", "no"): - self.tls_policies[domain]["required"] = False - else: - raise ValueError, "Unknown require-tls value " + `value` - elif policy == "min-tls-version": - reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] - reasonable = map(string.lower, reasonable) - if not value in reasonable: - raise ValueError, "Not a valid TLS version string: " + `value` - self.tls_policies[domain]["min-tls-version"] = str(value) - elif policy == "enforce-mode": - if value == "enforce": - self.tls_policies[domain]["enforce"] = True - elif value == "log-only": - self.tls_policies[domain]["enforce"] = False - else: - raise ValueError, "Not a known enoforcement policy " + `value` - elif atr == "acceptable-mxs": - self.acceptable_mxs = val - self.mx_domain_to_address_domains = collections.defaultdict(set) - for address_domain, properties in self.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - self.mx_domain_to_address_domains[mx_domain].add(address_domain) - pass - else: - sys.stderr.write("Unknown attribute: " + `atr` + "\n") - # XXX is it ever permissible to have a domain with an acceptable-mx - # that does not point to a TLS security policy? If not, check/warn/fail - # here - print self.tls_policies - - def get_address_domains(self, mx_hostname): - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in self.mx_domain_to_address_domains: - return self.mx_domain_to_address_domains[parent] - return None - - def check_tls_policy_domains(self, val): - if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` - for domain, policies in val.items(): - try: - assert type(domain) == unicode - d = str(domain) # convert from unicode - except: - raise TypeError, "tls-policy domain not a string" + `domain` - yield (d, policies) - -if __name__ == "__main__": - c = Config() diff --git a/bigger_test_config.json b/examples/bigger_test_config.json similarity index 100% rename from bigger_test_config.json rename to examples/bigger_test_config.json diff --git a/config.json b/examples/config.json similarity index 100% rename from config.json rename to examples/config.json diff --git a/starttls-everywhere.json b/examples/starttls-everywhere.json similarity index 100% rename from starttls-everywhere.json rename to examples/starttls-everywhere.json diff --git a/Config.py b/letsencrypt-postfix/Config.py similarity index 99% rename from Config.py rename to letsencrypt-postfix/Config.py index ac5202d0c..cc0df00d1 100644 --- a/Config.py +++ b/letsencrypt-postfix/Config.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python from datetime import datetime from dateutil import parser as dateutil_parser import collections diff --git a/MTAConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py similarity index 96% rename from MTAConfigGenerator.py rename to letsencrypt-postfix/PostfixConfigGenerator.py index 43dad5f37..af1953208 100755 --- a/MTAConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import sys import string import os, os.path @@ -18,18 +17,14 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) -class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config - class ExistingConfigError(ValueError): pass -class PostfixConfigGenerator(MTAConfigGenerator): +class PostfixConfigGenerator: def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - MTAConfigGenerator.__init__(self, policy_config) + self.fixup = fixup + self.postfix_dir = postfix_dir + self.policy_config = policy_config + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") def ensure_cf_var(self, var, ideal, also_acceptable): """ diff --git a/PostfixLogSummary.py b/letsencrypt-postfix/PostfixLogSummary.py similarity index 99% rename from PostfixLogSummary.py rename to letsencrypt-postfix/PostfixLogSummary.py index f9e717f66..956a069eb 100755 --- a/PostfixLogSummary.py +++ b/letsencrypt-postfix/PostfixLogSummary.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.7 +#!/usr/bin/env python import re import sys import collections diff --git a/TestConfig.py b/letsencrypt-postfix/TestConfig.py old mode 100644 new mode 100755 similarity index 99% rename from TestConfig.py rename to letsencrypt-postfix/TestConfig.py index f323554c0..ca8e77654 --- a/TestConfig.py +++ b/letsencrypt-postfix/TestConfig.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import copy import itertools import logging diff --git a/golden-domains.txt b/share/golden-domains.txt similarity index 100% rename from golden-domains.txt rename to share/golden-domains.txt diff --git a/google-starttls-domains.csv b/share/google-starttls-domains.csv similarity index 100% rename from google-starttls-domains.csv rename to share/google-starttls-domains.csv diff --git a/CheckSTARTTLS.py b/tools/CheckSTARTTLS.py similarity index 99% rename from CheckSTARTTLS.py rename to tools/CheckSTARTTLS.py index e5c5a4323..ef0bf2e5c 100755 --- a/CheckSTARTTLS.py +++ b/tools/CheckSTARTTLS.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys import os import errno diff --git a/ProcessGoogleSTARTTLSDomains.py b/tools/ProcessGoogleSTARTTLSDomains.py similarity index 98% rename from ProcessGoogleSTARTTLSDomains.py rename to tools/ProcessGoogleSTARTTLSDomains.py index 3078bd93a..815ec5d4e 100755 --- a/ProcessGoogleSTARTTLSDomains.py +++ b/tools/ProcessGoogleSTARTTLSDomains.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ Process Google's TLS delivery data from https://www.google.com/transparencyreport/saferemail/data/?hl=en From 6db285882552e73e7ffcb6eff10293dbe0f48ce8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 24 Feb 2016 18:19:47 -0800 Subject: [PATCH 097/639] Correct policy map delimitation --- MTAConfigGenerator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py index 43dad5f37..ea4eeaa26 100755 --- a/MTAConfigGenerator.py +++ b/MTAConfigGenerator.py @@ -126,11 +126,11 @@ class PostfixConfigGenerator(MTAConfigGenerator): mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2,!SSLv3" + entry += " protocols=!SSLv2:!SSLv3" elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1" elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1" + entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: print mx_policy.min_tls_version self.policy_lines.append(entry) From 6e2b6a081706ced2e6ca1ddbc7cd3556e141934f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:37:56 -0800 Subject: [PATCH 098/639] Update README --- README.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 288a6ea10..f244d91e6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,36 @@ # STARTTLS Everywhere -NOTE: this is a pre-alpha codebase. Do not run it on non-experimental systems -yet! + +## Example usage + +**WARNING: this is a pre-alpha codebase. Do not run it on production +mailservers!!!** + + +If you have a Postfix server you're willing to endanger deliverability on, you +can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), not the directory it lives in below `/etc/letsencrypt/live` and then do: + +``` +git clone https://github.com/EFForg/starttls-everywhere +cd starttls-everywhere +# Promise you don't care if deliverability breaks on this mail server +letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/YOUR.DOMAIN.EXAMPLE.COM +``` + +This will: +* Install the cert in Postfix +* Enforce mandatory TLS to some major email domains +* Enforce minimum TLS versions to some major email domains + +## Project status + +* Postfix configuration generation: working pre-alpha, not yet safe +* Email security database: working pre-alpha, definitely not yet safe +* Let's Encrypt client plugin: in progress ## Authors -Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox +Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox , Aaron Zauner ## Mailing List From 1210c04f147f6bf6f539ee0683f46af9c8939de1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:38:55 -0800 Subject: [PATCH 099/639] tweak README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f244d91e6..8a4e04601 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ mailservers!!!** If you have a Postfix server you're willing to endanger deliverability on, you -can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), not the directory it lives in below `/etc/letsencrypt/live` and then do: +can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), note the directory it lives in below `/etc/letsencrypt/live` and then do: ``` git clone https://github.com/EFForg/starttls-everywhere @@ -26,7 +26,7 @@ This will: * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe -* Let's Encrypt client plugin: in progress +* Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready ## Authors From 3a8e1d7a707a509022839fa31944b1daf8f4b2e6 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 16:40:03 -0800 Subject: [PATCH 100/639] Further status update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8a4e04601..e0746c4a1 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ This will: * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready +* DANE support: none yet +* SMTP-STS integration: none yet ## Authors From a435036a1ecedcfe143d00f186dbf808e06d1983 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 19:03:47 -0800 Subject: [PATCH 101/639] Document MVP objectives --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e0746c4a1..9f20c4457 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,18 @@ This will: ## Project status +STARTTLS Everywhere development is re-starting after a hiatus. Initial +objectives: + * Postfix configuration generation: working pre-alpha, not yet safe * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready * DANE support: none yet * SMTP-STS integration: none yet +* Direct mechanisms for mail domains to request inclusion: none yet +* Failure reporting mechanisms: early progress, not yet ready +* Mechanisms for secure mutli-organization signature on the policy database: + none yet ## Authors From b2977ad6a996aae6bf2cd1527098722286282b3e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 1 Mar 2016 19:06:02 -0800 Subject: [PATCH 102/639] Documentation details --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f20c4457..008907baf 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json ``` This will: -* Install the cert in Postfix +* Ensure your mail server initiates STARTTLS encryption +* Install the Let's Encrypt cert in Postfix * Enforce mandatory TLS to some major email domains * Enforce minimum TLS versions to some major email domains From 874c59012ab219d24678907891a1249b1277d0d8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 2 Mar 2016 10:47:52 -0800 Subject: [PATCH 103/639] Update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 008907baf..27563a899 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ objectives: * SMTP-STS integration: none yet * Direct mechanisms for mail domains to request inclusion: none yet * Failure reporting mechanisms: early progress, not yet ready -* Mechanisms for secure mutli-organization signature on the policy database: +* Mechanisms for secure multi-organization signature on the policy database: none yet +* Support for mail servers other than Postfix: none yet ## Authors From e88eac65da32fe33228c650eeee51ca40ef138ea Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 2 Mar 2016 10:47:59 -0800 Subject: [PATCH 104/639] [documentation] Add links to LEPC interface sources --- letsencrypt-postfix/PostfixConfigGenerator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 43a3c19cf..25cab9ce7 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -135,6 +135,7 @@ class PostfixConfigGenerator: f.close() ### Let's Encrypt client IPlugin ### + # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 def prepare(self): """Prepare the plugin. @@ -164,6 +165,7 @@ class PostfixConfigGenerator: ### Let's Encrypt client IInstaller ### + # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 def get_all_names(self): """Returns all names that may be authenticated. From 0ce1684ba6895706422349dce5d2b1c94e60ae05 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:50:15 -0800 Subject: [PATCH 105/639] Changes to MTAConfigGenerator needed to be moved by hand --- MTAConfigGenerator.py | 160 ------------------ letsencrypt-postfix/PostfixConfigGenerator.py | 18 +- 2 files changed, 12 insertions(+), 166 deletions(-) delete mode 100755 MTAConfigGenerator.py diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py deleted file mode 100755 index 5ca8f5aee..000000000 --- a/MTAConfigGenerator.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python - -import sys -import string -import os, os.path - -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) - -class MTAConfigGenerator: - def __init__(self, policy_config): - self.policy_config = policy_config - -class ExistingConfigError(ValueError): pass - -class PostfixConfigGenerator(MTAConfigGenerator): - def __init__(self, policy_config, postfix_dir, fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.postfix_cf_file = self.find_postfix_cf() - if not os.access(self.postfix_cf_file, os.W_OK): - raise Exception("Can't write to %s, please re-run as root." - % self.postfix_cf_file) - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") - self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") - MTAConfigGenerator.__init__(self, policy_config) - self.wrangle_existing_config() - self.set_domainwise_tls_policies() - self.update_CAfile() - os.system("sudo service postfix reload") - - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - """ - acceptable = [ideal] + also_acceptable - - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Conflicting existing config values " + `l` - val = values[0][2] - if val not in acceptable: - if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) - - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file - - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - - self.maybe_add_config_lines() - - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] - new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - self.new_cf = "" - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() - - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - - def set_domainwise_tls_policies(self): - self.policy_lines = [] - for address_domain, properties in self.policy_config.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - mx_policy = self.policy_config.tls_policies[mx_domain] - entry = address_domain + " encrypt" - if "min-tls-version" in mx_policy: - if mx_policy["min-tls-version"].lower() == "tlsv1": - entry += " protocols=!SSLv2:!SSLv3" - elif mx_policy["min-tls-version"].lower() == "tlsv1.1": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - elif mx_policy["min-tls-version"].lower() == "tlsv1.2": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - else: - print mx_policy["min-tls-version"] - self.policy_lines.append(entry) - - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() - - def update_CAfile(self): - os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + - self.ca_file) - -if __name__ == "__main__": - import ConfigParser - if len(sys.argv) != 3: - print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" - sys.exit(1) - c = ConfigParser.Config(sys.argv[1]) - postfix_dir = sys.argv[2] - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) - print "Done." diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 25cab9ce7..af29d493c 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -25,6 +25,7 @@ class PostfixConfigGenerator: self.postfix_dir = postfix_dir self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -40,7 +41,6 @@ class PostfixConfigGenerator: values = map(parse_line, l) if len(set(values)) > 1: if self.fixup: - #print "Scheduling deletions:" + `values` conflicting_lines = [num for num,_var,val in values] self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) @@ -48,7 +48,6 @@ class PostfixConfigGenerator: raise ExistingConfigError, "Conflicting existing config values " + `l` val = values[0][2] if val not in acceptable: - #print "Scheduling deletions:" + `values` if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) @@ -99,10 +98,11 @@ class PostfixConfigGenerator: self.new_cf += line self.new_cf += sep + new_cf_lines - #print self.new_cf - f = open(self.fn, "w") - f.write(self.new_cf) - f.close() + if not os.access(self.postfix_cf_file, os.W_OK): + raise Exception("Can't write to %s, please re-run as root." + % self.postfix_cf_file) + with open(self.fn, "w") as f: + f.write(self.new_cf) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -186,6 +186,7 @@ class PostfixConfigGenerator: self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) self.ensure_cf_var("smtpd_tls_key_file", key_path, []) self.set_domainwise_tls_policies() + self.update_CAfile() def enhance(self, domain, enhancement, options=None): """Perform a configuration enhancement. @@ -259,17 +260,22 @@ class PostfixConfigGenerator: """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ + print "Reloading postfix config..." if os.geteuid() != 0: os.system("sudo service postfix reload") else: os.system("service postfix reload") + def update_CAfile(self): + os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + def usage(): print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % sys.argv[0]) sys.exit(1) + if __name__ == "__main__": import Config as config if len(sys.argv) != 4: From a58c6524436d04adc43a89a0fa7534e7d6cdfdba Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:54:15 -0800 Subject: [PATCH 106/639] ConfigParser is gone! --- ConfigParser.py | 114 ------------------------------------------------ 1 file changed, 114 deletions(-) delete mode 100755 ConfigParser.py diff --git a/ConfigParser.py b/ConfigParser.py deleted file mode 100755 index d1c413f74..000000000 --- a/ConfigParser.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -import sys -import json -from datetime import datetime -import string -import collections - -def parse_timestamp(ts): - try: - int(ts) - dt = datetime.fromtimestamp(ts) - return dt - except: - raise ValueError, "Invalid timestamp integer: " + `ts` - -legal = string.letters + string.digits + ".-" -known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list -def looks_like_a_domain(s): - "Return true if string looks like a domain, as best we can tell..." - global known_tlds - try: - domain = s.lower() - assert domain[0].islower() - assert all([c in legal for c in domain]) - tld = s.split(".")[-1] - if tld not in known_tlds: - # XXX perform DNS query to determine that this TLD exists - pass - return True - except: - return False - -class Config: - def __init__(self, cfg_file_name = "config.json"): - f = open(cfg_file_name) - self.cfg = json.loads(f.read()) - self.tls_policies = {} - self.mx_map = {} - for atr, val in self.cfg.items(): - # Verify each attribute of the structure - if atr.startswith("comment"): - continue - if atr == "author": - if type(val) not in [str, unicode]: - raise TypeError, "Author must be a string: " + `val` - elif atr == "timestamp": - self.timestamp = parse_timestamp(val) - elif atr == "expires": - self.expires = parse_timestamp(val) - elif atr == "tls-policies": - for domain, policies in self.check_tls_policy_domains(val): - if type(policies) != dict: - raise TypeError, domain + "'s policies should be a dict: " + `policies` - self.tls_policies[domain] = {} # being here enforces TLS at all - for policy, v in policies.items(): - value = str(v).lower() - if policy == "require-tls": - if value in ("true", "1", "yes"): - self.tls_policies[domain]["required"] = True - elif value in ("false", "0", "no"): - self.tls_policies[domain]["required"] = False - else: - raise ValueError, "Unknown require-tls value " + `value` - elif policy == "min-tls-version": - reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] - reasonable = map(string.lower, reasonable) - if not value in reasonable: - raise ValueError, "Not a valid TLS version string: " + `value` - self.tls_policies[domain]["min-tls-version"] = str(value) - elif policy == "enforce-mode": - if value == "enforce": - self.tls_policies[domain]["enforce"] = True - elif value == "log-only": - self.tls_policies[domain]["enforce"] = False - else: - raise ValueError, "Not a known enoforcement policy " + `value` - elif atr == "acceptable-mxs": - self.acceptable_mxs = val - self.mx_domain_to_address_domains = collections.defaultdict(set) - for address_domain, properties in self.acceptable_mxs.items(): - mx_list = properties["accept-mx-domains"] - if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain - mx_domain = mx_list[0] - self.mx_domain_to_address_domains[mx_domain].add(address_domain) - pass - else: - sys.stderr.write("Unknown attribute: " + `atr` + "\n") - # XXX is it ever permissible to have a domain with an acceptable-mx - # that does not point to a TLS security policy? If not, check/warn/fail - # here - - def get_address_domains(self, mx_hostname): - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in self.mx_domain_to_address_domains: - return self.mx_domain_to_address_domains[parent] - return None - - def check_tls_policy_domains(self, val): - if type(val) != dict: - raise TypeError, "tls-policies should be a dict" + `val` - for domain, policies in val.items(): - try: - assert type(domain) == unicode - d = str(domain) # convert from unicode - except: - raise TypeError, "tls-policy domain not a string" + `domain` - yield (d, policies) - -if __name__ == "__main__": - c = Config() From 87022782fb0fa8431632a5713f12927874d0a7b3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 9 Mar 2016 10:59:10 -0800 Subject: [PATCH 107/639] Catch stray missing line --- letsencrypt-postfix/PostfixConfigGenerator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index af29d493c..20ca3edb4 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -76,6 +76,7 @@ class PostfixConfigGenerator: policy_cf_entry = "texthash:" + self.policy_file self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) def maybe_add_config_lines(self): From e42a222c5d54d08836371200afb0af30106d3685 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 15:02:59 +0200 Subject: [PATCH 108/639] preliminary Postfix version check --- letsencrypt-postfix/PostfixConfigGenerator.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 25cab9ce7..162a0d832 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys import string +import subprocess import os, os.path def parse_line(line_data): @@ -148,13 +149,29 @@ class PostfixConfigGenerator: :raises .NoInstallationError: when the necessary programs/files cannot be located. Plugin will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - """ + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + """ # XXX ensure we raise the right kinds of exceptions self.postfix_cf_file = self.find_postfix_cf() + # Parse Postfix version number (feature support, syntax changes etc.) + mail_version = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], \ + stdout=subprocess.PIPE) \ + .communicate()[0].split()[2] + maj, min, rev = mail_version.split('.') + + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html + self.postfix_version = mail_version + + return maj, min, rev def more_info(self): """Human-readable string to help the user. From 1cde7f9b5429bc70bee06c38034b51947c5cb758 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 16:19:34 +0200 Subject: [PATCH 109/639] added doc. on postfix version dependent features --- letsencrypt-postfix/PostfixConfigGenerator.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 162a0d832..5e219cc43 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -162,6 +162,7 @@ class PostfixConfigGenerator: stdout=subprocess.PIPE) \ .communicate()[0].split()[2] maj, min, rev = mail_version.split('.') + self.postfix_version = mail_version # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -169,8 +170,44 @@ class PostfixConfigGenerator: # see: # http://www.postfix.org/TLS_README.html # http://www.postfix.org/FORWARD_SECRECY_README.html - self.postfix_version = mail_version + + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added + + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added + + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html + return maj, min, rev def more_info(self): From 1d37e94e17b552712d1ade5f981aed8ba4835b17 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 18:43:08 +0200 Subject: [PATCH 110/639] disable SSLv3 and v3 by default #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 4b6783633..dfb20ccea 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -79,6 +79,10 @@ class PostfixConfigGenerator: self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) + # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed + # between Postfix version 2.5 and 2.6, since we only support => 2.11 + # we don't use nor support legacy Postfix syntax. + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: From 0ba508ee2dc88bd86aa4209efd55c498e394ffea Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 19:02:00 +0200 Subject: [PATCH 111/639] disable SSLv2,3 client-side too #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index dfb20ccea..609477f0b 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -82,7 +82,10 @@ class PostfixConfigGenerator: # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed # between Postfix version 2.5 and 2.6, since we only support => 2.11 # we don't use nor support legacy Postfix syntax. + # - Server: self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + # - Client: + self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: From 2900d5122ca91531fe9696fba4a13de0f7cefb8c Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Tue, 29 Mar 2016 19:29:46 +0200 Subject: [PATCH 112/639] set _all_ client&server options to exclude v2 and v3 #24 --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 609477f0b..2066b74e2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -83,8 +83,10 @@ class PostfixConfigGenerator: # between Postfix version 2.5 and 2.6, since we only support => 2.11 # we don't use nor support legacy Postfix syntax. # - Server: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) # - Client: + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): From 5cc317408ca73c7b741a65db5b20ccde4e032402 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:19:13 -0700 Subject: [PATCH 113/639] Move attributes into init and allow for injecting file contents for testing. --- letsencrypt-postfix/PostfixConfigGenerator.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index af1953208..34654a2d1 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -3,6 +3,7 @@ import sys import string import os, os.path + def parse_line(line_data): """ Return the (line number, left hand side, right hand side) of a stripped @@ -17,15 +18,30 @@ def parse_line(line_data): return None return (num, left.strip(), right.strip()) + class ExistingConfigError(ValueError): pass + class PostfixConfigGenerator: - def __init__(self, policy_config, postfix_dir, fixup=False): + def __init__(self, policy_config, postfix_dir, fixup=False, fopen=open): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = fopen(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + self.policy_lines = [] + self.new_cf = "" + + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.postfix_dir, "main.cf") + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -60,13 +76,6 @@ class PostfixConfigGenerator: Try to ensure/mutate that the config file is in a sane state. Fixup means we'll delete existing lines if necessary to get there. """ - self.additions = [] - self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - # Check we're currently accepting inbound STARTTLS sensibly self.ensure_cf_var("smtpd_use_tls", "yes", []) # Ideally we use it opportunistically in the outbound direction @@ -91,7 +100,6 @@ class PostfixConfigGenerator: if self.raw_cf[-1][-1] == "\n": sep = "" else: sep = "\n" - self.new_cf = "" for num, line in enumerate(self.raw_cf): if self.fixup and num in self.deletions: self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line @@ -104,12 +112,7 @@ class PostfixConfigGenerator: f.write(self.new_cf) f.close() - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") - def set_domainwise_tls_policies(self): - self.policy_lines = [] all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -152,7 +155,6 @@ class PostfixConfigGenerator: currently supported. """ # XXX ensure we raise the right kinds of exceptions - self.postfix_cf_file = self.find_postfix_cf() def more_info(self): From fee9c862334bfd290a2b9bd1e07b93badfb66ae5 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:20:33 -0700 Subject: [PATCH 114/639] Add failing test for get_all_names. --- .../TestPostfixConfigGenerator.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 letsencrypt-postfix/TestPostfixConfigGenerator.py diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py new file mode 100644 index 000000000..98a0afb5d --- /dev/null +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import io +import logging +import unittest + +import Config +import PostfixConfigGenerator as pcg + + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) + + +# Fake Postfix Configs +names_only_config = """myhostname = mail.fubard.org +mydomain = fubard.org +myorigin = fubard.org""" + + +def GetFakeOpen(fake_file_contents): + fake_file = io.StringIO() + # cast this to unicode for py2 + fake_file.write(fake_file_contents) + fake_file.seek(0) + + def FakeOpen(_): + return fake_file + + return FakeOpen + + +class TestPostfixConfigGenerator(unittest.TestCase): + + def setUp(self): + self.fopen_names_only_config = GetFakeOpen(names_only_config) + #self.config = Config.Config() + self.config = None + self.postfix_dir = 'tests/' + + def tearDown(self): + pass + + def testGetAllNames(self): + sorted_names = ('fubard.org', 'mail.fubard.org') + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_names_only_config + ) + self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + + +if __name__ == '__main__': + unittest.main() From fd1cef3fa03d026e1a0936a84985d68f67b98614 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:31:27 -0700 Subject: [PATCH 115/639] Implement get_all_names. --- letsencrypt-postfix/PostfixConfigGenerator.py | 9 +++++++++ letsencrypt-postfix/TestPostfixConfigGenerator.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 34654a2d1..9863d05d2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -171,6 +171,15 @@ class PostfixConfigGenerator: """Returns all names that may be authenticated. :rtype: `list` of `str` """ + var_names = ('myhostname', 'mydomain', 'myorigin') + names_found = set() + for num, line in enumerate(self.cf): + num, found_var, found_value = parse_line((num, line)) + if found_var in var_names: + names_found.add(found_value) + name_list = list(names_found) + name_list.sort() + return name_list def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index 98a0afb5d..e1d921c6f 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -48,7 +48,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): pass def testGetAllNames(self): - sorted_names = ('fubard.org', 'mail.fubard.org') + sorted_names = ['fubard.org', 'mail.fubard.org'] postfix_config_gen = pcg.PostfixConfigGenerator( self.config, self.postfix_dir, From 0bf2537a55db2752ee5a8f536b80e5e544128155 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Tue, 29 Mar 2016 14:32:53 -0700 Subject: [PATCH 116/639] Add initial gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 94c6f7089..cc957df18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .* *.orig +*.pyc From 1f95ac964071cb904585599ac565fd0fe7dda914 Mon Sep 17 00:00:00 2001 From: Aaron Zauner Date: Wed, 20 Apr 2016 12:43:51 +0700 Subject: [PATCH 117/639] README fixup pt. 1 --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 27563a899..774556e7e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ objectives: * Email security database: working pre-alpha, definitely not yet safe * Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready * DANE support: none yet +* DEEP support: none yet * SMTP-STS integration: none yet * Direct mechanisms for mail domains to request inclusion: none yet * Failure reporting mechanisms: early progress, not yet ready @@ -41,7 +42,10 @@ objectives: ## Authors -Jacob Hoffman-Andrews , Peter Eckersley , Daniel Wilcox , Aaron Zauner +Jacob Hoffman-Andrews , +Peter Eckersley , +Daniel Wilcox , +Aaron Zauner ## Mailing List @@ -49,13 +53,13 @@ starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-eve ## Background -Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack. +Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to trivial downgrade attacks. -To illustrate an easy version of this attack, suppose a network-based attacker Mallory notices that Alice has just uploaded message to her mail server. Mallory can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So Alice's message will be transmitted in the clear. +To illustrate an easy version of this attack, suppose a network-based attacker `Mallory` notices that `Alice` has just uploaded message to her mail server. `Mallory` can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So `Alice`'s message will be transmitted in the clear. Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. +Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM or Denial of Service is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. @@ -63,7 +67,8 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d * Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. * Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. -* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring +* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring. +* Create feedback-loops on targeted attacks and bulk surveilance in an opt-in, anonymized way. ## Non-goals @@ -86,7 +91,7 @@ Attacker has control of routers on the path between two MTAs of interest. Attack ## Alternatives -Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of a DANE solution easier. +Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of DANE-based solutions easier. ## Detailed design @@ -147,7 +152,7 @@ The basic file format will be JSON with comments (http://blog.getify.com/json-co A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. -The _timestamp_ field is an integer number of epoch seconds. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. +The _timestamp_ field is an integer number of epoch seconds from 00:00:00 UTC on 1 January 1970. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. @@ -183,7 +188,7 @@ The _accept-pinset_ field references an entry in the pinsets list, which has the ## Pinning and hostname verification -Like Chrome (and soon Firefox) we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. +Like Chrome and Firefox we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. @@ -194,7 +199,7 @@ We do not attempt to solve the self-signed certificate problem. For mail hosts w We have three options for creating the configuration file: 1. Ask mail operators to submit policies for their domains which we incorporate. -2. Manually curate a set of policies for the top N mail domains. +2. Manually curate a set of policies for the top `N` mail domains. 3. Programmatically create a set of policies by connecting to the top N mail domains. For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. @@ -225,6 +230,6 @@ Additionally, for ongoing monitoring of third-party deployments, we will create ## Failure reporting -For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job and send notices only when STARTTLS Everywhere-related failures exceed 0.1% for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. +For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job or small opt-in daemon and send notices only when STARTTLS Everywhere-related failures exceed a certain percentage for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. From cc83e9ba52d9bd13dd43e72942458e95b0e6fba8 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 12:26:56 -0700 Subject: [PATCH 118/639] Wrap some lines, new style exceptions, return check for restart. --- letsencrypt-postfix/PostfixConfigGenerator.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 1ac9d9fc6..927b7e729 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -28,7 +28,8 @@ class PostfixConfigGenerator: self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config - self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.policy_file = os.path.join(postfix_dir, + "starttls_everywhere_policy") self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") self.additions = [] @@ -51,7 +52,8 @@ class PostfixConfigGenerator: """ acceptable = [ideal] + also_acceptable - l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] + l = [(num,line) for num,line in enumerate(self.cf) + if line.startswith(var)] if not any(l): self.additions.append(var + " = " + ideal) else: @@ -62,14 +64,18 @@ class PostfixConfigGenerator: self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Conflicting existing config values " + `l` + raise ExistingConfigError( + "Conflicting existing config values " + `l` + ) val = values[0][2] if val not in acceptable: if self.fixup: self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + raise ExistingConfigError( + "Existing config has %s=%s"%(var,val) + ) def wrangle_existing_config(self): """ @@ -96,12 +102,14 @@ class PostfixConfigGenerator: # - Client: self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - def maybe_add_config_lines(self): + def maybe_add_config_lines(self, fopen=open): if not self.additions: return if self.fixup: print "Deleting lines:", self.deletions - self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + self.additions[:0]=["#", + "# New config lines added by STARTTLS Everywhere", + "#"] new_cf_lines = "\n".join(self.additions) + "\n" print "Adding to %s:" % self.fn print new_cf_lines @@ -118,10 +126,10 @@ class PostfixConfigGenerator: if not os.access(self.postfix_cf_file, os.W_OK): raise Exception("Can't write to %s, please re-run as root." % self.postfix_cf_file) - with open(self.fn, "w") as f: + with fopen(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self): + def set_domainwise_tls_policies(self, fopen=open): all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -142,9 +150,8 @@ class PostfixConfigGenerator: print mx_policy.min_tls_version self.policy_lines.append(entry) - f = open(self.policy_file, "w") - f.write("\n".join(self.policy_lines) + "\n") - f.close() + with fopen(self.policy_file, "w") as f: + f.write("\n".join(self.policy_lines) + "\n") ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 @@ -335,17 +342,19 @@ class PostfixConfigGenerator: """ print "Reloading postfix config..." if os.geteuid() != 0: - os.system("sudo service postfix reload") + rc = os.system("sudo service postfix reload") else: - os.system("service postfix reload") + rc = os.system("service postfix reload") + if rc != 0: + raise Exception('PluginError: cannot restart postfix') def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) def usage(): - print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" % - sys.argv[0]) + print ("Usage: %s starttls-everywhere.json /etc/postfix " + "/etc/letsencrypt/live/example.com/" % sys.argv[0]) sys.exit(1) From e75bafa43908d9f66f94ffae1393f18c9262f5a1 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 12:27:49 -0700 Subject: [PATCH 119/639] Add basic test for get_all_certs_keys IInstaller interface method. --- .../TestPostfixConfigGenerator.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index e1d921c6f..ebcf12a4c 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -24,6 +24,12 @@ mydomain = fubard.org myorigin = fubard.org""" +certs_only_config = """ +smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem +smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem +""" + + def GetFakeOpen(fake_file_contents): fake_file = io.StringIO() # cast this to unicode for py2 @@ -40,6 +46,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.fopen_names_only_config = GetFakeOpen(names_only_config) + self.fopen_certs_only_config = GetFakeOpen(certs_only_config) #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' @@ -57,6 +64,18 @@ class TestPostfixConfigGenerator(unittest.TestCase): ) self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + def testGetAllCertAndKeys(self): + return_vals = ('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', + '/etc/letsencrypt/live/www.fubard.org/privkey.pem', + None) + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_certs_only_config + ) + self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + if __name__ == '__main__': unittest.main() From c6baa82ee4030bc75c8da3ea60ff22865d3a9137 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 15:14:06 -0700 Subject: [PATCH 120/639] Implement basic get_all_certs_keys, tests pass. --- letsencrypt-postfix/PostfixConfigGenerator.py | 11 +++++++++++ letsencrypt-postfix/TestPostfixConfigGenerator.py | 13 ++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 927b7e729..b5adba1a2 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -296,6 +296,17 @@ class PostfixConfigGenerator: - `path` - file path to configuration file :rtype: list """ + cert_materials = {'smtpd_tls_key_file': None, + 'smtpd_tls_cert_file': None, + } + for num, line in enumerate(self.cf): + print 'Line is: %s' % line + num, found_var, found_value = parse_line((num, line)) + if found_var in cert_materials.keys(): + cert_materials[found_var] = found_value + return [(cert_materials['smtpd_tls_cert_file'], + cert_materials['smtpd_tls_key_file'], + self.fn),] def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index ebcf12a4c..57cbb59c0 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -24,10 +24,9 @@ mydomain = fubard.org myorigin = fubard.org""" -certs_only_config = """ -smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem -smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem -""" +certs_only_config = ( +"""smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem +smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") def GetFakeOpen(fake_file_contents): @@ -65,9 +64,9 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): - return_vals = ('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', - '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - None) + return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', + '/etc/letsencrypt/live/www.fubard.org/privkey.pem', + 'tests/main.cf'),] postfix_config_gen = pcg.PostfixConfigGenerator( self.config, self.postfix_dir, From 7edceec8ac976ea5680ab081d8884ec6e6ff2a2c Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 15:27:11 -0700 Subject: [PATCH 121/639] Add test case and fix to properly handle configs with no smtpd_tls_* vars. --- letsencrypt-postfix/PostfixConfigGenerator.py | 12 ++++++++---- letsencrypt-postfix/TestPostfixConfigGenerator.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index b5adba1a2..859d86ffe 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -300,13 +300,17 @@ class PostfixConfigGenerator: 'smtpd_tls_cert_file': None, } for num, line in enumerate(self.cf): - print 'Line is: %s' % line num, found_var, found_value = parse_line((num, line)) if found_var in cert_materials.keys(): cert_materials[found_var] = found_value - return [(cert_materials['smtpd_tls_cert_file'], - cert_materials['smtpd_tls_key_file'], - self.fn),] + + if not all(cert_materials.values()): + cert_material_tuples = [] + else: + cert_material_tuples = [(cert_materials['smtpd_tls_cert_file'], + cert_materials['smtpd_tls_key_file'], + self.fn),] + return cert_material_tuples def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/letsencrypt-postfix/TestPostfixConfigGenerator.py index 57cbb59c0..4a96aa30f 100644 --- a/letsencrypt-postfix/TestPostfixConfigGenerator.py +++ b/letsencrypt-postfix/TestPostfixConfigGenerator.py @@ -46,6 +46,8 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.fopen_names_only_config = GetFakeOpen(names_only_config) self.fopen_certs_only_config = GetFakeOpen(certs_only_config) + self.fopen_no_certs_only_config = self.fopen_names_only_config + #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' @@ -75,6 +77,15 @@ class TestPostfixConfigGenerator(unittest.TestCase): ) self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + def testGetAllCertsAndKeys_With_None(self): + postfix_config_gen = pcg.PostfixConfigGenerator( + self.config, + self.postfix_dir, + fixup=True, + fopen=self.fopen_no_certs_only_config + ) + self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + if __name__ == '__main__': unittest.main() From 4d24eb83a82095b7a740263629bbc55f7972e0f9 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:11:37 -0700 Subject: [PATCH 122/639] Move version fetching into get_version and implement more_info method. --- letsencrypt-postfix/PostfixConfigGenerator.py | 57 ++++++++++++++++--- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 859d86ffe..0163e4bff 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -24,7 +24,12 @@ class ExistingConfigError(ValueError): pass class PostfixConfigGenerator: - def __init__(self, policy_config, postfix_dir, fixup=False, fopen=open): + def __init__(self, + policy_config, + postfix_dir, + fixup=False, + fopen=open, + version=None): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config @@ -41,6 +46,9 @@ class PostfixConfigGenerator: self.policy_lines = [] self.new_cf = "" + # Set in .prepare() unless running in a test + self.postfix_version = version + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -174,13 +182,14 @@ class PostfixConfigGenerator: """ # XXX ensure we raise the right kinds of exceptions - # Parse Postfix version number (feature support, syntax changes etc.) - mail_version = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], \ - stdout=subprocess.PIPE) \ - .communicate()[0].split()[2] - maj, min, rev = mail_version.split('.') - self.postfix_version = mail_version - + if not self.postfix_version: + self.postfix_version = self.get_version() + + if self.postfix_version < (2, 11, 0): + raise Exception( + 'NotSupportedError: Postfix version is too old -- test.' + ) + # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend # differences here and pass them onto other configuration functions. @@ -225,7 +234,28 @@ class PostfixConfigGenerator: # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html - return maj, min, rev + def get_version(self): + """Return the mail version of Postfix. + + Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + + :returns: version + :rtype: tuple + + :raises .PluginError: + Unable to find Postfix version. + """ + # Parse Postfix version number (feature support, syntax changes etc.) + cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + stdout=subprocess.PIPE) + stdout, _ = cmd.communicate() + if cmd.returncode != 0: + raise Exception('PluginError: Unable to determine Postfix version.') + + # grabs version component of string like "mail_version = 2.11.3" + mail_version = stdout.split()[2] + postfix_version = tuple([int(i) for i in mail_version.split('.')]) + return postfix_version def more_info(self): """Human-readable string to help the user. @@ -233,6 +263,15 @@ class PostfixConfigGenerator: decide which plugin to use. :rtype str: """ + return ( + "Configures Postfix to try to authenticate mail servers, use " + "installed certificates and disable weak ciphers and protocols.{0}" + "Server root: {root}{0}" + "Version: {version}".format( + os.linesep, + root=self.postfix_dir, + version='.'.join([str(i) for i in self.postfix_version])) + ) ### Let's Encrypt client IInstaller ### From c43602c90808d19e78bd2ee213f007e6ee7fc9ad Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:30:08 -0700 Subject: [PATCH 123/639] Add simple config_test implementation. --- letsencrypt-postfix/PostfixConfigGenerator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 0163e4bff..166206560 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -364,7 +364,6 @@ class PostfixConfigGenerator: be quickly reversed in the future (challenges) :raises .PluginError: when save is unsuccessful """ - self.maybe_add_config_lines() def rollback_checkpoints(self, rollback=1): @@ -389,6 +388,12 @@ class PostfixConfigGenerator: """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ + if os.geteuid() != 0: + rc = os.system('sudo /usr/sbin/postfix check') + else: + rc = os.system('/usr/sbin/postfix check') + if rc != 0: + raise Exception('MisconfigurationError: Postfix failed self-check.') def restart(self): """Restart or refresh the server content. From 5d07b702695f08dfd0fb35a59f74c411f756c22c Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:40:06 -0700 Subject: [PATCH 124/639] Change over to using logging module from print statements. --- letsencrypt-postfix/PostfixConfigGenerator.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 166206560..43a8fa116 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -1,10 +1,15 @@ #!/usr/bin/env python + +import logging import sys import string import subprocess import os, os.path +logger = logging.getLogger(__name__) + + def parse_line(line_data): """ Return the (line number, left hand side, right hand side) of a stripped @@ -114,13 +119,13 @@ class PostfixConfigGenerator: if not self.additions: return if self.fixup: - print "Deleting lines:", self.deletions + logger.info('Deleting lines: {}'.format(self.deletions)) self.additions[:0]=["#", "# New config lines added by STARTTLS Everywhere", "#"] new_cf_lines = "\n".join(self.additions) + "\n" - print "Adding to %s:" % self.fn - print new_cf_lines + logger.info('Adding to {}:'.format(self.fn)) + logger.info(new_cf_lines) if self.raw_cf[-1][-1] == "\n": sep = "" else: sep = "\n" @@ -142,9 +147,12 @@ class PostfixConfigGenerator: for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains if len(mx_list) > 1: - print "Lists of multiple accept-mx-domains not yet supported." - print "Using MX %s for %s" % (mx_list[0], address_domain) - print "Ignoring: %s" % (', '.join(mx_list[1:])) + logger.warn('Lists of multiple accept-mx-domains not yet ' + 'supported.') + logger.warn('Using MX {} for {}".format(mx_list[0], + address_domain) + ) + logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) mx_domain = mx_list[0] mx_policy = self.policy_config.get_tls_policy(mx_domain) entry = address_domain + " encrypt" @@ -155,7 +163,9 @@ class PostfixConfigGenerator: elif mx_policy.min_tls_version.lower() == "tlsv1.2": entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" else: - print mx_policy.min_tls_version + logger.warn('Unknown minimum TLS version: {} '.format( + mx_policy.min_tls_version) + ) self.policy_lines.append(entry) with fopen(self.policy_file, "w") as f: @@ -399,7 +409,7 @@ class PostfixConfigGenerator: """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted """ - print "Reloading postfix config..." + logger.info('Reloading postfix config...') if os.geteuid() != 0: rc = os.system("sudo service postfix reload") else: From 887871833d407f4597066ca18c07a0c204a4e097 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 16:44:25 -0700 Subject: [PATCH 125/639] Fix typo in changing quotes. --- letsencrypt-postfix/PostfixConfigGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 43a8fa116..214d98954 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -149,7 +149,7 @@ class PostfixConfigGenerator: if len(mx_list) > 1: logger.warn('Lists of multiple accept-mx-domains not yet ' 'supported.') - logger.warn('Using MX {} for {}".format(mx_list[0], + logger.warn('Using MX {} for {}'.format(mx_list[0], address_domain) ) logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) From af38c30c9c506cb6e40cd4b1b32103d964ab9bee Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 17:02:29 -0700 Subject: [PATCH 126/639] Fix path to postfix config variable. --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 214d98954..8fcfb7e8b 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -136,9 +136,9 @@ class PostfixConfigGenerator: self.new_cf += line self.new_cf += sep + new_cf_lines - if not os.access(self.postfix_cf_file, os.W_OK): + if not os.access(self.fn, os.W_OK): raise Exception("Can't write to %s, please re-run as root." - % self.postfix_cf_file) + % self.fn) with fopen(self.fn, "w") as f: f.write(self.new_cf) From a5f23b5314c6afbac26c27e494d3d92603d777e1 Mon Sep 17 00:00:00 2001 From: Daniel Wilcox Date: Thu, 28 Apr 2016 17:10:21 -0700 Subject: [PATCH 127/639] Configure logger to be a touch louder... than silent --- letsencrypt-postfix/PostfixConfigGenerator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/letsencrypt-postfix/PostfixConfigGenerator.py index 8fcfb7e8b..38ef0f4c9 100755 --- a/letsencrypt-postfix/PostfixConfigGenerator.py +++ b/letsencrypt-postfix/PostfixConfigGenerator.py @@ -8,6 +8,10 @@ import os, os.path logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +log_handler = logging.StreamHandler() +log_handler.setLevel(logging.DEBUG) +logger.addHandler(log_handler) def parse_line(line_data): From 619e273ae5ef6800c584cbe1ef9160fb6045a1e6 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Wed, 10 May 2017 15:44:55 +0200 Subject: [PATCH 128/639] Correct markdown link syntax --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 774556e7e..7413228f6 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,7 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d ## Motivating examples * [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to - prevent negotiating an encrypted connection] - (https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) + prevent negotiating an encrypted connection](https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) * [Unknown party removes STARTTLS flag from all SMTP connections leaving Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) From e2d95b371992247ba5806c5b107b5449047e8c8d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:02:56 -0700 Subject: [PATCH 129/639] Create packaging around PostfixConfigGenerator. --- certbot-postfix/LICENSE.txt | 190 ++++++++++++++++++ certbot-postfix/MANIFEST.in | 2 + certbot-postfix/README.rst | 1 + certbot-postfix/certbot_postfix/__init__.py | 3 + .../certbot_postfix/installer.py | 0 .../certbot_postfix/installer_test.py | 0 certbot-postfix/setup.cfg | 2 + certbot-postfix/setup.py | 57 ++++++ 8 files changed, 255 insertions(+) create mode 100644 certbot-postfix/LICENSE.txt create mode 100644 certbot-postfix/MANIFEST.in create mode 100644 certbot-postfix/README.rst create mode 100644 certbot-postfix/certbot_postfix/__init__.py rename letsencrypt-postfix/PostfixConfigGenerator.py => certbot-postfix/certbot_postfix/installer.py (100%) mode change 100755 => 100644 rename letsencrypt-postfix/TestPostfixConfigGenerator.py => certbot-postfix/certbot_postfix/installer_test.py (100%) create mode 100644 certbot-postfix/setup.cfg create mode 100644 certbot-postfix/setup.py diff --git a/certbot-postfix/LICENSE.txt b/certbot-postfix/LICENSE.txt new file mode 100644 index 000000000..c8314fd1c --- /dev/null +++ b/certbot-postfix/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2017 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-postfix/MANIFEST.in b/certbot-postfix/MANIFEST.in new file mode 100644 index 000000000..97e2ad3df --- /dev/null +++ b/certbot-postfix/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE.txt +include README.rst diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst new file mode 100644 index 000000000..ee88648d3 --- /dev/null +++ b/certbot-postfix/README.rst @@ -0,0 +1 @@ +Postfix plugin for Certbot diff --git a/certbot-postfix/certbot_postfix/__init__.py b/certbot-postfix/certbot_postfix/__init__.py new file mode 100644 index 000000000..703cb659b --- /dev/null +++ b/certbot-postfix/certbot_postfix/__init__.py @@ -0,0 +1,3 @@ +"""Certbot Postfix plugin.""" + +from certbot_postfix.installer import PostfixConfigGenerator as Installer diff --git a/letsencrypt-postfix/PostfixConfigGenerator.py b/certbot-postfix/certbot_postfix/installer.py old mode 100755 new mode 100644 similarity index 100% rename from letsencrypt-postfix/PostfixConfigGenerator.py rename to certbot-postfix/certbot_postfix/installer.py diff --git a/letsencrypt-postfix/TestPostfixConfigGenerator.py b/certbot-postfix/certbot_postfix/installer_test.py similarity index 100% rename from letsencrypt-postfix/TestPostfixConfigGenerator.py rename to certbot-postfix/certbot_postfix/installer_test.py diff --git a/certbot-postfix/setup.cfg b/certbot-postfix/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-postfix/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py new file mode 100644 index 000000000..df78948bb --- /dev/null +++ b/certbot-postfix/setup.py @@ -0,0 +1,57 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.18.0.dev0' + +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', +] + +setup( + name='certbot-postfix', + version=version, + description="Postfix plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Communications :: Email :: Mail Transport Agents', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + entry_points={ + 'certbot.plugins': [ + 'postfix = certbot_postfix:Installer', + ], + }, + test_suite='certbot_postfix', +) From 74b22a596e07e76626a014dac68a90fb7e50e30e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:03:30 -0700 Subject: [PATCH 130/639] Ignore egg-info dirs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cc957df18..e36c9c50b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .* *.orig *.pyc +*.egg-info/ From f89051cc2ad7ddd514cacf873d0758fece796c06 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:18:51 -0700 Subject: [PATCH 131/639] Completely implement the Certbot plugin interfaces --- certbot-postfix/certbot_postfix/installer.py | 12 +++++++++++- certbot-postfix/setup.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d660e35f5..a4500cf24 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -6,6 +6,11 @@ import string import subprocess import os, os.path +import zope.interface + +from certbot import interfaces +from certbot.plugins import common as plugins_common + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -32,7 +37,12 @@ def parse_line(line_data): class ExistingConfigError(ValueError): pass -class PostfixConfigGenerator: +@zope.interface.implementer(interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) +class PostfixConfigGenerator(plugins_common.Plugin): + + description = "Configure TLS with the Postfix MTA" + def __init__(self, policy_config, postfix_dir, diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index df78948bb..1f087ccc0 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -12,6 +12,7 @@ install_requires = [ # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'zope.interface', ] setup( From ae08dc6beaab6e6d090de85e710e2d913a0cd1aa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:24:04 -0700 Subject: [PATCH 132/639] Fix Postfix installer tests --- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 4a96aa30f..cc86653f6 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -10,8 +10,7 @@ import io import logging import unittest -import Config -import PostfixConfigGenerator as pcg +import certbot_postfix logger = logging.getLogger(__name__) @@ -57,7 +56,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, @@ -69,7 +68,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, @@ -78,7 +77,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = pcg.PostfixConfigGenerator( + postfix_config_gen = certbot_postfix.Installer( self.config, self.postfix_dir, fixup=True, From 5bf4ad1f52b28badc6a0dd2d82f7d987d6cb184b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:25:12 -0700 Subject: [PATCH 133/639] Rename PostfixConfigGenerator to simply Installer --- certbot-postfix/certbot_postfix/__init__.py | 2 +- certbot-postfix/certbot_postfix/installer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/__init__.py b/certbot-postfix/certbot_postfix/__init__.py index 703cb659b..122c54bc6 100644 --- a/certbot-postfix/certbot_postfix/__init__.py +++ b/certbot-postfix/certbot_postfix/__init__.py @@ -1,3 +1,3 @@ """Certbot Postfix plugin.""" -from certbot_postfix.installer import PostfixConfigGenerator as Installer +from certbot_postfix.installer import Installer diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index a4500cf24..ac2adbdf0 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -39,7 +39,7 @@ class ExistingConfigError(ValueError): pass @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) -class PostfixConfigGenerator(plugins_common.Plugin): +class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" From c2a8ce59ae298b80cdc334f2bb62d74629c45043 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:28:22 -0700 Subject: [PATCH 134/639] Remove code to run the installer as on its own. --- certbot-postfix/certbot_postfix/installer.py | 34 -------------------- 1 file changed, 34 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index ac2adbdf0..52244ce92 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import sys import string @@ -13,10 +11,6 @@ from certbot.plugins import common as plugins_common logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -log_handler = logging.StreamHandler() -log_handler.setLevel(logging.DEBUG) -logger.addHandler(log_handler) def parse_line(line_data): @@ -435,31 +429,3 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - - -def usage(): - print ("Usage: %s starttls-everywhere.json /etc/postfix " - "/etc/letsencrypt/live/example.com/" % sys.argv[0]) - sys.exit(1) - - -if __name__ == "__main__": - import Config as config - if len(sys.argv) != 4: - usage() - c = config.Config() - c.load_from_json_file(sys.argv[1]) - postfix_dir = sys.argv[2] - le_lineage = sys.argv[3] - pieces = [os.path.join(le_lineage, f) for f in ( - "cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] - if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) : - print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" - print - usage() - cert, key, chain, fullchain = pieces - pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) - pcgen.prepare() - pcgen.deploy_cert("example.com", cert, key, chain, fullchain) - pcgen.save() - pcgen.restart() From 6c4b3c08a7ce8e5453fcafcb9333fca043951e7d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:30:11 -0700 Subject: [PATCH 135/639] Clean up installer imports --- certbot-postfix/certbot_postfix/installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 52244ce92..0a109e1de 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,8 +1,8 @@ import logging -import sys +import os import string import subprocess -import os, os.path +import sys import zope.interface From 1c258c0a2c382a3a404091d1b099c19a9bd3b8ef Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:31:10 -0700 Subject: [PATCH 136/639] Add basic docstrings to Installer --- certbot-postfix/certbot_postfix/installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 0a109e1de..cd9c00714 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,3 +1,4 @@ +"""Certbot installer plugin for Postfix.""" import logging import os import string @@ -34,6 +35,7 @@ class ExistingConfigError(ValueError): pass @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): + """Certbot installer plugin for Postfix.""" description = "Configure TLS with the Postfix MTA" From 694746409f45904082f635557a933b900e34e901 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:42:26 -0700 Subject: [PATCH 137/639] s/ExistingConfigError/MisconfigurationError --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index cd9c00714..b0a075eeb 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -7,6 +7,7 @@ import sys import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import common as plugins_common @@ -29,9 +30,6 @@ def parse_line(line_data): return (num, left.strip(), right.strip()) -class ExistingConfigError(ValueError): pass - - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): @@ -72,6 +70,10 @@ class Installer(plugins_common.Plugin): """ Ensure that existing postfix config @var is in the list of @acceptable values; if not, set it to the ideal value. + + :raises .errors.MisconfigurationError: if conflicting existing values + are found for var + """ acceptable = [ideal] + also_acceptable @@ -87,8 +89,8 @@ class Installer(plugins_common.Plugin): self.deletions.extend(conflicting_lines) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError( - "Conflicting existing config values " + `l` + raise errors.MisconfigurationError( + "Conflicting existing config values {0}".format(l) ) val = values[0][2] if val not in acceptable: @@ -96,7 +98,7 @@ class Installer(plugins_common.Plugin): self.deletions.append(values[0][0]) self.additions.append(var + " = " + ideal) else: - raise ExistingConfigError( + raise errors.MisconfigurationError( "Existing config has %s=%s"%(var,val) ) From 61c2209110292c84d18b0bd5087ba783d3269b25 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:56:39 -0700 Subject: [PATCH 138/639] Use Certbot error types in the Postfix Installer --- certbot-postfix/certbot_postfix/installer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index b0a075eeb..00491e735 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -210,9 +210,7 @@ class Installer(plugins_common.Plugin): self.postfix_version = self.get_version() if self.postfix_version < (2, 11, 0): - raise Exception( - 'NotSupportedError: Postfix version is too old -- test.' - ) + raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -274,7 +272,7 @@ class Installer(plugins_common.Plugin): stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: - raise Exception('PluginError: Unable to determine Postfix version.') + raise errors.PluginError('Unable to determine Postfix version.') # grabs version component of string like "mail_version = 2.11.3" mail_version = stdout.split()[2] @@ -417,7 +415,7 @@ class Installer(plugins_common.Plugin): else: rc = os.system('/usr/sbin/postfix check') if rc != 0: - raise Exception('MisconfigurationError: Postfix failed self-check.') + raise errors.MisconfigurationError('Postfix failed self-check.') def restart(self): """Restart or refresh the server content. @@ -429,7 +427,7 @@ class Installer(plugins_common.Plugin): else: rc = os.system("service postfix reload") if rc != 0: - raise Exception('PluginError: cannot restart postfix') + raise errors.MisconfigurationError('cannot restart postfix') def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) From 66ba0b5276b63ccdfc0c986f1c751d214f41f8b7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 09:57:44 -0700 Subject: [PATCH 139/639] Remove invalid permissions exception. Once things like locks are added, this error shouldn't be possible as it will have occurred earlier. --- certbot-postfix/certbot_postfix/installer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 00491e735..5196e1e64 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -150,9 +150,6 @@ class Installer(plugins_common.Plugin): self.new_cf += line self.new_cf += sep + new_cf_lines - if not os.access(self.fn, os.W_OK): - raise Exception("Can't write to %s, please re-run as root." - % self.fn) with fopen(self.fn, "w") as f: f.write(self.new_cf) From 4a3fd19c93b8dc5543859459d920662138274156 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:00:04 -0700 Subject: [PATCH 140/639] Move parse_line to the end of installer.py --- certbot-postfix/certbot_postfix/installer.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 5196e1e64..da573baaa 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -15,21 +15,6 @@ from certbot.plugins import common as plugins_common logger = logging.getLogger(__name__) -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) - - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Plugin): @@ -428,3 +413,18 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + + +def parse_line(line_data): + """ + Return the (line number, left hand side, right hand side) of a stripped + postfix config line. + + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + num,line = line_data + left, sep, right = line.partition("=") + if not sep: + return None + return (num, left.strip(), right.strip()) From b37be61807159443442f1df738c1caf16849a40c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:12:54 -0700 Subject: [PATCH 141/639] Import installer module directly in tests. --- certbot-postfix/certbot_postfix/installer_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index cc86653f6..7ee40a955 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -10,7 +10,7 @@ import io import logging import unittest -import certbot_postfix +from certbot_postfix import installer logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, @@ -68,7 +68,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, @@ -77,7 +77,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = certbot_postfix.Installer( + postfix_config_gen = installer.Installer( self.config, self.postfix_dir, fixup=True, From b50a71ff4e0b95dd9e491ea89bf2ce1bb1cd7bd6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:19:46 -0700 Subject: [PATCH 142/639] Remove fopen argument in favor of mock. This simplifies the actual production code and is a more standard approach in Python. --- certbot-postfix/certbot_postfix/installer.py | 11 ++-- .../certbot_postfix/installer_test.py | 63 +++++++------------ 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index da573baaa..96fc1b4d1 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -26,7 +26,6 @@ class Installer(plugins_common.Plugin): policy_config, postfix_dir, fixup=False, - fopen=open, version=None): self.fixup = fixup self.postfix_dir = postfix_dir @@ -38,7 +37,7 @@ class Installer(plugins_common.Plugin): self.additions = [] self.deletions = [] self.fn = self.find_postfix_cf() - self.raw_cf = fopen(self.fn).readlines() + self.raw_cf = open(self.fn).readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] self.policy_lines = [] @@ -114,7 +113,7 @@ class Installer(plugins_common.Plugin): self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - def maybe_add_config_lines(self, fopen=open): + def maybe_add_config_lines(self): if not self.additions: return if self.fixup: @@ -135,10 +134,10 @@ class Installer(plugins_common.Plugin): self.new_cf += line self.new_cf += sep + new_cf_lines - with fopen(self.fn, "w") as f: + with open(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self, fopen=open): + def set_domainwise_tls_policies(self): all_acceptable_mxs = self.policy_config.acceptable_mxs for address_domain, properties in all_acceptable_mxs.items(): mx_list = properties.accept_mx_domains @@ -164,7 +163,7 @@ class Installer(plugins_common.Plugin): ) self.policy_lines.append(entry) - with fopen(self.policy_file, "w") as f: + with open(self.policy_file, "w") as f: f.write("\n".join(self.policy_lines) + "\n") ### Let's Encrypt client IPlugin ### diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 7ee40a955..93aa171a9 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -6,10 +6,12 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import io import logging import unittest +import mock +import six + from certbot_postfix import installer @@ -28,61 +30,44 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -def GetFakeOpen(fake_file_contents): - fake_file = io.StringIO() - # cast this to unicode for py2 - fake_file.write(fake_file_contents) - fake_file.seek(0) - - def FakeOpen(_): - return fake_file - - return FakeOpen - - class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): - self.fopen_names_only_config = GetFakeOpen(names_only_config) - self.fopen_certs_only_config = GetFakeOpen(certs_only_config) - self.fopen_no_certs_only_config = self.fopen_names_only_config - - #self.config = Config.Config() self.config = None self.postfix_dir = 'tests/' - def tearDown(self): - pass - def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_names_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(names_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_certs_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(certs_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): - postfix_config_gen = installer.Installer( - self.config, - self.postfix_dir, - fixup=True, - fopen=self.fopen_no_certs_only_config - ) + with mock.patch('certbot_postfix.installer.open') as mock_open: + mock_open.return_value = six.StringIO(names_only_config) + postfix_config_gen = installer.Installer( + self.config, + self.postfix_dir, + fixup=True, + ) self.assertEqual([], postfix_config_gen.get_all_certs_keys()) From 49cdfcec06564a9c56b1f592225b8600d0417cda Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:20:36 -0700 Subject: [PATCH 143/639] add six dependency --- certbot-postfix/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 1f087ccc0..2fdcd46aa 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -12,6 +12,7 @@ install_requires = [ # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'six', 'zope.interface', ] From a66500ea386e16d88b858469707436f767c3fa18 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:25:09 -0700 Subject: [PATCH 144/639] Remove unused version argument. --- certbot-postfix/certbot_postfix/installer.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 96fc1b4d1..7fa32b8f5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -25,8 +25,7 @@ class Installer(plugins_common.Plugin): def __init__(self, policy_config, postfix_dir, - fixup=False, - version=None): + fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir self.policy_config = policy_config @@ -43,9 +42,6 @@ class Installer(plugins_common.Plugin): self.policy_lines = [] self.new_cf = "" - # Set in .prepare() unless running in a test - self.postfix_version = version - def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -187,10 +183,7 @@ class Installer(plugins_common.Plugin): """ # XXX ensure we raise the right kinds of exceptions - if not self.postfix_version: - self.postfix_version = self.get_version() - - if self.postfix_version < (2, 11, 0): + if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions @@ -273,7 +266,7 @@ class Installer(plugins_common.Plugin): "Version: {version}".format( os.linesep, root=self.postfix_dir, - version='.'.join([str(i) for i in self.postfix_version])) + version='.'.join([str(i) for i in self.get_version()])) ) From 6c5a8423b865881dddc15a1ee7b60c69d42faf19 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Aug 2017 10:28:52 -0700 Subject: [PATCH 145/639] Remove unused logger from tests --- certbot-postfix/certbot_postfix/installer_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 93aa171a9..db35511ac 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -15,10 +15,6 @@ import six from certbot_postfix import installer -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org From a15fe57225e06c9c00b3d21b2b2176eb0027038f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Aug 2017 15:57:19 -0700 Subject: [PATCH 146/639] remove policy config param --- certbot-postfix/certbot_postfix/installer.py | 60 +++++++++---------- .../certbot_postfix/installer_test.py | 4 -- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 7fa32b8f5..a6c1cd9fe 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -23,12 +23,10 @@ class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" def __init__(self, - policy_config, postfix_dir, fixup=False): self.fixup = fixup self.postfix_dir = postfix_dir - self.policy_config = policy_config self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") @@ -133,35 +131,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - def set_domainwise_tls_policies(self): - all_acceptable_mxs = self.policy_config.acceptable_mxs - for address_domain, properties in all_acceptable_mxs.items(): - mx_list = properties.accept_mx_domains - if len(mx_list) > 1: - logger.warn('Lists of multiple accept-mx-domains not yet ' - 'supported.') - logger.warn('Using MX {} for {}'.format(mx_list[0], - address_domain) - ) - logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) - mx_domain = mx_list[0] - mx_policy = self.policy_config.get_tls_policy(mx_domain) - entry = address_domain + " encrypt" - if mx_policy.min_tls_version.lower() == "tlsv1": - entry += " protocols=!SSLv2:!SSLv3" - elif mx_policy.min_tls_version.lower() == "tlsv1.1": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - elif mx_policy.min_tls_version.lower() == "tlsv1.2": - entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - else: - logger.warn('Unknown minimum TLS version: {} '.format( - mx_policy.min_tls_version) - ) - self.policy_lines.append(entry) - - with open(self.policy_file, "w") as f: - f.write("\n".join(self.policy_lines) + "\n") - ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 @@ -406,6 +375,35 @@ class Installer(plugins_common.Plugin): def update_CAfile(self): os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + # def set_domainwise_tls_policies(self): + # all_acceptable_mxs = self.policy_config.acceptable_mxs + # for address_domain, properties in all_acceptable_mxs.items(): + # mx_list = properties.accept_mx_domains + # if len(mx_list) > 1: + # logger.warn('Lists of multiple accept-mx-domains not yet ' + # 'supported.') + # logger.warn('Using MX {} for {}'.format(mx_list[0], + # address_domain) + # ) + # logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) + # mx_domain = mx_list[0] + # mx_policy = self.policy_config.get_tls_policy(mx_domain) + # entry = address_domain + " encrypt" + # if mx_policy.min_tls_version.lower() == "tlsv1": + # entry += " protocols=!SSLv2:!SSLv3" + # elif mx_policy.min_tls_version.lower() == "tlsv1.1": + # entry += " protocols=!SSLv2:!SSLv3:!TLSv1" + # elif mx_policy.min_tls_version.lower() == "tlsv1.2": + # entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" + # else: + # logger.warn('Unknown minimum TLS version: {} '.format( + # mx_policy.min_tls_version) + # ) + # self.policy_lines.append(entry) + + # with open(self.policy_file, "w") as f: + # f.write("\n".join(self.policy_lines) + "\n") + def parse_line(line_data): """ diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index db35511ac..04d86468e 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -29,7 +29,6 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): - self.config = None self.postfix_dir = 'tests/' def testGetAllNames(self): @@ -37,7 +36,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) @@ -50,7 +48,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) @@ -60,7 +57,6 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = installer.Installer( - self.config, self.postfix_dir, fixup=True, ) From d97a15861bc96eb7638bb9af80bdf105f47af9bc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Aug 2017 16:06:06 -0700 Subject: [PATCH 147/639] Add --postfix-config-dir argument --- certbot-postfix/certbot_postfix/installer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index a6c1cd9fe..6175c06f3 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -22,6 +22,12 @@ class Installer(plugins_common.Plugin): description = "Configure TLS with the Postfix MTA" + @classmethod + def add_parser_arguments(cls, add): + add("config-dir", help="Path to the directory containing the " + "Postfix main.cf file to modify instead of using the " + "default configuration paths") + def __init__(self, postfix_dir, fixup=False): From 481fb8413b6f6a68a403e2080834abfdf589aea2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 08:50:08 -0700 Subject: [PATCH 148/639] Fix Postfix Installer __init__() --- certbot-postfix/certbot_postfix/installer.py | 13 ++++----- .../certbot_postfix/installer_test.py | 29 +++++++++++-------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 6175c06f3..5f9fe7322 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -28,14 +28,13 @@ class Installer(plugins_common.Plugin): "Postfix main.cf file to modify instead of using the " "default configuration paths") - def __init__(self, - postfix_dir, - fixup=False): - self.fixup = fixup - self.postfix_dir = postfix_dir - self.policy_file = os.path.join(postfix_dir, + def __init__(self, *args, **kwargs): + super(Installer, self).__init__(*args, **kwargs) + self.fixup = False + self.postfix_dir = self.conf("config-dir") + self.policy_file = os.path.join(self.postfix_dir, "starttls_everywhere_policy") - self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") + self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] self.deletions = [] diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 04d86468e..b9e9e1463 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -35,10 +35,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): @@ -47,21 +44,29 @@ class TestPostfixConfigGenerator(unittest.TestCase): 'tests/main.cf'),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = installer.Installer( - self.postfix_dir, - fixup=True, - ) + postfix_config_gen = self._create_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def _create_installer(self): + """Creates and returns a new Postfix Installer. + + :returns: a new Postfix installer + :rtype: certbot_postfix.installer.Installer + + """ + config = mock.MagicMock(postfix_config_dir=self.postfix_dir) + name = "postfix" + + from certbot_postfix import installer + return installer.Installer(config, name) + + if __name__ == '__main__': unittest.main() From 2a217189a6cbf9768bcdc9a4783cc1340caf500b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 10:58:27 -0700 Subject: [PATCH 149/639] (temporarily) remove policy_file --- certbot-postfix/certbot_postfix/installer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 5f9fe7322..c3bf2a189 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -32,8 +32,6 @@ class Installer(plugins_common.Plugin): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.postfix_dir = self.conf("config-dir") - self.policy_file = os.path.join(self.postfix_dir, - "starttls_everywhere_policy") self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] @@ -97,9 +95,9 @@ class Installer(plugins_common.Plugin): # Maximum verbosity lets us collect failure information self.ensure_cf_var("smtp_tls_loglevel", "1", []) # Inject a reference to our per-domain policy map - policy_cf_entry = "texthash:" + self.policy_file + # policy_cf_entry = "texthash:" + self.policy_file - self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed From 89ae874f890bde600f12e52ae56508631225f5f1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:03:47 -0700 Subject: [PATCH 150/639] (temporarily) remove ca-certificates logic --- certbot-postfix/certbot_postfix/installer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index c3bf2a189..2663e44fd 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -32,7 +32,6 @@ class Installer(plugins_common.Plugin): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.postfix_dir = self.conf("config-dir") - self.ca_file = os.path.join(self.postfix_dir, "starttls_everywhere_CAfile") self.additions = [] self.deletions = [] @@ -98,7 +97,7 @@ class Installer(plugins_common.Plugin): # policy_cf_entry = "texthash:" + self.policy_file # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) + # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed # between Postfix version 2.5 and 2.6, since we only support => 2.11 @@ -375,9 +374,9 @@ class Installer(plugins_common.Plugin): if rc != 0: raise errors.MisconfigurationError('cannot restart postfix') - def update_CAfile(self): - os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - + # def update_CAfile(self): + # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) + # # def set_domainwise_tls_policies(self): # all_acceptable_mxs = self.policy_config.acceptable_mxs # for address_domain, properties in all_acceptable_mxs.items(): From 86fe5ad3623d590b3f93212a712b38a9f8234bdd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:16:46 -0700 Subject: [PATCH 151/639] Move calls to postconf to prepare(). --- certbot-postfix/certbot_postfix/installer.py | 11 +++++--- .../certbot_postfix/installer_test.py | 28 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 2663e44fd..43037e886 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -35,12 +35,11 @@ class Installer(plugins_common.Plugin): self.additions = [] self.deletions = [] - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] self.policy_lines = [] self.new_cf = "" + self.fn = None + self.raw_cf = [] + self.cf = [] def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -152,6 +151,10 @@ class Installer(plugins_common.Plugin): currently supported. :rtype tuple: """ + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] # XXX ensure we raise the right kinds of exceptions if self.get_version() < (2, 11, 0): diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index b9e9e1463..9be3744e3 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -12,9 +12,6 @@ import unittest import mock import six -from certbot_postfix import installer - - # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org @@ -35,7 +32,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) def testGetAllCertAndKeys(self): @@ -44,15 +41,34 @@ class TestPostfixConfigGenerator(unittest.TestCase): 'tests/main.cf'),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) def testGetAllCertsAndKeys_With_None(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_installer() + postfix_config_gen = self._create_prepared_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def _create_prepared_installer(self): + """Creates and returns a new prepared Postfix Installer. + + Calls in prepare() are mocked out so the Postfix version check + is successful. + + :returns: a prepared Postfix installer + :rtype: certbot_postfix.installer.Installer + + """ + installer = self._create_installer() + + popen_path = "certbot_postfix.installer.subprocess.Popen" + with mock.patch(popen_path) as mock_popen: + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + installer.prepare() + + return installer def _create_installer(self): """Creates and returns a new Postfix Installer. From a2dbf2fe4cc03750c1d9dfb2f4b4d185ce6ed104 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:20:21 -0700 Subject: [PATCH 152/639] Fix spacing --- certbot-postfix/certbot_postfix/installer.py | 119 ++++++++++--------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 43037e886..aade01f9b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -98,15 +98,15 @@ class Installer(plugins_common.Plugin): # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed - # between Postfix version 2.5 and 2.6, since we only support => 2.11 - # we don't use nor support legacy Postfix syntax. - # - Server: - self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - # - Client: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) + # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed + # between Postfix version 2.5 and 2.6, since we only support => 2.11 + # we don't use nor support legacy Postfix syntax. + # - Server: + self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) + # - Client: + self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) + self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) def maybe_add_config_lines(self): if not self.additions: @@ -137,7 +137,9 @@ class Installer(plugins_common.Plugin): def prepare(self): """Prepare the plugin. + Finish up any additional initialization. + :raises .PluginError: when full initialization cannot be completed. :raises .MisconfigurationError: @@ -146,11 +148,12 @@ class Installer(plugins_common.Plugin): :raises .NoInstallationError: when the necessary programs/files cannot be located. Plugin will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: - """ + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + + """ self.fn = self.find_postfix_cf() self.raw_cf = open(self.fn).readlines() self.cf = map(string.strip, self.raw_cf) @@ -160,49 +163,49 @@ class Installer(plugins_common.Plugin): if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html def get_version(self): """Return the mail version of Postfix. @@ -212,12 +215,12 @@ class Installer(plugins_common.Plugin): :returns: version :rtype: tuple - :raises .PluginError: - Unable to find Postfix version. + :raises .PluginError: Unable to find Postfix version. + """ - # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], - stdout=subprocess.PIPE) + # Parse Postfix version number (feature support, syntax changes etc.) + cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: raise errors.PluginError('Unable to determine Postfix version.') @@ -225,7 +228,7 @@ class Installer(plugins_common.Plugin): # grabs version component of string like "mail_version = 2.11.3" mail_version = stdout.split()[2] postfix_version = tuple([int(i) for i in mail_version.split('.')]) - return postfix_version + return postfix_version def more_info(self): """Human-readable string to help the user. From b395b72d1ba6704ea26489840645858001976714 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 11:51:01 -0700 Subject: [PATCH 153/639] Don't hardcode postconf path. --- certbot-postfix/certbot_postfix/installer.py | 165 ++++++++++-------- .../certbot_postfix/installer_test.py | 10 +- 2 files changed, 99 insertions(+), 76 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index aade01f9b..394ed5525 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -9,7 +9,9 @@ import zope.interface from certbot import errors from certbot import interfaces +from certbot import util from certbot.plugins import common as plugins_common +from certbot.plugins import util as plugins_util logger = logging.getLogger(__name__) @@ -27,6 +29,8 @@ class Installer(plugins_common.Plugin): add("config-dir", help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " "default configuration paths") + add("config-utility", default="postconf", + help="Path to the 'postconf' executable.") def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) @@ -41,6 +45,94 @@ class Installer(plugins_common.Plugin): self.raw_cf = [] self.cf = [] + def prepare(self): + """Prepare the installer. + + Finish up any additional initialization. + + :raises .PluginError: + when full initialization cannot be completed. + :raises .MisconfigurationError: + when full initialization cannot be completed. Plugin will + be displayed on a list of available plugins. + :raises .NoInstallationError: + when the necessary programs/files cannot be located. Plugin + will NOT be displayed on a list of available plugins. + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. + :rtype tuple: + + """ + self._verify_postconf_available() + + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + # XXX ensure we raise the right kinds of exceptions + + if self.get_version() < (2, 11, 0): + raise errors.NotSupportedError('Postfix version is too old') + + # Postfix has changed support for TLS features, supported protocol versions + # KEX methods, ciphers et cetera over the years. We sort out version dependend + # differences here and pass them onto other configuration functions. + # see: + # http://www.postfix.org/TLS_README.html + # http://www.postfix.org/FORWARD_SECRECY_README.html + + # Postfix == 2.2: + # - TLS support introduced via 3rd party patch, see: + # http://www.postfix.org/TLS_LEGACY_README.html + + # Postfix => 2.2: + # - built-in TLS support added + # - Support for PFS introduced + # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k + + # Postfix => 2.5: + # - Syntax to specify mandatory protocol version changes: + # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` + # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` + # - Certificate fingerprint verification added + + # Postfix => 2.6: + # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) + # - Support for configurable cipher-suites and protocol versions added, pre-2.6 + # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` + # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added + + # Postfix => 2.8: + # - Override Client suite preference w. `tls_preempt_cipherlist = yes` + # - Elliptic curve crypto. support enabled by default + + # Postfix => 2.9: + # - Public key fingerprint support added + # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and + # `check_ccert_access` config. options added + + # Postfix <= 2.9.5: + # - BUG: Public key fingerprint is computed incorrectly + + # Postfix => 3.1: + # - Built-in support for TLS management and DANE added, see: + # http://www.postfix.org/postfix-tls.1.html + + def _verify_postconf_available(self): + """Ensure 'postconf' can be found. + + :raises .NoInstallationError: when unable to find 'postconf' + + """ + if not util.exe_exists(self.conf("config-utility")): + if not plugins_util.path_surgery(self.conf("config-utility")): + raise errors.NoInstallationError( + "Cannot find executable '{0}'. You can provide the " + "path to this command with --{1}".format( + self.conf("config-utility"), + self.option_name("config-utility"))) + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.postfix_dir, "main.cf") @@ -135,77 +227,6 @@ class Installer(plugins_common.Plugin): ### Let's Encrypt client IPlugin ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def prepare(self): - """Prepare the plugin. - - Finish up any additional initialization. - - :raises .PluginError: - when full initialization cannot be completed. - :raises .MisconfigurationError: - when full initialization cannot be completed. Plugin will - be displayed on a list of available plugins. - :raises .NoInstallationError: - when the necessary programs/files cannot be located. Plugin - will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: - - """ - self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - # XXX ensure we raise the right kinds of exceptions - - if self.get_version() < (2, 11, 0): - raise errors.NotSupportedError('Postfix version is too old') - - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html - - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added - - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added - - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly - - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html def get_version(self): """Return the mail version of Postfix. @@ -219,7 +240,7 @@ class Installer(plugins_common.Plugin): """ # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], + cmd = subprocess.Popen([self.conf('config-utility'), '-d', 'mail_version'], stdout=subprocess.PIPE) stdout, _ = cmd.communicate() if cmd.returncode != 0: diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9be3744e3..332c6ec60 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -62,11 +62,13 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() + exe_exists_path = "certbot_postfix.installer.util.exe_exists" popen_path = "certbot_postfix.installer.subprocess.Popen" - with mock.patch(popen_path) as mock_popen: - mock_popen().returncode = 0 - mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") - installer.prepare() + with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: + with mock.patch(popen_path) as mock_popen: + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + installer.prepare() return installer From 192f0f60daadac4a44541b10aa90181b4156a248 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 12:19:59 -0700 Subject: [PATCH 154/639] test add_parser_arguments --- certbot-postfix/certbot_postfix/installer_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 332c6ec60..b3e8f560d 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,6 +28,15 @@ class TestPostfixConfigGenerator(unittest.TestCase): def setUp(self): self.postfix_dir = 'tests/' + def test_add_parser_arguments(self): + mock_add = mock.MagicMock() + + from certbot_postfix import installer + installer.Installer.add_parser_arguments(mock_add) + + for call in mock_add.call_args_list: + self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: From dfd1cceb9b5e18d646ef55d0b12b561712510379 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 12:26:47 -0700 Subject: [PATCH 155/639] Test prepare() failure due to missing postconf --- certbot-postfix/certbot_postfix/installer_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index b3e8f560d..721790797 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -12,6 +12,8 @@ import unittest import mock import six +from certbot import errors + # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org mydomain = fubard.org @@ -37,6 +39,17 @@ class TestPostfixConfigGenerator(unittest.TestCase): for call in mock_add.call_args_list: self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + def test_no_postconf_prepare(self): + installer = self._create_installer() + + installer_path = "certbot_postfix.installer" + exe_exists_path = installer_path + ".util.exe_exists" + path_surgery_path = installer_path + ".plugins_util.path_surgery" + + with mock.patch(path_surgery_path, return_value=False): + with mock.patch(exe_exists_path, return_value=False): + self.assertRaises(errors.NoInstallationError, installer.prepare) + def testGetAllNames(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: From 5beaae3b6502fa4e39ddc80a564f8308347cb67f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:17:13 -0700 Subject: [PATCH 156/639] Add check_output function and tests. --- certbot-postfix/certbot_postfix/util.py | 49 +++++++++++++++++++ certbot-postfix/certbot_postfix/util_test.py | 50 ++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 certbot-postfix/certbot_postfix/util.py create mode 100644 certbot-postfix/certbot_postfix/util_test.py diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py new file mode 100644 index 000000000..8ad6408c4 --- /dev/null +++ b/certbot-postfix/certbot_postfix/util.py @@ -0,0 +1,49 @@ +"""Utility functions for use in the Postfix installer.""" + +import logging +import subprocess + + +logger = logging.getLogger(__name__) + + +def check_output(*args, **kwargs): + """Backported version of subprocess.check_output for Python 2.6+. + + This is the same as subprocess.check_output from newer versions of + Python, except: + + 1. The return value is a string rather than a byte string. To + accomplish this, the caller cannot set the parameter + universal_newlines. + 2. If the command exits with a nonzero status, output is not + included in the raised subprocess.CalledProcessError because + subprocess.CalledProcessError on Python 2.6 does not support this. + Instead, the failure including the output is logged. + + :param tuple args: positional arguments for Popen + :param dict kwargs: keyword arguments for Popen + + :returns: data printed to stdout + :rtype: str + + """ + for keyword in ('stdout', 'universal_newlines',): + if keyword in kwargs: + raise ValueError( + keyword + ' argument not allowed, it will be overridden.') + + kwargs['stdout'] = subprocess.PIPE + kwargs['universal_newlines'] = True + + process = subprocess.Popen(*args, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get('args') + if cmd is None: + cmd = args[0] + logger.debug( + "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) + raise subprocess.CalledProcessError(retcode, cmd) + return output diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py new file mode 100644 index 000000000..019f34532 --- /dev/null +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -0,0 +1,50 @@ +"""Tests for certbot_postfix.util.""" + +import subprocess +import unittest + +import mock + +class CheckOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_output + return check_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + + mock_popen().communicate.return_value = (output, '') + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + + log_args = mock_logger.debug.call_args[0] + self.assertTrue(command in log_args) + self.assertTrue(retcode in log_args) + self.assertTrue(output in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + output = 'bar' + mock_popen().communicate.return_value = (output, '') + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), output) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() From 4715b2b12ceede6b8b40e5a336a546435a7315dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:18:27 -0700 Subject: [PATCH 157/639] Further document check_output --- certbot-postfix/certbot_postfix/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index 8ad6408c4..d7145b7c8 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -27,6 +27,9 @@ def check_output(*args, **kwargs): :returns: data printed to stdout :rtype: str + :raises ValueError: if arguments are invalid + :raises subprocess.CalledProcessError: if the command fails + """ for keyword in ('stdout', 'universal_newlines',): if keyword in kwargs: From 5a1d031f07cc2a64410258920c48f14562a1addd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:19:44 -0700 Subject: [PATCH 158/639] Rename util to certbot_util --- certbot-postfix/certbot_postfix/installer.py | 4 ++-- certbot-postfix/certbot_postfix/installer_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 394ed5525..3de96d9a5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -9,7 +9,7 @@ import zope.interface from certbot import errors from certbot import interfaces -from certbot import util +from certbot import util as certbot_util from certbot.plugins import common as plugins_common from certbot.plugins import util as plugins_util @@ -125,7 +125,7 @@ class Installer(plugins_common.Plugin): :raises .NoInstallationError: when unable to find 'postconf' """ - if not util.exe_exists(self.conf("config-utility")): + if not certbot_util.exe_exists(self.conf("config-utility")): if not plugins_util.path_surgery(self.conf("config-utility")): raise errors.NoInstallationError( "Cannot find executable '{0}'. You can provide the " diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 721790797..9c8aa4f77 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -43,7 +43,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): installer = self._create_installer() installer_path = "certbot_postfix.installer" - exe_exists_path = installer_path + ".util.exe_exists" + exe_exists_path = installer_path + ".certbot_util.exe_exists" path_surgery_path = installer_path + ".plugins_util.path_surgery" with mock.patch(path_surgery_path, return_value=False): @@ -84,7 +84,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() - exe_exists_path = "certbot_postfix.installer.util.exe_exists" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" popen_path = "certbot_postfix.installer.subprocess.Popen" with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: with mock.patch(popen_path) as mock_popen: From 4e5740615c2d300b57785126ec84030aadb55445 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:25:08 -0700 Subject: [PATCH 159/639] Use util.check_output in Postfix installer --- certbot-postfix/certbot_postfix/installer.py | 13 +++++++++---- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3de96d9a5..44f77227e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -13,6 +13,8 @@ from certbot import util as certbot_util from certbot.plugins import common as plugins_common from certbot.plugins import util as plugins_util +from certbot_postfix import util + logger = logging.getLogger(__name__) @@ -240,10 +242,13 @@ class Installer(plugins_common.Plugin): """ # Parse Postfix version number (feature support, syntax changes etc.) - cmd = subprocess.Popen([self.conf('config-utility'), '-d', 'mail_version'], - stdout=subprocess.PIPE) - stdout, _ = cmd.communicate() - if cmd.returncode != 0: + try: + stdout = util.check_output( + [self.conf('config-utility'), '-d', 'mail_version']) + except subprocess.CalledProcessError: + logger.debug( + 'Encountered an error when trying to determine Postfix version.', + exc_info=True) raise errors.PluginError('Unable to determine Postfix version.') # grabs version component of string like "mail_version = 2.11.3" diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9c8aa4f77..a0aa060e8 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,12 +84,11 @@ class TestPostfixConfigGenerator(unittest.TestCase): """ installer = self._create_installer() + check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - popen_path = "certbot_postfix.installer.subprocess.Popen" - with mock.patch(exe_exists_path, return_value=True) as mock_exe_exists: - with mock.patch(popen_path) as mock_popen: - mock_popen().returncode = 0 - mock_popen().communicate.return_value = ("mail_version = 3.1.4", "") + with mock.patch(check_output_path) as mock_check_output: + with mock.patch(exe_exists_path, return_value=True): + mock_check_output.return_value = "mail_version = 3.1.4" installer.prepare() return installer From 7334fc3066dc51ada34c0c8e88abf6f3552c711d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 14:29:56 -0700 Subject: [PATCH 160/639] Rename postfix_dir to config_dir --- certbot-postfix/certbot_postfix/installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 44f77227e..54928279b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -37,7 +37,7 @@ class Installer(plugins_common.Plugin): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) self.fixup = False - self.postfix_dir = self.conf("config-dir") + self.config_dir = self.conf("config-dir") self.additions = [] self.deletions = [] @@ -137,7 +137,7 @@ class Installer(plugins_common.Plugin): def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" - return os.path.join(self.postfix_dir, "main.cf") + return os.path.join(self.config_dir, "main.cf") def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -268,7 +268,7 @@ class Installer(plugins_common.Plugin): "Server root: {root}{0}" "Version: {version}".format( os.linesep, - root=self.postfix_dir, + root=self.config_dir, version='.'.join([str(i) for i in self.get_version()])) ) From b72dfc0c08e43149f9e9d11e1a6e3ff111dfb1a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:07:32 -0700 Subject: [PATCH 161/639] Add get_config_var --- certbot-postfix/certbot_postfix/installer.py | 69 ++++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 54928279b..51455a340 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -241,20 +241,8 @@ class Installer(plugins_common.Plugin): :raises .PluginError: Unable to find Postfix version. """ - # Parse Postfix version number (feature support, syntax changes etc.) - try: - stdout = util.check_output( - [self.conf('config-utility'), '-d', 'mail_version']) - except subprocess.CalledProcessError: - logger.debug( - 'Encountered an error when trying to determine Postfix version.', - exc_info=True) - raise errors.PluginError('Unable to determine Postfix version.') - - # grabs version component of string like "mail_version = 2.11.3" - mail_version = stdout.split()[2] - postfix_version = tuple([int(i) for i in mail_version.split('.')]) - return postfix_version + mail_version = self.get_config_var("mail_version", default=True) + return tuple(int(i) for i in mail_version.split('.')) def more_info(self): """Human-readable string to help the user. @@ -406,6 +394,59 @@ class Installer(plugins_common.Plugin): if rc != 0: raise errors.MisconfigurationError('cannot restart postfix') + def get_config_var(self, name, default=False): + """Return the value of the specified Postfix config parameter. + + :param str name: name of the Postfix config parameter to return + :param bool default: whether or not to return the default value + instead of the actual value + + :returns: value of the specified configuration parameter + :rtype: str + + """ + cmd = self._build_cmd_for_config_var(name, default) + + try: + output = util.check_output(cmd) + except subprocess.CalledProcessError: + logger.debug("Encountered an error when running 'postconf'", + exc_info=True) + raise errors.PluginError( + "Unable to determine the value " + "of Postfix parameter {0}".format(name)) + + expected_prefix = name + " =" + if not output.startswith(expected_prefix): + raise errors.PluginError( + "Unexpected output from '{0}'".format(''.join(cmd))) + + return output[len(expected_prefix):].strip() + + def _build_cmd_for_config_var(self, name, default): + """Return a command to run to get a Postfix config parameter. + + :param str name: name of the Postfix config parameter to return + :param bool default: whether or not to return the default value + instead of the actual value + + :returns: command to run + :rtype: list + + """ + cmd = [self.conf("config-utility")] + + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + + if default: + cmd.append("-d") + + cmd.append(name) + + return cmd + + # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) # From 02c7eca6daa6727683161105ab2758b7f3adff07 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:08:41 -0700 Subject: [PATCH 162/639] Rename test classes and methods. --- certbot-postfix/certbot_postfix/installer_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a0aa060e8..6c7a3b628 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -25,7 +25,7 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -class TestPostfixConfigGenerator(unittest.TestCase): +class InstallerTest(unittest.TestCase): def setUp(self): self.postfix_dir = 'tests/' @@ -50,14 +50,14 @@ class TestPostfixConfigGenerator(unittest.TestCase): with mock.patch(exe_exists_path, return_value=False): self.assertRaises(errors.NoInstallationError, installer.prepare) - def testGetAllNames(self): + def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = self._create_prepared_installer() self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) - def testGetAllCertAndKeys(self): + def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 'tests/main.cf'),] @@ -66,7 +66,7 @@ class TestPostfixConfigGenerator(unittest.TestCase): postfix_config_gen = self._create_prepared_installer() self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) - def testGetAllCertsAndKeys_With_None(self): + def test_get_all_certs_and_keys_with_none(self): with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(names_only_config) postfix_config_gen = self._create_prepared_installer() From 4c4b63437f9c2652be9d0633c61ac19c186a7b21 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 15:28:17 -0700 Subject: [PATCH 163/639] Test building of get_config_var command --- .../certbot_postfix/installer_test.py | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 6c7a3b628..3eeb5288c 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,7 +28,7 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class InstallerTest(unittest.TestCase): def setUp(self): - self.postfix_dir = 'tests/' + self.config = mock.MagicMock(postfix_config_dir="tests/") def test_add_parser_arguments(self): mock_add = mock.MagicMock() @@ -72,6 +72,36 @@ class InstallerTest(unittest.TestCase): postfix_config_gen = self._create_prepared_installer() self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + def test_get_config_var_success(self): + self.config.postfix_config_dir = None + + command = self._test_get_config_var_success_common('foo', False) + self.assertFalse("-c" in command) + self.assertFalse("-d" in command) + + def test_get_config_var_success_with_config(self): + command = self._test_get_config_var_success_common('foo', False) + self.assertTrue("-c" in command) + self.assertFalse("-d" in command) + + def test_get_config_var_success_with_default(self): + self.config.postfix_config_dir = None + + command = self._test_get_config_var_success_common('foo', True) + self.assertFalse("-c" in command) + self.assertTrue("-d" in command) + + def _test_get_config_var_success_common(self, name, default): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + with mock.patch(check_output_path) as mock_check_output: + value = "bar" + mock_check_output.return_value = name + " = " + value + self.assertEqual(installer.get_config_var(name, default), value) + + return mock_check_output.call_args[0][0] + def _create_prepared_installer(self): """Creates and returns a new prepared Postfix Installer. @@ -100,11 +130,10 @@ class InstallerTest(unittest.TestCase): :rtype: certbot_postfix.installer.Installer """ - config = mock.MagicMock(postfix_config_dir=self.postfix_dir) name = "postfix" from certbot_postfix import installer - return installer.Installer(config, name) + return installer.Installer(self.config, name) if __name__ == '__main__': From 25d1f6ec7522ccb76b20ef89db2527946067f335 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:11:09 -0700 Subject: [PATCH 164/639] Test all branches of test_get_config_var --- .../certbot_postfix/installer_test.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 3eeb5288c..9fd192d66 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -7,6 +7,7 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import subprocess import unittest import mock @@ -28,7 +29,8 @@ smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") class InstallerTest(unittest.TestCase): def setUp(self): - self.config = mock.MagicMock(postfix_config_dir="tests/") + self.config = mock.MagicMock(postfix_config_dir="tests/", + postfix_config_utility="postconf") def test_add_parser_arguments(self): mock_add = mock.MagicMock() @@ -91,6 +93,22 @@ class InstallerTest(unittest.TestCase): self.assertFalse("-c" in command) self.assertTrue("-d" in command) + @mock.patch("certbot_postfix.installer.logger") + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_config_var_failure(self, mock_check_output, mock_logger): + mock_check_output.side_effect = subprocess.CalledProcessError(42, "foo") + installer = self._create_installer() + self.assertRaises(errors.PluginError, installer.get_config_var, "foo") + self.assertTrue(mock_logger.debug.call_args[1]["exc_info"]) + + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_config_var_unexpected_output(self, mock_check_output): + self.config.postfix_config_dir = None + mock_check_output.return_value = "foo" + + installer = self._create_installer() + self.assertRaises(errors.PluginError, installer.get_config_var, "foo") + def _test_get_config_var_success_common(self, name, default): installer = self._create_installer() From 2e8a8dfed528be9e485775215a5bf976757d3576 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:17:46 -0700 Subject: [PATCH 165/639] add _set_config_dir --- certbot-postfix/certbot_postfix/installer.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 51455a340..9756da684 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -37,7 +37,7 @@ class Installer(plugins_common.Plugin): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) self.fixup = False - self.config_dir = self.conf("config-dir") + self.config_dir = None self.additions = [] self.deletions = [] @@ -67,6 +67,7 @@ class Installer(plugins_common.Plugin): """ self._verify_postconf_available() + self._set_config_dir() self.fn = self.find_postfix_cf() self.raw_cf = open(self.fn).readlines() @@ -135,6 +136,19 @@ class Installer(plugins_common.Plugin): self.conf("config-utility"), self.option_name("config-utility"))) + def _set_config_dir(self): + """Ensure self.config_dir is set to the correct path. + + If the configuration directory to use was set by the user, we'll + use that value, otherwise, we'll find the default path using + 'postconf'. + + """ + if self.conf("config-dir") is None: + self.config_dir = self.get_config_var("config_directory") + else: + self.config_dir = self.conf("config-dir") + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.config_dir, "main.cf") From 749f758adb699cb540d4d0fc1e781c0e87a79a6e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:25:30 -0700 Subject: [PATCH 166/639] use a temporary directory --- certbot-postfix/certbot_postfix/installer_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9fd192d66..08c9d478f 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -7,6 +7,7 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import os import subprocess import unittest @@ -14,6 +15,7 @@ import mock import six from certbot import errors +from certbot.tests import util as certbot_test_util # Fake Postfix Configs names_only_config = """myhostname = mail.fubard.org @@ -26,10 +28,11 @@ certs_only_config = ( smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") -class InstallerTest(unittest.TestCase): +class InstallerTest(certbot_test_util.TempDirTestCase): def setUp(self): - self.config = mock.MagicMock(postfix_config_dir="tests/", + super(InstallerTest, self).setUp() + self.config = mock.MagicMock(postfix_config_dir=self.tempdir, postfix_config_utility="postconf") def test_add_parser_arguments(self): @@ -62,7 +65,7 @@ class InstallerTest(unittest.TestCase): def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - 'tests/main.cf'),] + os.path.join(self.tempdir, 'main.cf')),] with mock.patch('certbot_postfix.installer.open') as mock_open: mock_open.return_value = six.StringIO(certs_only_config) postfix_config_gen = self._create_prepared_installer() From 48c5731a6b18032b244ba1ed2dd9f26b17efecf4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:29:08 -0700 Subject: [PATCH 167/639] Write out temp config instead of mocking. --- .../certbot_postfix/installer_test.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 08c9d478f..9e83699ab 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -57,25 +57,26 @@ class InstallerTest(certbot_test_util.TempDirTestCase): def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) + self._write_config(names_only_config) + installer = self._create_prepared_installer() + self.assertEqual(sorted_names, installer.get_all_names()) def test_get_all_certs_and_keys(self): return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', '/etc/letsencrypt/live/www.fubard.org/privkey.pem', os.path.join(self.tempdir, 'main.cf')),] - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(certs_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) + self._write_config(certs_only_config) + installer = self._create_prepared_installer() + self.assertEqual(return_vals, installer.get_all_certs_keys()) def test_get_all_certs_and_keys_with_none(self): - with mock.patch('certbot_postfix.installer.open') as mock_open: - mock_open.return_value = six.StringIO(names_only_config) - postfix_config_gen = self._create_prepared_installer() - self.assertEqual([], postfix_config_gen.get_all_certs_keys()) + self._write_config(names_only_config) + installer = self._create_prepared_installer() + self.assertEqual([], installer.get_all_certs_keys()) + + def _write_config(self, content): + with open(os.path.join(self.tempdir, "main.cf"), "w") as f: + f.write(content) def test_get_config_var_success(self): self.config.postfix_config_dir = None From 290f5b8ce7ae544fc42dca1d383906fcc5dad7b1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:35:21 -0700 Subject: [PATCH 168/639] add test_set_config_dir --- certbot-postfix/certbot_postfix/installer.py | 2 +- .../certbot_postfix/installer_test.py | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 9756da684..546cf49cc 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -433,7 +433,7 @@ class Installer(plugins_common.Plugin): expected_prefix = name + " =" if not output.startswith(expected_prefix): raise errors.PluginError( - "Unexpected output from '{0}'".format(''.join(cmd))) + "Unexpected output from '{0}'".format(' '.join(cmd))) return output[len(expected_prefix):].strip() diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 9e83699ab..aae61bd59 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -55,6 +55,25 @@ class InstallerTest(certbot_test_util.TempDirTestCase): with mock.patch(exe_exists_path, return_value=False): self.assertRaises(errors.NoInstallationError, installer.prepare) + def test_set_config_dir(self): + self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") + os.mkdir(self.config.postfix_config_dir) + self._write_config(names_only_config) + installer = self._create_installer() + + expected = self.config.postfix_config_dir + self.config.postfix_config_dir = None + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = [ + "config_directory = " + expected, "mail_version = 3.1.4" + ] + with mock.patch(exe_exists_path, return_value=True): + installer.prepare() + self.assertEqual(installer.config_dir, expected) + def test_get_all_names(self): sorted_names = ['fubard.org', 'mail.fubard.org'] self._write_config(names_only_config) @@ -75,7 +94,8 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertEqual([], installer.get_all_certs_keys()) def _write_config(self, content): - with open(os.path.join(self.tempdir, "main.cf"), "w") as f: + config_dir = self.config.postfix_config_dir + with open(os.path.join(config_dir, "main.cf"), "w") as f: f.write(content) def test_get_config_var_success(self): From 8c4ff5cb63dab6632f643da49f87cc86dcc3d8c9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:36:08 -0700 Subject: [PATCH 169/639] Use context manager to read conf file. --- certbot-postfix/certbot_postfix/installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 546cf49cc..bba46a168 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -70,7 +70,8 @@ class Installer(plugins_common.Plugin): self._set_config_dir() self.fn = self.find_postfix_cf() - self.raw_cf = open(self.fn).readlines() + with open(self.fn) as f: + self.raw_cf = f.readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] # XXX ensure we raise the right kinds of exceptions From 50a1f6340ff2c33d80fe2bf1f2b7b95b3cdebe69 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:38:27 -0700 Subject: [PATCH 170/639] Add _check_version. --- certbot-postfix/certbot_postfix/installer.py | 61 +++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index bba46a168..2c3f4397e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -74,11 +74,42 @@ class Installer(plugins_common.Plugin): self.raw_cf = f.readlines() self.cf = map(string.strip, self.raw_cf) #self.cf = [line for line in cf if line and not line.startswith("#")] - # XXX ensure we raise the right kinds of exceptions + def _verify_postconf_available(self): + """Ensure 'postconf' can be found. + + :raises .NoInstallationError: when unable to find 'postconf' + + """ + if not certbot_util.exe_exists(self.conf("config-utility")): + if not plugins_util.path_surgery(self.conf("config-utility")): + raise errors.NoInstallationError( + "Cannot find executable '{0}'. You can provide the " + "path to this command with --{1}".format( + self.conf("config-utility"), + self.option_name("config-utility"))) + + def _set_config_dir(self): + """Ensure self.config_dir is set to the correct path. + + If the configuration directory to use was set by the user, we'll + use that value, otherwise, we'll find the default path using + 'postconf'. + + """ + if self.conf("config-dir") is None: + self.config_dir = self.get_config_var("config_directory") + else: + self.config_dir = self.conf("config-dir") + + def _check_version(self): + """Verifies that the installed Postfix version is supported. + + :raises errors.NotSupportedError: if the version is unsupported + + """ if self.get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend # differences here and pass them onto other configuration functions. @@ -123,32 +154,6 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html - def _verify_postconf_available(self): - """Ensure 'postconf' can be found. - - :raises .NoInstallationError: when unable to find 'postconf' - - """ - if not certbot_util.exe_exists(self.conf("config-utility")): - if not plugins_util.path_surgery(self.conf("config-utility")): - raise errors.NoInstallationError( - "Cannot find executable '{0}'. You can provide the " - "path to this command with --{1}".format( - self.conf("config-utility"), - self.option_name("config-utility"))) - - def _set_config_dir(self): - """Ensure self.config_dir is set to the correct path. - - If the configuration directory to use was set by the user, we'll - use that value, otherwise, we'll find the default path using - 'postconf'. - - """ - if self.conf("config-dir") is None: - self.config_dir = self.get_config_var("config_directory") - else: - self.config_dir = self.conf("config-dir") def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" From b98f541b9107fe6ccc34963b22d4b93f8d1ec67c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:41:59 -0700 Subject: [PATCH 171/639] clean up prepare() --- certbot-postfix/certbot_postfix/installer.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 2c3f4397e..bac33fd59 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -52,22 +52,14 @@ class Installer(plugins_common.Plugin): Finish up any additional initialization. - :raises .PluginError: - when full initialization cannot be completed. - :raises .MisconfigurationError: - when full initialization cannot be completed. Plugin will - be displayed on a list of available plugins. - :raises .NoInstallationError: - when the necessary programs/files cannot be located. Plugin - will NOT be displayed on a list of available plugins. - :raises .NotSupportedError: - when the installation is recognized, but the version is not - currently supported. - :rtype tuple: + :raises errors.PluginError: when an unexpected error occurs + :raises errors.NoInstallationError: when can't find installation + :raises errors.NotSupportedError: when version is not supported """ self._verify_postconf_available() self._set_config_dir() + self._check_version() self.fn = self.find_postfix_cf() with open(self.fn) as f: From c9813a44d7b8a86964012ff09e62211833ace1bd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 10 Aug 2017 16:42:48 -0700 Subject: [PATCH 172/639] protect get_version() --- certbot-postfix/certbot_postfix/installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index bac33fd59..c0200ff3d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -100,7 +100,7 @@ class Installer(plugins_common.Plugin): :raises errors.NotSupportedError: if the version is unsupported """ - if self.get_version() < (2, 11, 0): + if self._get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') # Postfix has changed support for TLS features, supported protocol versions # KEX methods, ciphers et cetera over the years. We sort out version dependend @@ -242,7 +242,7 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def get_version(self): + def _get_version(self): """Return the mail version of Postfix. Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) @@ -269,7 +269,7 @@ class Installer(plugins_common.Plugin): "Version: {version}".format( os.linesep, root=self.config_dir, - version='.'.join([str(i) for i in self.get_version()])) + version='.'.join([str(i) for i in self._get_version()])) ) From d1f3a2deefacc5b9bd678c3c241ac23df3e13765 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:13:06 -0700 Subject: [PATCH 173/639] move _get_version --- certbot-postfix/certbot_postfix/installer.py | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index c0200ff3d..292be3f8d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -146,6 +146,19 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def _get_version(self): + """Return the mail version of Postfix. + + Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + + :returns: version + :rtype: tuple + + :raises .PluginError: Unable to find Postfix version. + + """ + mail_version = self.get_config_var("mail_version", default=True) + return tuple(int(i) for i in mail_version.split('.')) def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" @@ -242,20 +255,6 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def _get_version(self): - """Return the mail version of Postfix. - - Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) - - :returns: version - :rtype: tuple - - :raises .PluginError: Unable to find Postfix version. - - """ - mail_version = self.get_config_var("mail_version", default=True) - return tuple(int(i) for i in mail_version.split('.')) - def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user From 83e37acc8b794fad3ce495fcb209be65c8becf3e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:16:07 -0700 Subject: [PATCH 174/639] group IPlugin methods --- certbot-postfix/certbot_postfix/installer.py | 41 ++++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 292be3f8d..d8e8420d9 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -146,6 +146,26 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.config_dir, "main.cf") + + def more_info(self): + """Human-readable string to help the user. + Should describe the steps taken and any relevant info to help the user + decide which plugin to use. + :rtype str: + """ + return ( + "Configures Postfix to try to authenticate mail servers, use " + "installed certificates and disable weak ciphers and protocols.{0}" + "Server root: {root}{0}" + "Version: {version}".format( + os.linesep, + root=self.config_dir, + version='.'.join([str(i) for i in self._get_version()])) + ) + def _get_version(self): """Return the mail version of Postfix. @@ -160,10 +180,6 @@ class Installer(plugins_common.Plugin): mail_version = self.get_config_var("mail_version", default=True) return tuple(int(i) for i in mail_version.split('.')) - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.config_dir, "main.cf") - def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -255,23 +271,6 @@ class Installer(plugins_common.Plugin): # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - def more_info(self): - """Human-readable string to help the user. - Should describe the steps taken and any relevant info to help the user - decide which plugin to use. - :rtype str: - """ - return ( - "Configures Postfix to try to authenticate mail servers, use " - "installed certificates and disable weak ciphers and protocols.{0}" - "Server root: {root}{0}" - "Version: {version}".format( - os.linesep, - root=self.config_dir, - version='.'.join([str(i) for i in self._get_version()])) - ) - - ### Let's Encrypt client IInstaller ### # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 From b342f40c2ba68b73ed7428b4190705671c5e7caf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Aug 2017 15:16:36 -0700 Subject: [PATCH 175/639] remove old comments --- certbot-postfix/certbot_postfix/installer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d8e8420d9..92ba3f13d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -267,13 +267,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - ### Let's Encrypt client IPlugin ### - # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 - - - ### Let's Encrypt client IInstaller ### - # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 - def get_all_names(self): """Returns all names that may be authenticated. :rtype: `list` of `str` From 90ffe2aac0d310947b7931792acfcfc96c036601 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 09:04:57 -0700 Subject: [PATCH 176/639] Remove legacy get_all_certs_and_keys() method --- certbot-postfix/certbot_postfix/installer.py | 24 ------------------- .../certbot_postfix/installer_test.py | 13 ---------- 2 files changed, 37 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 92ba3f13d..3eb670161 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -317,30 +317,6 @@ class Installer(plugins_common.Plugin): :rtype: :class:`list` of :class:`str` """ - def get_all_certs_keys(self): - """Retrieve all certs and keys set in configuration. - :returns: tuples with form `[(cert, key, path)]`, where: - - `cert` - str path to certificate file - - `key` - str path to associated key file - - `path` - file path to configuration file - :rtype: list - """ - cert_materials = {'smtpd_tls_key_file': None, - 'smtpd_tls_cert_file': None, - } - for num, line in enumerate(self.cf): - num, found_var, found_value = parse_line((num, line)) - if found_var in cert_materials.keys(): - cert_materials[found_var] = found_value - - if not all(cert_materials.values()): - cert_material_tuples = [] - else: - cert_material_tuples = [(cert_materials['smtpd_tls_cert_file'], - cert_materials['smtpd_tls_key_file'], - self.fn),] - return cert_material_tuples - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index aae61bd59..0c83a25e6 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -80,19 +80,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer = self._create_prepared_installer() self.assertEqual(sorted_names, installer.get_all_names()) - def test_get_all_certs_and_keys(self): - return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', - '/etc/letsencrypt/live/www.fubard.org/privkey.pem', - os.path.join(self.tempdir, 'main.cf')),] - self._write_config(certs_only_config) - installer = self._create_prepared_installer() - self.assertEqual(return_vals, installer.get_all_certs_keys()) - - def test_get_all_certs_and_keys_with_none(self): - self._write_config(names_only_config) - installer = self._create_prepared_installer() - self.assertEqual([], installer.get_all_certs_keys()) - def _write_config(self, content): config_dir = self.config.postfix_config_dir with open(os.path.join(config_dir, "main.cf"), "w") as f: From 967a1830e6fdab696d9d0edf9d8416622811b341 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 14:59:25 -0700 Subject: [PATCH 177/639] Rewrite get_all_names --- certbot-postfix/certbot_postfix/installer.py | 26 ++++++++----------- .../certbot_postfix/installer_test.py | 14 ++++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3eb670161..3bbb112ba 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -180,6 +180,15 @@ class Installer(plugins_common.Plugin): mail_version = self.get_config_var("mail_version", default=True) return tuple(int(i) for i in mail_version.split('.')) + def get_all_names(self): + """Returns all names that may be authenticated. + + :rtype: `set` of `str` + + """ + return set(self.get_config_var(var) + for var in ('mydomain', 'myhostname', 'myorigin',)) + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -267,20 +276,6 @@ class Installer(plugins_common.Plugin): with open(self.fn, "w") as f: f.write(self.new_cf) - def get_all_names(self): - """Returns all names that may be authenticated. - :rtype: `list` of `str` - """ - var_names = ('myhostname', 'mydomain', 'myorigin') - names_found = set() - for num, line in enumerate(self.cf): - num, found_var, found_value = parse_line((num, line)) - if found_var in var_names: - names_found.add(found_value) - name_list = list(names_found) - name_list.sort() - return name_list - def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): """Deploy certificate. :param str domain: domain to deploy certificate file @@ -398,7 +393,8 @@ class Installer(plugins_common.Plugin): expected_prefix = name + " =" if not output.startswith(expected_prefix): raise errors.PluginError( - "Unexpected output from '{0}'".format(' '.join(cmd))) + "Unexpected output '{0}' from '{1}'".format(output, + ' '.join(cmd))) return output[len(expected_prefix):].strip() diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 0c83a25e6..467eed752 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -18,8 +18,8 @@ from certbot import errors from certbot.tests import util as certbot_test_util # Fake Postfix Configs -names_only_config = """myhostname = mail.fubard.org -mydomain = fubard.org +names_only_config = """mydomain = fubard.org +myhostname = mail.fubard.org myorigin = fubard.org""" @@ -74,11 +74,15 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer.prepare() self.assertEqual(installer.config_dir, expected) - def test_get_all_names(self): - sorted_names = ['fubard.org', 'mail.fubard.org'] + @mock.patch("certbot_postfix.installer.util.check_output") + def test_get_all_names(self, mock_check_output): self._write_config(names_only_config) installer = self._create_prepared_installer() - self.assertEqual(sorted_names, installer.get_all_names()) + mock_check_output.side_effect = names_only_config.splitlines() + + result = installer.get_all_names() + self.assertTrue("fubard.org" in result) + self.assertTrue("mail.fubard.org" in result) def _write_config(self, content): config_dir = self.config.postfix_config_dir From 0efc02d6eeeabe6383ffa0c3a9c43842fca9aedb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:03:11 -0700 Subject: [PATCH 178/639] Lock the Postfix config dir --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3bbb112ba..362fe5e8c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -60,6 +60,7 @@ class Installer(plugins_common.Plugin): self._verify_postconf_available() self._set_config_dir() self._check_version() + self._lock_config_dir() self.fn = self.find_postfix_cf() with open(self.fn) as f: @@ -146,6 +147,19 @@ class Installer(plugins_common.Plugin): # - Built-in support for TLS management and DANE added, see: # http://www.postfix.org/postfix-tls.1.html + def _lock_config_dir(self): + """Stop two Postfix plugins from modifying the config at once. + + :raises .PluginError: if unable to acquire the lock + + """ + try: + certbot_util.lock_dir_until_exit(self.config_dir) + except (OSError, errors.LockError): + logger.debug("Encountered error:", exc_info=True) + raise errors.PluginError( + "Unable to lock %s", self.config_dir) + def find_postfix_cf(self): "Search far and wide for the correct postfix configuration file" return os.path.join(self.config_dir, "main.cf") From 3b2e9e49be4183f980f5d0c981438df2151209d8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:04:36 -0700 Subject: [PATCH 179/639] Remove unneeded instance variables --- certbot-postfix/certbot_postfix/installer.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 362fe5e8c..9a7c3802e 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -39,14 +39,6 @@ class Installer(plugins_common.Plugin): self.fixup = False self.config_dir = None - self.additions = [] - self.deletions = [] - self.policy_lines = [] - self.new_cf = "" - self.fn = None - self.raw_cf = [] - self.cf = [] - def prepare(self): """Prepare the installer. @@ -62,12 +54,6 @@ class Installer(plugins_common.Plugin): self._check_version() self._lock_config_dir() - self.fn = self.find_postfix_cf() - with open(self.fn) as f: - self.raw_cf = f.readlines() - self.cf = map(string.strip, self.raw_cf) - #self.cf = [line for line in cf if line and not line.startswith("#")] - def _verify_postconf_available(self): """Ensure 'postconf' can be found. From b9177948d393b590d11342d2463681e8a2bdc01e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:05:24 -0700 Subject: [PATCH 180/639] Remove _write_config --- certbot-postfix/certbot_postfix/installer_test.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 467eed752..5122bcfe9 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -58,7 +58,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): def test_set_config_dir(self): self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") os.mkdir(self.config.postfix_config_dir) - self._write_config(names_only_config) installer = self._create_installer() expected = self.config.postfix_config_dir @@ -76,7 +75,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): @mock.patch("certbot_postfix.installer.util.check_output") def test_get_all_names(self, mock_check_output): - self._write_config(names_only_config) installer = self._create_prepared_installer() mock_check_output.side_effect = names_only_config.splitlines() @@ -84,11 +82,6 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) - def _write_config(self, content): - config_dir = self.config.postfix_config_dir - with open(os.path.join(config_dir, "main.cf"), "w") as f: - f.write(content) - def test_get_config_var_success(self): self.config.postfix_config_dir = None From b94e268f83f383b13d3ea61e1a16e6b166ddeffa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:05:39 -0700 Subject: [PATCH 181/639] Remove unused certs_only_config --- certbot-postfix/certbot_postfix/installer_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 5122bcfe9..429974e49 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -23,11 +23,6 @@ myhostname = mail.fubard.org myorigin = fubard.org""" -certs_only_config = ( -"""smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem -smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") - - class InstallerTest(certbot_test_util.TempDirTestCase): def setUp(self): From cc3896d5d45bc3c68cc7b888447ce49a62ec0e2e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:12:11 -0700 Subject: [PATCH 182/639] Add test_lock_error --- certbot-postfix/certbot_postfix/installer_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 429974e49..29dcf3200 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -6,6 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +import functools import logging import os import subprocess @@ -68,6 +69,12 @@ class InstallerTest(certbot_test_util.TempDirTestCase): installer.prepare() self.assertEqual(installer.config_dir, expected) + def test_lock_error(self): + assert_raises = functools.partial(self.assertRaises, + errors.PluginError, + self._create_prepared_installer) + certbot_test_util.lock_and_call(assert_raises, self.tempdir) + @mock.patch("certbot_postfix.installer.util.check_output") def test_get_all_names(self, mock_check_output): installer = self._create_prepared_installer() From baf0d3343aceba3568dc06898c654c0322bd428a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:13:02 -0700 Subject: [PATCH 183/639] Remove postfix version notes --- certbot-postfix/certbot_postfix/installer.py | 43 -------------------- 1 file changed, 43 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 9a7c3802e..0f3796e1f 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -89,49 +89,6 @@ class Installer(plugins_common.Plugin): """ if self._get_version() < (2, 11, 0): raise errors.NotSupportedError('Postfix version is too old') - # Postfix has changed support for TLS features, supported protocol versions - # KEX methods, ciphers et cetera over the years. We sort out version dependend - # differences here and pass them onto other configuration functions. - # see: - # http://www.postfix.org/TLS_README.html - # http://www.postfix.org/FORWARD_SECRECY_README.html - - # Postfix == 2.2: - # - TLS support introduced via 3rd party patch, see: - # http://www.postfix.org/TLS_LEGACY_README.html - - # Postfix => 2.2: - # - built-in TLS support added - # - Support for PFS introduced - # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k - - # Postfix => 2.5: - # - Syntax to specify mandatory protocol version changes: - # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` - # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` - # - Certificate fingerprint verification added - - # Postfix => 2.6: - # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) - # - Support for configurable cipher-suites and protocol versions added, pre-2.6 - # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` - # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added - - # Postfix => 2.8: - # - Override Client suite preference w. `tls_preempt_cipherlist = yes` - # - Elliptic curve crypto. support enabled by default - - # Postfix => 2.9: - # - Public key fingerprint support added - # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and - # `check_ccert_access` config. options added - - # Postfix <= 2.9.5: - # - BUG: Public key fingerprint is computed incorrectly - - # Postfix => 3.1: - # - Built-in support for TLS management and DANE added, see: - # http://www.postfix.org/postfix-tls.1.html def _lock_config_dir(self): """Stop two Postfix plugins from modifying the config at once. From b92df1b71cf6b21b5414176c74f048e0af9c793d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:15:41 -0700 Subject: [PATCH 184/639] Remove unused find_postfix_cf --- certbot-postfix/certbot_postfix/installer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 0f3796e1f..d9aeb4887 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -103,10 +103,6 @@ class Installer(plugins_common.Plugin): raise errors.PluginError( "Unable to lock %s", self.config_dir) - def find_postfix_cf(self): - "Search far and wide for the correct postfix configuration file" - return os.path.join(self.config_dir, "main.cf") - def more_info(self): """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user From 00e28592b6163f314f68a7c81dc5faa1854ca2f6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:18:40 -0700 Subject: [PATCH 185/639] Add supported_enhancements --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++------ certbot-postfix/certbot_postfix/installer_test.py | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d9aeb4887..45768df6c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -142,6 +142,14 @@ class Installer(plugins_common.Plugin): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def supported_enhancements(self): + """Returns a list of supported enhancements. + + :rtype: list + + """ + return [] + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -258,12 +266,6 @@ class Installer(plugins_common.Plugin): an error occurs during the enhancement. """ - def supported_enhancements(self): - """Returns a list of supported enhancements. - :returns: supported enhancements which should be a subset of - :const:`~letsencrypt.constants.ENHANCEMENTS` - :rtype: :class:`list` of :class:`str` - """ def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 29dcf3200..ea7d0e503 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,6 +84,10 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) + def test_supported_enhancements(self): + self.assertEqual( + self._create_prepared_installer().supported_enhancements(), []) + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 60c6cc5f2abf12884506172233894e358e18e670 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 15:26:52 -0700 Subject: [PATCH 186/639] Write enhance() --- certbot-postfix/certbot_postfix/installer.py | 24 ++++++++----------- .../certbot_postfix/installer_test.py | 5 ++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 45768df6c..189836bbe 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -142,6 +142,16 @@ class Installer(plugins_common.Plugin): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def enhance(self, domain, enhancement, options=None): + """Raises an exception for request for unsupported enhancement. + + :raises .PluginError: this is always raised as no enhancements + are currently supported + + """ + raise errors.PluginError( + "Unsupported enhancement: {0}".format(enhancement)) + def supported_enhancements(self): """Returns a list of supported enhancements. @@ -253,20 +263,6 @@ class Installer(plugins_common.Plugin): self.set_domainwise_tls_policies() self.update_CAfile() - def enhance(self, domain, enhancement, options=None): - """Perform a configuration enhancement. - :param str domain: domain for which to provide enhancement - :param str enhancement: An enhancement as defined in - :const:`~letsencrypt.constants.ENHANCEMENTS` - :param options: Flexible options parameter for enhancement. - Check documentation of - :const:`~letsencrypt.constants.ENHANCEMENTS` - for expected options for each enhancement. - :raises .PluginError: If Enhancement is not supported, or if - an error occurs during the enhancement. - """ - - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index ea7d0e503..1796c60ca 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -84,6 +84,11 @@ class InstallerTest(certbot_test_util.TempDirTestCase): self.assertTrue("fubard.org" in result) self.assertTrue("mail.fubard.org" in result) + def test_enhance(self): + self.assertRaises(errors.PluginError, + self._create_prepared_installer().enhance, + "example.org", "redirect") + def test_supported_enhancements(self): self.assertEqual( self._create_prepared_installer().supported_enhancements(), []) From 0f4c5c230538458f99418f28b4f3658477d59bc7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 11:41:30 -0700 Subject: [PATCH 187/639] Use common installer base --- certbot-postfix/certbot_postfix/installer.py | 20 +------------------ .../certbot_postfix/installer_test.py | 6 +++--- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 189836bbe..366824f07 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) -class Installer(plugins_common.Plugin): +class Installer(plugins_common.Installer): """Certbot installer plugin for Postfix.""" description = "Configure TLS with the Postfix MTA" @@ -278,24 +278,6 @@ class Installer(plugins_common.Plugin): """ self.maybe_add_config_lines() - def rollback_checkpoints(self, rollback=1): - """Revert `rollback` number of configuration checkpoints. - :raises .PluginError: when configuration cannot be fully reverted - """ - - def recovery_routine(self): - """Revert configuration to most recent finalized checkpoint. - Remove all changes (temporary and permanent) that have not been - finalized. This is useful to protect against crashes and other - execution interruptions. - :raises .errors.PluginError: If unable to recover the configuration - """ - - def view_config_changes(self): - """Display all of the LE config changes. - :raises .PluginError: when config changes cannot be parsed - """ - def config_test(self): """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 1796c60ca..547a954e2 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -24,12 +24,12 @@ myhostname = mail.fubard.org myorigin = fubard.org""" -class InstallerTest(certbot_test_util.TempDirTestCase): +class InstallerTest(certbot_test_util.ConfigTestCase): def setUp(self): super(InstallerTest, self).setUp() - self.config = mock.MagicMock(postfix_config_dir=self.tempdir, - postfix_config_utility="postconf") + self.config.postfix_config_dir = self.tempdir + self.config.postfix_config_utility = "postconf" def test_add_parser_arguments(self): mock_add = mock.MagicMock() From a29a99fb6ff359630b34629f2b1ec35cfc1455a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:34:03 -0700 Subject: [PATCH 188/639] Add --postfix-ctl flag. --- certbot-postfix/certbot_postfix/installer.py | 4 +++- certbot-postfix/certbot_postfix/installer_test.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 366824f07..3c4a4a85b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -28,9 +28,11 @@ class Installer(plugins_common.Installer): @classmethod def add_parser_arguments(cls, add): + add("ctl", default="postfix", + help="Path to the 'postfix' control program.") add("config-dir", help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " - "default configuration paths") + "default configuration paths.") add("config-utility", default="postconf", help="Path to the 'postconf' executable.") diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 547a954e2..a761a4386 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -32,13 +32,14 @@ class InstallerTest(certbot_test_util.ConfigTestCase): self.config.postfix_config_utility = "postconf" def test_add_parser_arguments(self): + options = set(('ctl', 'config-dir', 'config-utility',)) mock_add = mock.MagicMock() from certbot_postfix import installer installer.Installer.add_parser_arguments(mock_add) for call in mock_add.call_args_list: - self.assertTrue(call[0][0] in ('config-dir', 'config-utility')) + self.assertTrue(call[0][0] in options) def test_no_postconf_prepare(self): installer = self._create_installer() From b4b5c447500ef3f328d14efc99b6ca0a2ce54f28 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:38:07 -0700 Subject: [PATCH 189/639] Check postfix executable is found. --- certbot-postfix/certbot_postfix/installer.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3c4a4a85b..73e54b4f5 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -51,24 +51,27 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: when version is not supported """ - self._verify_postconf_available() + for param in ("ctl", "config_dir",): + self._verify_executable_is_available(param) self._set_config_dir() self._check_version() self._lock_config_dir() - def _verify_postconf_available(self): - """Ensure 'postconf' can be found. + def _verify_executable_is_available(self, config_name): + """Asserts the program in the specified config param is found. - :raises .NoInstallationError: when unable to find 'postconf' + :param str config_name: name of the config param + + :raises .NoInstallationError: when the executable isn't found """ - if not certbot_util.exe_exists(self.conf("config-utility")): - if not plugins_util.path_surgery(self.conf("config-utility")): + if not certbot_util.exe_exists(self.conf(config_name)): + if not plugins_util.path_surgery(self.conf(config_name)): raise errors.NoInstallationError( "Cannot find executable '{0}'. You can provide the " "path to this command with --{1}".format( - self.conf("config-utility"), - self.option_name("config-utility"))) + self.conf(config_name), + self.option_name(config_name))) def _set_config_dir(self): """Ensure self.config_dir is set to the correct path. From 2ae187b1d6bffb6cc63aaa36ddea14be3b7675d3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:55:21 -0700 Subject: [PATCH 190/639] Update config_test method --- certbot-postfix/certbot_postfix/installer.py | 46 +++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 73e54b4f5..60c8b3109 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -165,6 +165,41 @@ class Installer(plugins_common.Installer): """ return [] + def config_test(self): + """Make sure the configuration is valid. + + :raises .MisconfigurationError: if the config is invalid + + """ + try: + self._run_postfix_subcommand("check") + except subprocess.CalledProcessError: + raise errors.MisconfigurationError( + "Postfix failed internal configuration check.") + + def _run_postfix_subcommand(self, subcommand): + """Runs a subcommand of the 'postfix' control program. + + If the command fails, the exception is logged at the DEBUG + level. + + :param str subcommand: subcommand to run + + :raises subprocess.CalledProcessError: if the command fails + + """ + cmd = [self.conf("ctl")] + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + cmd.append(subcommand) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + logger.debug("%s exited with a non-zero status.", + "".join(cmd), exc_info=True) + raise + def ensure_cf_var(self, var, ideal, also_acceptable): """ Ensure that existing postfix config @var is in the list of @acceptable @@ -283,17 +318,6 @@ class Installer(plugins_common.Installer): """ self.maybe_add_config_lines() - def config_test(self): - """Make sure the configuration is valid. - :raises .MisconfigurationError: when the config is not in a usable state - """ - if os.geteuid() != 0: - rc = os.system('sudo /usr/sbin/postfix check') - else: - rc = os.system('/usr/sbin/postfix check') - if rc != 0: - raise errors.MisconfigurationError('Postfix failed self-check.') - def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted From 5f3be9b1cf667bae9ff8c66c6ee3ae38c1873ae8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 13:58:45 -0700 Subject: [PATCH 191/639] test config_test failure --- certbot-postfix/certbot_postfix/installer_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a761a4386..a7a9d27fa 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -28,6 +28,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): def setUp(self): super(InstallerTest, self).setUp() + self.config.postfix_ctl = "postfix" self.config.postfix_config_dir = self.tempdir self.config.postfix_config_utility = "postconf" @@ -94,6 +95,12 @@ class InstallerTest(certbot_test_util.ConfigTestCase): self.assertEqual( self._create_prepared_installer().supported_enhancements(), []) + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_config_test_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") + self.assertRaises(errors.MisconfigurationError, installer.config_test) + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 68dc678eed9d997a446f54eca55266a9e8531d4f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:02:57 -0700 Subject: [PATCH 192/639] Call config_test in prepare --- certbot-postfix/certbot_postfix/installer.py | 1 + certbot-postfix/certbot_postfix/installer_test.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 60c8b3109..910fa67a6 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -55,6 +55,7 @@ class Installer(plugins_common.Installer): self._verify_executable_is_available(param) self._set_config_dir() self._check_version() + self.config_test() self._lock_config_dir() def _verify_executable_is_available(self, config_name): diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index a7a9d27fa..425f0dd54 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -61,6 +61,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): expected = self.config.postfix_config_dir self.config.postfix_config_dir = None + check_call_path = "certbot_postfix.installer.subprocess.check_call" check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" with mock.patch(check_output_path) as mock_check_output: @@ -68,7 +69,8 @@ class InstallerTest(certbot_test_util.ConfigTestCase): "config_directory = " + expected, "mail_version = 3.1.4" ] with mock.patch(exe_exists_path, return_value=True): - installer.prepare() + with mock.patch(check_call_path): + installer.prepare() self.assertEqual(installer.config_dir, expected) def test_lock_error(self): @@ -159,12 +161,14 @@ class InstallerTest(certbot_test_util.ConfigTestCase): """ installer = self._create_installer() + check_call_path = "certbot_postfix.installer.subprocess.check_call" check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" with mock.patch(check_output_path) as mock_check_output: with mock.patch(exe_exists_path, return_value=True): - mock_check_output.return_value = "mail_version = 3.1.4" - installer.prepare() + with mock.patch(check_call_path): + mock_check_output.return_value = "mail_version = 3.1.4" + installer.prepare() return installer From a6c08a2e255801eaa9daedb050ee4d588c1b89ab Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:03:29 -0700 Subject: [PATCH 193/639] Update prepare docstring. --- certbot-postfix/certbot_postfix/installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 910fa67a6..94cfa3194 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -47,6 +47,7 @@ class Installer(plugins_common.Installer): Finish up any additional initialization. :raises errors.PluginError: when an unexpected error occurs + :raises errors.MisconfigurationError: when the config is invalid :raises errors.NoInstallationError: when can't find installation :raises errors.NotSupportedError: when version is not supported From 218e15c9d4fc5b74ae57b1a57e8c1ab28a57b620 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:25:22 -0700 Subject: [PATCH 194/639] Make restart() more robust. --- certbot-postfix/certbot_postfix/installer.py | 65 ++++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 94cfa3194..d73b73a37 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -179,6 +179,59 @@ class Installer(plugins_common.Installer): raise errors.MisconfigurationError( "Postfix failed internal configuration check.") + def restart(self): + """Restart or refresh the server content. + + :raises .PluginError: when server cannot be restarted + + """ + logger.info("Reloading Postfix configuration...") + if self._is_postfix_running(): + self._reload() + else: + self._start() + + def _is_postfix_running(self): + """Is Postfix currently running? + + Uses the 'postfix status' command to determine if Postfix is + currently running using the specified configuration files. + + :returns: True if Postfix is running, otherwise, False + :rtype: bool + + """ + try: + self._run_postfix_subcommand("status") + except subprocess.CalledProcessError: + return False + return True + + def _reload(self): + """Instructions Postfix to reload its configuration. + + If Postfix isn't currently running, this method will fail. + + :raises .PluginError: when Postfix cannot reload + + """ + try: + self._run_postfix_subcommand("reload") + except subprocess.CalledProcessError: + raise errors.PluginError( + "Postfix failed to reload its configuration.") + + def _start(self): + """Instructions Postfix to start running. + + :raises .PluginError: when Postfix cannot start + + """ + try: + self._run_postfix_subcommand("start") + except subprocess.CalledProcessError: + raise errors.PluginError("Postfix failed to start") + def _run_postfix_subcommand(self, subcommand): """Runs a subcommand of the 'postfix' control program. @@ -320,18 +373,6 @@ class Installer(plugins_common.Installer): """ self.maybe_add_config_lines() - def restart(self): - """Restart or refresh the server content. - :raises .PluginError: when server cannot be restarted - """ - logger.info('Reloading postfix config...') - if os.geteuid() != 0: - rc = os.system("sudo service postfix reload") - else: - rc = os.system("service postfix reload") - if rc != 0: - raise errors.MisconfigurationError('cannot restart postfix') - def get_config_var(self, name, default=False): """Return the value of the specified Postfix config parameter. From b21b66c0c07ac3b33db0dc199a946c667dee6106 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Aug 2017 14:29:50 -0700 Subject: [PATCH 195/639] Test restart command. --- .../certbot_postfix/installer_test.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 425f0dd54..69d999b31 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -103,6 +103,33 @@ class InstallerTest(certbot_test_util.ConfigTestCase): mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") self.assertRaises(errors.MisconfigurationError, installer.config_test) + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_reload_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = [ + None, subprocess.CalledProcessError(42, "foo") + ] + self.assertRaises(errors.PluginError, installer.restart) + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_reload_success(self, mock_check_call): + installer = self._create_prepared_installer() + installer.restart() + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_start_failure(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") + self.assertRaises(errors.PluginError, installer.restart) + + @mock.patch("certbot_postfix.installer.subprocess.check_call") + def test_postfix_start_success(self, mock_check_call): + installer = self._create_prepared_installer() + mock_check_call.side_effect = [ + subprocess.CalledProcessError(42, "foo"), None + ] + installer.restart() + def test_get_config_var_success(self): self.config.postfix_config_dir = None From 11b820c0e4b8a60b9145cfea97d9a0ba89149bb7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:41:07 -0700 Subject: [PATCH 196/639] add set_config_var --- certbot-postfix/certbot_postfix/installer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index d73b73a37..e384e0396 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -40,6 +40,7 @@ class Installer(plugins_common.Installer): super(Installer, self).__init__(*args, **kwargs) self.fixup = False self.config_dir = None + self.proposed_changes = {} def prepare(self): """Prepare the installer. @@ -426,6 +427,19 @@ class Installer(plugins_common.Installer): return cmd + def set_config_var(self, name, value): + """Set the Postfix config parameter name to value. + + This method only stores the requested change in memory. The + Postfix configuration is not modified until save() is called. + + :param str name: name of the Postfix config parameter + :param str value: value to set the Postfix config parameter to + + """ + assert isinstance(name, str), "Invalid name value" + assert isinstance(value, str), "Invalid key value" + self.proposed_changes[name] = value # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) From 4805fb4b88945863f03a3aaa7638ce2684ca6fd8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:52:05 -0700 Subject: [PATCH 197/639] Add deploy_cert --- certbot-postfix/certbot_postfix/installer.py | 36 +++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index e384e0396..3b67cfa9c 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -150,6 +150,26 @@ class Installer(plugins_common.Installer): return set(self.get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def deploy_cert(self, domain, cert_path, + key_path, chain_path, fullchain_path): + """Configure the Postfix SMTP server to use the given TLS cert. + + :param str domain: domain to deploy certificate file + :param str cert_path: absolute path to the certificate file + :param str key_path: absolute path to the private key file + :param str chain_path: absolute path to the certificate chain file + :param str fullchain_path: absolute path to the certificate fullchain + file (cert plus chain) + + :raises .PluginError: when cert cannot be deployed + + """ + self.set_config_var("smtpd_tls_cert_file", fullchain_path) + self.set_config_var("smtpd_tls_key_file", key_path) + self.set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") + self.set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") + self.set_config_var("smtpd_use_tls", "yes") + def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -343,22 +363,6 @@ class Installer(plugins_common.Installer): with open(self.fn, "w") as f: f.write(self.new_cf) - def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): - """Deploy certificate. - :param str domain: domain to deploy certificate file - :param str cert_path: absolute path to the certificate file - :param str key_path: absolute path to the private key file - :param str chain_path: absolute path to the certificate chain file - :param str fullchain_path: absolute path to the certificate fullchain - file (cert plus chain) - :raises .PluginError: when cert cannot be deployed - """ - self.wrangle_existing_config() - self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) - self.ensure_cf_var("smtpd_tls_key_file", key_path, []) - self.set_domainwise_tls_policies() - self.update_CAfile() - def save(self, title=None, temporary=False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be From d0ea5958f90505b6ac6c3b45b3bc06eda165e17e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 14:54:53 -0700 Subject: [PATCH 198/639] Protect _set_config_var --- certbot-postfix/certbot_postfix/installer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3b67cfa9c..7281a7e4d 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -164,11 +164,11 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ - self.set_config_var("smtpd_tls_cert_file", fullchain_path) - self.set_config_var("smtpd_tls_key_file", key_path) - self.set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") - self.set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - self.set_config_var("smtpd_use_tls", "yes") + self._set_config_var("smtpd_tls_cert_file", fullchain_path) + self._set_config_var("smtpd_tls_key_file", key_path) + self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") + self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") + self._set_config_var("smtpd_use_tls", "yes") def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -431,7 +431,7 @@ class Installer(plugins_common.Installer): return cmd - def set_config_var(self, name, value): + def _set_config_var(self, name, value): """Set the Postfix config parameter name to value. This method only stores the requested change in memory. The From 72637b2cf6ae546c1d2b316e677b937206967cf8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 15:20:22 -0700 Subject: [PATCH 199/639] Add util.check_call --- certbot-postfix/certbot_postfix/util.py | 33 ++++++++++++++++++-- certbot-postfix/certbot_postfix/util_test.py | 23 ++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index d7145b7c8..b65e4231e 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -7,6 +7,24 @@ import subprocess logger = logging.getLogger(__name__) +def check_call(*args, **kwargs): + """A simple wrapper of subprocess.check_call that logs errors. + + :param tuple args: positional arguments to subprocess.check_call + :param dict kargs: keyword arguments to subprocess.check_call + + :raises subprocess.CalledProcessError: if the call fails + + """ + try: + subprocess.check_call(*args, **kwargs) + except subprocess.CalledProcessError: + cmd = _get_cmd(*args, **kwargs) + logger.debug("%s exited with a non-zero status.", + "".join(cmd), exc_info=True) + raise + + def check_output(*args, **kwargs): """Backported version of subprocess.check_output for Python 2.6+. @@ -43,10 +61,19 @@ def check_output(*args, **kwargs): output, unused_err = process.communicate() retcode = process.poll() if retcode: - cmd = kwargs.get('args') - if cmd is None: - cmd = args[0] + cmd = _get_cmd(*args, **kwargs) logger.debug( "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) raise subprocess.CalledProcessError(retcode, cmd) return output + + +def _get_cmd(*args, **kwargs): + """Return the command from Popen args. + + :param tuple args: Popen args + :param dict kwargs: Popen kwargs + + """ + cmd = kwargs.get('args') + return args[0] if cmd is None else cmd diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py index 019f34532..95253e1fd 100644 --- a/certbot-postfix/certbot_postfix/util_test.py +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -5,6 +5,29 @@ import unittest import mock +class CheckCallTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_call.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_call + return check_call(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.check_call') + def test_failure(self, mock_check_call, mock_logger): + cmd = "postconf smtpd_use_tls=yes".split() + mock_check_call.side_effect = subprocess.CalledProcessError(42, cmd) + self.assertRaises(subprocess.CalledProcessError, self._call, cmd) + self.assertTrue(mock_logger.method_calls) + + @mock.patch('certbot_postfix.util.subprocess.check_call') + def test_success(self, mock_check_call): + cmd = "postconf smtpd_use_tls=yes".split() + self._call(cmd) + mock_check_call.assert_called_once_with(cmd) + + class CheckOutputTest(unittest.TestCase): """Tests for certbot_postfix.util.check_output.""" From d663f7981a08ff47477047a8d37e09f3be6aaa63 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 15:24:51 -0700 Subject: [PATCH 200/639] Add _write_config_changes --- certbot-postfix/certbot_postfix/installer.py | 37 ++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 7281a7e4d..3f4ff0227 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -269,12 +269,7 @@ class Installer(plugins_common.Installer): cmd.extend(("-c", self.conf("config-dir"),)) cmd.append(subcommand) - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - logger.debug("%s exited with a non-zero status.", - "".join(cmd), exc_info=True) - raise + util.check_call(cmd) def ensure_cf_var(self, var, ideal, also_acceptable): """ @@ -419,10 +414,7 @@ class Installer(plugins_common.Installer): :rtype: list """ - cmd = [self.conf("config-utility")] - - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) + cmd = self._postconf_command_base() if default: cmd.append("-d") @@ -445,6 +437,31 @@ class Installer(plugins_common.Installer): assert isinstance(value, str), "Invalid key value" self.proposed_changes[name] = value + def _write_config_changes(self): + """Write proposed changes to the Postfix config. + + :raises errors.PluginError: if an error occurs + + """ + cmd = self._postconf_command_base() + cmd.extend("{0}={1}".format(name, value) + for name, value in self.proposed_changes.items()) + + try: + util.check_call(cmd) + except subprocess.CalledProcessError: + raise errors.PluginError( + "An error occurred while updating your Postfix config.") + + def _postconf_command_base(self): + """Builds start of a postconf command using the selected config.""" + cmd = [self.conf("config-utility")] + + if self.conf("config-dir") is not None: + cmd.extend(("-c", self.conf("config-dir"),)) + + return cmd + # def update_CAfile(self): # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) # From a339de80f43966e5551a19ba8b5c53a7be129a40 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 28 Aug 2017 17:17:29 -0700 Subject: [PATCH 201/639] Add save() --- certbot-postfix/certbot_postfix/installer.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 3f4ff0227..71515c687 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -41,6 +41,7 @@ class Installer(plugins_common.Installer): self.fixup = False self.config_dir = None self.proposed_changes = {} + self.save_notes = [] def prepare(self): """Prepare the installer. @@ -164,6 +165,7 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ + self.save_notes.append("Configuring TLS for {0}".format(domain)) self._set_config_var("smtpd_tls_cert_file", fullchain_path) self._set_config_var("smtpd_tls_key_file", key_path) self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") @@ -188,6 +190,32 @@ class Installer(plugins_common.Installer): """ return [] + def save(self, title=None, temporary=False): + """Creates backups and writes changes to configuration files. + + :param str title: The title of the save. If a title is given, the + configuration will be saved as a new checkpoint and put in a + timestamped directory. `title` has no effect if temporary is true. + + :param bool temporary: Indicates whether the changes made will + be quickly reversed in the future (challenges) + + :raises errors.PluginError: when save is unsuccessful + + """ + if not self.proposed_changes: + return + + self.add_to_checkpoint(os.path.join(self.config_dir, "main.cf"), + "\n".join(self.save_notes), temporary) + self._write_config_changes() + + self.proposed_changes.clear() + del self.save_notes[:] + + if title and not temporary: + self.finalize_checkpoint(title) + def config_test(self): """Make sure the configuration is valid. @@ -436,6 +464,7 @@ class Installer(plugins_common.Installer): assert isinstance(name, str), "Invalid name value" assert isinstance(value, str), "Invalid key value" self.proposed_changes[name] = value + self.save_notes.append("\t* Set {0} to {1}".format(name, value)) def _write_config_changes(self): """Write proposed changes to the Postfix config. From 142bc3354568853c4b32e8998699bc37c0d966ce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Aug 2017 10:38:53 -0700 Subject: [PATCH 202/639] Remove dead code --- certbot-postfix/certbot_postfix/installer.py | 152 +------------------ 1 file changed, 1 insertion(+), 151 deletions(-) diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 71515c687..09e1cc18b 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -38,8 +38,7 @@ class Installer(plugins_common.Installer): def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) - self.fixup = False - self.config_dir = None + self.config_dir = None self.proposed_changes = {} self.save_notes = [] @@ -299,108 +298,6 @@ class Installer(plugins_common.Installer): util.check_call(cmd) - def ensure_cf_var(self, var, ideal, also_acceptable): - """ - Ensure that existing postfix config @var is in the list of @acceptable - values; if not, set it to the ideal value. - - :raises .errors.MisconfigurationError: if conflicting existing values - are found for var - - """ - acceptable = [ideal] + also_acceptable - - l = [(num,line) for num,line in enumerate(self.cf) - if line.startswith(var)] - if not any(l): - self.additions.append(var + " = " + ideal) - else: - values = map(parse_line, l) - if len(set(values)) > 1: - if self.fixup: - conflicting_lines = [num for num,_var,val in values] - self.deletions.extend(conflicting_lines) - self.additions.append(var + " = " + ideal) - else: - raise errors.MisconfigurationError( - "Conflicting existing config values {0}".format(l) - ) - val = values[0][2] - if val not in acceptable: - if self.fixup: - self.deletions.append(values[0][0]) - self.additions.append(var + " = " + ideal) - else: - raise errors.MisconfigurationError( - "Existing config has %s=%s"%(var,val) - ) - - def wrangle_existing_config(self): - """ - Try to ensure/mutate that the config file is in a sane state. - Fixup means we'll delete existing lines if necessary to get there. - """ - # Check we're currently accepting inbound STARTTLS sensibly - self.ensure_cf_var("smtpd_use_tls", "yes", []) - # Ideally we use it opportunistically in the outbound direction - self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) - # Maximum verbosity lets us collect failure information - self.ensure_cf_var("smtp_tls_loglevel", "1", []) - # Inject a reference to our per-domain policy map - # policy_cf_entry = "texthash:" + self.policy_file - - # self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) - # self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) - - # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed - # between Postfix version 2.5 and 2.6, since we only support => 2.11 - # we don't use nor support legacy Postfix syntax. - # - Server: - self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - # - Client: - self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) - self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) - - def maybe_add_config_lines(self): - if not self.additions: - return - if self.fixup: - logger.info('Deleting lines: {}'.format(self.deletions)) - self.additions[:0]=["#", - "# New config lines added by STARTTLS Everywhere", - "#"] - new_cf_lines = "\n".join(self.additions) + "\n" - logger.info('Adding to {}:'.format(self.fn)) - logger.info(new_cf_lines) - if self.raw_cf[-1][-1] == "\n": sep = "" - else: sep = "\n" - - for num, line in enumerate(self.raw_cf): - if self.fixup and num in self.deletions: - self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line - else: - self.new_cf += line - self.new_cf += sep + new_cf_lines - - with open(self.fn, "w") as f: - f.write(self.new_cf) - - def save(self, title=None, temporary=False): - """Saves all changes to the configuration files. - Both title and temporary are needed because a save may be - intended to be permanent, but the save is not ready to be a full - checkpoint. If an exception is raised, it is assumed a new - checkpoint was not created. - :param str title: The title of the save. If a title is given, the - configuration will be saved as a new checkpoint and put in a - timestamped directory. `title` has no effect if temporary is true. - :param bool temporary: Indicates whether the changes made will - be quickly reversed in the future (challenges) - :raises .PluginError: when save is unsuccessful - """ - self.maybe_add_config_lines() - def get_config_var(self, name, default=False): """Return the value of the specified Postfix config parameter. @@ -490,50 +387,3 @@ class Installer(plugins_common.Installer): cmd.extend(("-c", self.conf("config-dir"),)) return cmd - - # def update_CAfile(self): - # os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) - # - # def set_domainwise_tls_policies(self): - # all_acceptable_mxs = self.policy_config.acceptable_mxs - # for address_domain, properties in all_acceptable_mxs.items(): - # mx_list = properties.accept_mx_domains - # if len(mx_list) > 1: - # logger.warn('Lists of multiple accept-mx-domains not yet ' - # 'supported.') - # logger.warn('Using MX {} for {}'.format(mx_list[0], - # address_domain) - # ) - # logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) - # mx_domain = mx_list[0] - # mx_policy = self.policy_config.get_tls_policy(mx_domain) - # entry = address_domain + " encrypt" - # if mx_policy.min_tls_version.lower() == "tlsv1": - # entry += " protocols=!SSLv2:!SSLv3" - # elif mx_policy.min_tls_version.lower() == "tlsv1.1": - # entry += " protocols=!SSLv2:!SSLv3:!TLSv1" - # elif mx_policy.min_tls_version.lower() == "tlsv1.2": - # entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" - # else: - # logger.warn('Unknown minimum TLS version: {} '.format( - # mx_policy.min_tls_version) - # ) - # self.policy_lines.append(entry) - - # with open(self.policy_file, "w") as f: - # f.write("\n".join(self.policy_lines) + "\n") - - -def parse_line(line_data): - """ - Return the (line number, left hand side, right hand side) of a stripped - postfix config line. - - Lines are like: - smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache - """ - num,line = line_data - left, sep, right = line.partition("=") - if not sep: - return None - return (num, left.strip(), right.strip()) From dde0bf08217c6fd31d83922977a233328124bcfd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Aug 2017 10:42:31 -0700 Subject: [PATCH 203/639] Remove non-plugin files --- .gitignore | 4 - README.md | 234 - Vagrantfile | 27 - examples/bigger_test_config.json | 36 - examples/config.json | 17 - examples/starttls-everywhere.json | 187 - letsencrypt-postfix/Config.py | 569 -- letsencrypt-postfix/PostfixLogSummary.py | 104 - letsencrypt-postfix/TestConfig.py | 132 - requirements.txt | 4 - share/golden-domains.txt | 32 - share/google-starttls-domains.csv | 6734 ----------------- tools/CheckSTARTTLS.py | 191 - tools/ProcessGoogleSTARTTLSDomains.py | 34 - vagrant-bootstrap.sh | 31 - vagrant-shared/certificates/ca.crt | 22 - vagrant-shared/certificates/ca.key | 27 - vagrant-shared/certificates/certificates | 1 - vagrant-shared/certificates/valid.crt | 21 - vagrant-shared/certificates/valid.csr | 18 - vagrant-shared/certificates/valid.key | 27 - .../postfix-config-sender-tls_policy | 1 - vagrant-shared/postfix-config-sender.cf | 39 - .../postfix-config-valid-example-recipient.cf | 46 - 24 files changed, 8538 deletions(-) delete mode 100644 .gitignore delete mode 100644 README.md delete mode 100644 Vagrantfile delete mode 100644 examples/bigger_test_config.json delete mode 100644 examples/config.json delete mode 100644 examples/starttls-everywhere.json delete mode 100644 letsencrypt-postfix/Config.py delete mode 100755 letsencrypt-postfix/PostfixLogSummary.py delete mode 100755 letsencrypt-postfix/TestConfig.py delete mode 100644 requirements.txt delete mode 100644 share/golden-domains.txt delete mode 100644 share/google-starttls-domains.csv delete mode 100755 tools/CheckSTARTTLS.py delete mode 100755 tools/ProcessGoogleSTARTTLSDomains.py delete mode 100755 vagrant-bootstrap.sh delete mode 100644 vagrant-shared/certificates/ca.crt delete mode 100644 vagrant-shared/certificates/ca.key delete mode 120000 vagrant-shared/certificates/certificates delete mode 100644 vagrant-shared/certificates/valid.crt delete mode 100644 vagrant-shared/certificates/valid.csr delete mode 100644 vagrant-shared/certificates/valid.key delete mode 100644 vagrant-shared/postfix-config-sender-tls_policy delete mode 100644 vagrant-shared/postfix-config-sender.cf delete mode 100644 vagrant-shared/postfix-config-valid-example-recipient.cf diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e36c9c50b..000000000 --- a/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.* -*.orig -*.pyc -*.egg-info/ diff --git a/README.md b/README.md deleted file mode 100644 index 7413228f6..000000000 --- a/README.md +++ /dev/null @@ -1,234 +0,0 @@ -# STARTTLS Everywhere - - -## Example usage - -**WARNING: this is a pre-alpha codebase. Do not run it on production -mailservers!!!** - - -If you have a Postfix server you're willing to endanger deliverability on, you -can try obtain a certificate with the [Let's Encrypt Python Client](https://github.com/letsencrypt/letsencrypt), note the directory it lives in below `/etc/letsencrypt/live` and then do: - -``` -git clone https://github.com/EFForg/starttls-everywhere -cd starttls-everywhere -# Promise you don't care if deliverability breaks on this mail server -letsencrypt-postfix/PostfixConfigGenerator.py examples/starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/YOUR.DOMAIN.EXAMPLE.COM -``` - -This will: -* Ensure your mail server initiates STARTTLS encryption -* Install the Let's Encrypt cert in Postfix -* Enforce mandatory TLS to some major email domains -* Enforce minimum TLS versions to some major email domains - -## Project status - -STARTTLS Everywhere development is re-starting after a hiatus. Initial -objectives: - -* Postfix configuration generation: working pre-alpha, not yet safe -* Email security database: working pre-alpha, definitely not yet safe -* Fully integrated Let's Encrypt client postfix plugin: in progress, not yet ready -* DANE support: none yet -* DEEP support: none yet -* SMTP-STS integration: none yet -* Direct mechanisms for mail domains to request inclusion: none yet -* Failure reporting mechanisms: early progress, not yet ready -* Mechanisms for secure multi-organization signature on the policy database: - none yet -* Support for mail servers other than Postfix: none yet - -## Authors - -Jacob Hoffman-Andrews , -Peter Eckersley , -Daniel Wilcox , -Aaron Zauner - -## Mailing List - -starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere - -## Background - -Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to trivial downgrade attacks. - -To illustrate an easy version of this attack, suppose a network-based attacker `Mallory` notices that `Alice` has just uploaded message to her mail server. `Mallory` can inject a TCP reset (RST) packet during the mail server's next TLS negotiation with another mail server. Nearly all mail servers that implement STARTTLS do so in opportunistic mode, which means that they will retry without encryption if there is any problem with a TLS connection. So `Alice`'s message will be transmitted in the clear. - -Opportunistic TLS in SMTP also extends to certificate validation. Mail servers commonly provide self-signed certificates or certificates with non-validatable hostnames, and senders commonly accept these. This means that if we say 'require TLS for this mail domain,' the domain may still be vulnerable to a man-in-the-middle using any key and certificate chosen by the attacker. - -Even if senders require a valid certificate that matches the hostname of a mail host, a DNS MITM or Denial of Service is still possible. The sender, to find the correct target hostname, queries DNS for MX records on the recipient domain. Absent DNSSEC, the response can be spoofed to provide the attacker's hostname, for which the attacker holds a valid certificate. - -STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. - -## Goals - -* Prevent RST attacks from revealing email contents in transit between major MTAs that support STARTTLS. -* Prevent MITM attacks at the DNS, SMTP, TLS, or other layers from revealing same. -* Zero or minimal decrease to deliverability rates unless network attacks are actually occurring. -* Create feedback-loops on targeted attacks and bulk surveilance in an opt-in, anonymized way. - -## Non-goals - -* Prevent fully-targeted exploits of vulnerabilities on endpoints or on mail hosts. -* Refuse delivery on the recipient side if sender does not negotiate TLS (this may be a future project). -* Develop a fully-decentralized solution. -* Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. - -## Motivating examples - -* [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to - prevent negotiating an encrypted connection](https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml) -* [Unknown party removes STARTTLS flag from all SMTP connections leaving - Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand) - -## Threat model - -Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs. - -## Alternatives - -Our goals can also be accomplished through use of [DNSSEC and DANE](http://tools.ietf.org/html/draft-ietf-dane-smtp-with-dane-10), which is certainly a more scalable solution. However, operators have been very slow to roll out DNSSEC supprt. We feel there is value in deploying an intermediate solution that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of DANE-based solutions easier. - -## Detailed design - -Senders need to know which target hosts are known to support STARTTLS, and how to authenticate them. Since the network cannot be trusted to provide this information, it must be communicated securely out-of-band. We will provide: - - (a) a configuration file format to convey STARTTLS support for recipient domains, - - (b) Python code (config-generator) to transform (a) into configuration files for popular MTAs., and - - (c) a method to create and securely distribute files of type (a) for major email domains that that agree to be included, plus any other domains that proactively request to be included. - -## File Format - -The basic file format will be JSON with comments (http://blog.getify.com/json-comments/). Example: - - { - // Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version - "timestamp": "2014-06-06T14:30:16+00:00", - // "timestamp": 1401414363, : also acceptable - "author": "Electronic Frontier Foundation https://eff.org", - "expires": "2014-06-06T14:30:16+00:00", - "tls-policies": { - // These match on the MX domain. - "*.yahoodns.net": { - "require-valid-certificate": true, - } - "*.eff.org": { - "require-tls": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "enforce" - "accept-spki-hashes": [ - "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - ] - } - "*.google.com": { - "require-valid-certificate": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "log-only", - "error-notification": "https://google.com/post/reports/here" - }, - } - // Since the MX lookup is not secure, we list valid responses for each - // address domain, to protect against DNS spoofing. - "acceptable-mxs": { - "yahoo.com": { - "accept-mx-domains": ["*.yahoodns.net"] - } - "gmail.com": { - "accept-mx-domains": ["*.google.com"] - } - "eff.org": { - "accept-mx-domains": ["*.eff.org"] - } - } - } - - -A user of this file format may choose to accept multiple files. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. If so, they override each other on a per-domain basis. - -The _timestamp_ field is an integer number of epoch seconds from 00:00:00 UTC on 1 January 1970. When retrieving a fresh configuration file, config-generator should validate that the timestamp is greater than or equal to the version number of the file it already has. - -There is no inline signature field. The configuration file should be distributed with authentication using an offline signing key. - -Option 1: Plain JSON distributed with a signature using gpg --clearsign. Config-generator should validate the signature against a known GPG public key before extracting. The public key is part of the permanent system configuration, like the fetch URL. - -Option 2: Git is a revision control system built on top of an authenticated, history-preserving file system. Let's use it as an authenticated, history preserving file system: valid versions of recipient policy files may be fetched and verified via signed git tags. [Here's an example shell recipe to do this.](https://gist.github.com/jsha/6230206e89759cc6e00d) - -Config-generator should attempt to fetch the configuration file daily and transform it into MTA configs. If there is a retrieval failure, and the cached configuration file has an 'expires' time past the current date, an alert should be raised to the system operator and all existing configs from config-generator should be removed, reverting the MTA configuration to use opportunistic TLS for all domains. - -**address-domains** - -The _address-domains_ field maps from mail domains (the part of an address after the "@") onto a list of properties for that domain. Matching of mail domains is on an exact-match basis, not a subdomain basis. For instance, eff.org would be listed separately from lists.eff.org in the _address-domains_ section. - -Currently the only property defined for _address-domains_ is _accept-mx-domains_, a list. If an MX lookup for a listed address domain returns a hostname that is not a subdomain of one of the domains listed in the _accept-mx-domains_ property, the MTA should fail delivery or log an advisory failure, as appropriate. Matching of MX hostnames against the _accept-mx-domains_ list is on a subdomain basis. For instance, if an MX record for yahoo.com lists mta7.am0.yahoodns.net, and the _accept-mx-domains_ property for yahoo.com is ["yahoodns.net"], that should be considered a match. All domains listed in any _accept-mx-domains_ list must correspond to an exactly matching field in the _mx-domains_ config section. - -The _accept-mx-domains_ mechanism partially solves the problem of DNS MITM. It doesn't completely solve the problem, since an attacker might somehow control a different hostname under an acceptable domain, e.g. evil.yahoodns.net. But it strikes a balance between improving security and allowing mail operators to change configuration as needed. Some mail operators delegate their MX handling to a third-party provider (i.e. Google Apps for Your Domain). If those operators are included in STARTTLS Everywhere and wish to change providers, they will have to first send an update to their _accept-mx-domains_ to include their new provider. - -**mx-domains** - -The keys of this section are MX domains as described above for the _accept-mx-domains_ property. Each _mx-domain_ entry must be an exact match with an entry in one of the _accept-mx-domains_ lists provided. No _mx-domain_can be a subdomain of any other _mx-domain_in the configuration file. Fields in this section specify minimum security requirements that should be applied when connecting to any MX hostname that is a subdomain of the specified _mx-domain_. - -Implicitly each _mx-domain_ listed has a property _require-tls: true_. MX domains that do not support TLS will not be listed. The only required property is _enforce-mode_, which must be either _log-only_ or _enforce_. If _enforce-mode_ is _log-only_, the generated configs will not stop mail delivery on policy failures, but will produce logging information. - -If the _min-tls-version_ property is present, sending mail to domains under this policy should fail if the sending MTA cannot negotiate a TLS version equal to or greater than the listed version. Valid values are _TLSv1, TLSv1.1, and TLSv1.2._ - -_Require-valid-certificate_defaults to false. If the _require-valid-certificate_ property is 'true' for a given _mx-domain_ the certificate presented must be valid for a hostname that is subdomain of the _mx-domain_. Validity means all of these must be true: - -1. The CN or a DNS entry under subjectAltName matches an appropriate hostname. -2. The certificate is unexpired. -3. There is a valid chain from the certificate to a root certificate included in [Mozilla's trust store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/included/) (available as [Debian package ca-certificates](https://packages.debian.org/sid/ca-certificates)). - -The _accept-pinset_ field references an entry in the pinsets list, which has the same format and semantics as [Chrome's pinning list](https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json). Most _mx-domain_s should specify a pinset that describes trust roots rather than leaf certificates, but both are possible. Pinning will only be added at the request of mail operators because it requires operators be careful when issuing new leaf certificates. - -## Pinning and hostname verification - -Like Chrome and Firefox we want to encourage pinning to a trusted root or intermediate rather than a leaf cert, to minimize spurious pinning failures when hosts rotate keys. - -The other option is to automatically pin leaf certs as observed in the wild. This would be one solution to the hostname verification and self-signed certificate problem. However, it is a non-starter. Even if we expect mail operators to auto-update configuration on a daily basis, this approach cannot add new certs until they are observed in the wild. That means that any time an operator rotates keys on a mail server, there would be a significant window of time in which the new keys would be rejected. - -We do not attempt to solve the self-signed certificate problem. For mail hosts with self-signed certificates, we can require TLS but will not require validation of the certificates. Such hosts should be encouraged to upgrade to a CA-signed certificate that can be validated by senders. - -## Creating configuration - -We have three options for creating the configuration file: - -1. Ask mail operators to submit policies for their domains which we incorporate. -2. Manually curate a set of policies for the top `N` mail domains. -3. Programmatically create a set of policies by connecting to the top N mail domains. - -For option (1), there's a bootstrapping problem: No one will opt in until it's useful; It won't be useful until people opt in. Option (1) does have the advantage that it's the only good way to get pinning directives. - -For option (3) we'd be likely to pull in bad policies that could result in failed delivery. - -We'll initially launch a demo using option (2), do some initial deployments to prove viability and delivery rate impact, and then start reaching out to operators to do option (1). - -## Distribution - -The configuration file will be provided at a long-term maintained URL. It will be signed using a key held offline on an airgapped machine or smartcard. - -Since recipient mail servers may abruptly stop supporting TLS, we will request that mail operators set up auto-updating of the configuration file, with signature verification. This allows us to minimize the delivery impact of such events. However, config-generator should not auto-update its own code, since that would amount to auto-deployment of third party code, which some operators may not wish to do. - -We may choose to implement a form of immutable log along the lines of certificate transparency. This would be appealing if we chose to use this mechanism to distribute expected leaf keys as a primary authentication mechanism, but as described in "Pinning and hostname verification," that's not a viable option. Instead we will rely on the CA ecosystem to do primary authentication, so an immutable log for this system is probably overkill, engineering-wise. - -## Python code - -Config-generator should parse input JSON and produce output configs for various mail servers. It should not be possible for any input JSON to cause arbitrary code execution or even any MTA config directives beyond the ones that specifically impact the decision to deliver or bounce based on TLS support. For instance, it must not be possible for config-generator to output a directive to forward mail from one domain to another. Config-generator will have the option to directly pull the latest config from a URL, or from a file on local disk distributed regularly from another system that has outside network access. - -Config-generator will be manually updated by mail operators. - -## Testing - -We will create a reproducible test configuration that can be run locally and exercises each of the major cases: Enforce mode vs log mode; Enforced TLS negotiation, enforced MX hostname match, and enforced valid certificates. - -Additionally, for ongoing monitoring of third-party deployments, we will create a canary mail domain that intentionally fails one of the tests but is included in the configuration file. For instance, starttls-canary.org would be listed in the configuration as requiring STARTTLS, but would not actually offer STARTTLS. Each time a mail operator commits to configuring STARTTLS Everywhere, we would request an account on their email domain from which to send automated daily email to starttls-canary.org. We should expect bounces. If such mail is successfully delivered to starttls-canary.org, that would indicate a configuration failure on the sending host, and we would manually notify the operator. - -## Failure reporting - -For the mail operator deploying STARTTLS Everywhere, we will provide log analysis scripts that can be used out-of-the-box to monitor how many delivery failures or would-be failures are due to STARTTLS Everywhere policies. These would be designed to run in a cron job or small opt-in daemon and send notices only when STARTTLS Everywhere-related failures exceed a certain percentage for any given recipient domains. For very high-volume mail operators, it would likely be necessary to adapt the analysis scripts to their own logging and analysis infrastructure. - -For recipient domains who are listed in the STARTTLS Everywhere configuration, we would provide a configuration field to specify an email address or HTTPS URL to which that sender domains could send failure information. This would provide a mechanism for recipient domains to identify problems with their TLS deployment and fix them. The reported information should not contain any personal information, including email addresses. Example fields for failure reports: timestamps at minute granularity, target MX hostname, resolved MX IP address, failure type, certificate. Since failures are likely to come in batches, the error sending mechanism should batch them up and summarize as necessary to avoid flooding the recipient. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index b7153a7b8..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,27 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "hashicorp/precise32" - - config.vm.define "sender" do |sender| - sender.vm.network "private_network", ip: "192.168.33.5" - sender.vm.hostname = "sender.example.com" - end - config.vm.define "valid" do |valid| - valid.vm.network "private_network", ip: "192.168.33.7" - valid.vm.hostname = "valid-example-recipient.com" - end - config.vm.synced_folder "vagrant-shared", "/vagrant" - config.vm.synced_folder "vagrant-shared/starttls-everywhere", "/vagrant/starttls-everywhere" - config.vm.provision :shell, path: "vagrant-bootstrap.sh" - - config.vm.provider "virtualbox" do |vb| - # vb.gui = true - vb.customize ["modifyvm", :id, "--memory", "256"] - end - -end diff --git a/examples/bigger_test_config.json b/examples/bigger_test_config.json deleted file mode 100644 index c3c23c455..000000000 --- a/examples/bigger_test_config.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "timestamp": 1401414363, - "author": "Electronic Frontier Foundation https://eff.org", - "expires": "2015-08-01T12:00:00+08:00", - "tls-policies": { - ".yahoodns.net": { - "require-valid-certificate": true - }, - ".eff.org": { - "require-tls": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "enforce", - "accept-spki-hashes": [ - "sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=", - "sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM=" - ] - }, - ".google.com": { - "require-valid-certificate": true, - "min-tls-version": "TLSv1.1", - "enforce-mode": "log-only", - "error-notification": "https://google.com/post/reports/here" - } - }, - "acceptable-mxs": { - "yahoo.com": { - "accept-mx-domains": [".yahoodns.net"] - }, - "gmail.com": { - "accept-mx-domains": [".google.com"] - }, - "eff.org": { - "accept-mx-domains": [".eff.org"] - } - } -} diff --git a/examples/config.json b/examples/config.json deleted file mode 100644 index 05fc237bf..000000000 --- a/examples/config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "comment": "Canonical URL https://eff.org/starttls-everywhere/config -- redirects to latest version", - "timestamp": 1401093333, - "author": "Electronic Frontier Foundation https://eff.org", - "expires": 1404677353, "comment 2:": "epoch seconds", - "tls-policies": { - ".valid-example-recipient.com": { - "min-tls-version": "TLSv1.1" - } - }, - "acceptable-mxs": { - "valid-example-recipient.com": { - "accept-mx-domains": [ ".valid-example-recipient.com" ] - } - } - -} diff --git a/examples/starttls-everywhere.json b/examples/starttls-everywhere.json deleted file mode 100644 index db76745ef..000000000 --- a/examples/starttls-everywhere.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "acceptable-mxs": { - "163.com": { - "accept-mx-domains": [ - ".163.com" - ] - }, - "aol.com": { - "accept-mx-domains": [ - ".aol.com" - ] - }, - "craigslist.org": { - "accept-mx-domains": [ - ".craigslist.org" - ] - }, - "gmail.com": { - "accept-mx-domains": [ - ".google.com" - ] - }, - "hotmail.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "icloud.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "live.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "mac.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "me.com": { - "accept-mx-domains": [ - ".icloud.com" - ] - }, - "msn.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "naver.com": { - "accept-mx-domains": [ - ".naver.com" - ] - }, - "outlook.com": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "qq.com": { - "accept-mx-domains": [ - ".qq.com" - ] - }, - "rocketmail.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "rogers.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "salesforce.com": { - "accept-mx-domains": [ - ".psmtp.com" - ] - }, - "sbcglobal.net": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "shaw.ca": { - "accept-mx-domains": [ - ".shaw.ca" - ] - }, - "sympatico.ca": { - "accept-mx-domains": [ - ".outlook.com" - ] - }, - "t-online.de": { - "accept-mx-domains": [ - ".t-online.de" - ] - }, - "wp.pl": { - "accept-mx-domains": [ - ".wp.pl" - ] - }, - "yahoo.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "yahoogroups.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - }, - "yandex.ru": { - "accept-mx-domains": [ - ".yandex.ru" - ] - }, - "ymail.com": { - "accept-mx-domains": [ - ".yahoo.com" - ] - } - }, - "tls-policies": { - ".163.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".aol.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".craigslist.org": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".google.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".icloud.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".naver.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".outlook.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".psmtp.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".qq.com": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".shaw.ca": { - "min-tls-version": "TLSv1", - "require-tls": true - }, - ".t-online.de": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".wp.pl": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yahoo.com": { - "min-tls-version": "TLSv1.1", - "require-tls": true - }, - ".yandex.ru": { - "min-tls-version": "TLSv1.1", - "require-tls": true - } - } -} diff --git a/letsencrypt-postfix/Config.py b/letsencrypt-postfix/Config.py deleted file mode 100644 index cc0df00d1..000000000 --- a/letsencrypt-postfix/Config.py +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env python -from datetime import datetime -from dateutil import parser as dateutil_parser -import collections -import json -import logging -import pprint - - -"""Idea here being to start with something that is decomposed so it's easier to -make do json in *and* out, differences between configs and config extension. -""" - -#TODO scope logging and handlers better, control verbosity by command line flags -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - -def parse_bool_from_json(value, attr_name): - if value in ('true', '1', 1, 'yes'): - bool_value = True - elif value in ('false', '0', 0, 'no'): - bool_value = False - elif value in (True, False): - bool_value = value - else: - raise ConfigError('Config value %s is an invalid boolean value.' % attr_name) - return bool_value - - -def parse_timestamp(value, attr_name): - if isinstance(value, datetime): - return value - try: - ts = int(value) - return datetime.fromtimestamp(ts) - except (TypeError, ValueError): - pass - try: - return dateutil_parser.parse(value) - except (TypeError, ValueError): - raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name) - - -def verify_member_of(value, member_list, attr_name): - if value not in member_list: - raise ConfigError('Config value "%s" must be one of (%s)' % ( - attr_name, ', '.join(member_list)) - ) - return value - - -def verify_string(value, attr_name, max_length=200): - if not isinstance(value, (str, unicode)): - raise ConfigError('Config value %s must be a string.' % attr_name) - if len(value) > max_length: - raise ConfigError('Config value %s is too long.' % attr_name) - return value - - -def to_dict(config_dict): - """Cleans up BaseConfig children to be serialized.""" - d = {} - for key, val in config_dict.iteritems(): - if isinstance(val, BaseConfig): - d[key] = to_dict(val._data) - elif isinstance(val, datetime): - d[key] = val.strftime('%Y-%m-%dT%H:%M:%S%z') - elif isinstance(val, dict): - d[key] = to_dict(val) - else: - d[key] = val - return d - - -class BaseConfig(object): - """Top level config class for common methods. - - Requirements for using class: - - list all properties with getters *and* setters in class - variable 'config_properties' - - __init__ of child classes must be callable with *only* - keyword arguments to allow method calls to update to create - a new config - ... more ... - """ - - config_properties = [] - - def __init__(self): - # container for validated properties with JSON names - self._data = {} - - def __repr__(self): - s = '< %s %s >' % (self.__class__.__name__, - pprint.pformat(self._data)) - return s - - def update(self, newer_config, merge=False, **kwargs): - """Create a fresh config combining the new and old configs. - - It does this by iterating over the 'config_properties' class - attribute which contains names of property attributes for the config. - - Two methods of combining configs are possible, an 'update' and - a 'merge', the latter set by the keyword argument 'merge=True'. - - An update overrides older values with new values -- even if those - new values are None. Update will remove values that are present in - the old config if they are not present in the new config. - - A merge by comparison will allow old values to persist if they are - not specified in the new config. This can be used for end-user - customizations to override specific settings without having to re-create - large portions of a config to override it. - - Arguments: - newer_config: A config object to combine with the current config. - merge: Allows old values not overridden to survive into the fresh config. - - Returns: - A config object of the same sort as called upon. - """ - # removed 'merge' kw arg - and it was passed to constructor - # make a note to not do that, consume it on the param list - fresh_config = self.__class__(**kwargs) - logger.debug('from parent update kwargs %s' % kwargs) - logger.debug('from parent update merge %s' % merge) - if not isinstance(newer_config, self.__class__): - raise ConfigError('Attempting to update a %s with a %s' % ( - self.__class__, - newer_config.__class__)) - for prop_name in self.config_properties: - # get the specified property off of the current class - prop = self.__class__.__dict__.get(prop_name) - assert prop - new_value = prop.fget(newer_config) - old_value = prop.fget(self) - if new_value is not None: - prop.fset(fresh_config, new_value) - elif merge and old_value is not None: - prop.fset(fresh_config, old_value) - return fresh_config - - def merge(self, newer_config, **kwargs): - """Combines configs and keeps old values if they are not overridden. - - See docstring for 'update' method for more details. - - Arguments: - newer_config: A config object to combine with the current config. - merge: Allows old values not overridden to survive into the fresh config. - - Returns: - A config object of the same sort as called upon. - """ - kwargs['merge'] = True - logger.debug('from parent merge: %s' % kwargs) - return self.update(newer_config, **kwargs) - - def to_json(self): - d = to_dict(self._data) - return json.dumps(d) - - def write_to_json_file(self, json_filename, f_open=open): - data = self.to_json() - try: - with f_open(json_filename, 'w') as f: - f.write(data) - except IOError: - raise - - def load_from_json_file(self, json_filename, f_open=open): - try: - with f_open(json_filename, 'r') as f: - json_str = f.read() - json_dict = json.loads(json_str) - except IOError: - raise - except ValueError: - raise ConfigError('No valid JSON found in file: %s' % json_filename) - self.from_json_dict(json_dict) - - def from_json_dict(self, json_dict): - raise NotImplmented('BaseConfig should not be populated.') - - -class Config(BaseConfig): - """Config container for StartTLS Everywhere configuration. - - Intended as a simple container that unifies where validatation occurs, - and is capable of comparing configs to warn of things like changing - certificate fingerprints from one scan to the next. - - There is a one to one mapping of the object attributes to the JSON - object keys, albeit with dashes replaced with underscores. - """ - - def __init__(self): - super(self.__class__, self).__init__() - self._data['tls-policies'] = {} - self._data['acceptable-mxs'] = {} - - def __add__(self, other_config): - """Allow addition but not really of *full* configs, need to flesh that out.""" - #TODO add this - raise NotImplemented - - def update(self, other_config): - """Update properties of config from a 'newer' config and force verification.""" - #TODO add this - new_config = Config() - raise NotImplemented - - def from_json_dict(self, json_dict): - """Assign JSON data to Config properties and declare sub-objects. - - Let's property verification methods do the heavy lifting and mostly - maps between the JSON config names and attributes. Keeps track of - unused variables and warns about them. - """ - for key, val in json_dict.iteritems(): - if key == 'author': - self.author = val - elif key == 'comment': - self.comment = val - elif key == 'expires': - self.expires = val - elif key == 'timestamp': - self.timestamp = val - elif key == 'tls-policies': - self.make_tls_policy_dict(val) - elif key == 'acceptable-mxs': - self.make_acceptable_mxs_dict(val) - else: - logger.warn('Unknown attribute "%s", skipping' % key) - - @property - def author(self): - return self._data.get('author') - - @author.setter - def author(self, value): - self._data['author'] = verify_string(value, 'author') - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - @property - def expires(self): - return self._data.get('expires') - - @expires.setter - def expires(self, value): - self._data['expires'] = parse_timestamp(value, 'expires') - - @property - def timestamp(self): - return self._data.get('timestamp') - - @timestamp.setter - def timestamp(self, value): - self._data['timestamp'] = parse_timestamp(value, 'timestamp') - - @property - def tls_policies(self): - return self._data.get('tls-policies') - - @property - def acceptable_mxs(self): - return self._data.get('acceptable-mxs') - - def make_tls_policy_dict(self, policy_dict): - tls_policy_dict = self.tls_policies - for domain_suffix, settings in policy_dict.iteritems(): - new_domain_policy = TLSPolicy(domain_suffix) - try: - new_domain_policy.from_json_dict(settings) - except ConfigError as e: - raise - tls_policy_dict[domain_suffix] = new_domain_policy - - def get_tls_policy(self, mx_domain): - return self.tls_policies.get(mx_domain) - - def make_acceptable_mxs_dict(self, mxs_dict): - acceptable_mxs_dict = self._data['acceptable-mxs'] - for domain, settings in mxs_dict.iteritems(): - new_domain_policy = AcceptableMX(domain) - try: - new_domain_policy.from_json_dict(settings) - except ConfigError as e: - raise - acceptable_mxs_dict[domain] = new_domain_policy - - def get_address_domains(self, mx_hostname, mx_to_domain_map): - """Do a fuzzy DNS host match on provided map to get lists of policies. - - Args: - mx_hostname (string): The hostname from an MX record. - mx_to_domain_map: Mapping from MX hosts to AcceptableMX - policies, the same AcceptableMX policy may occur more - than once. e.g. {'mx_host3': set(AcceptableMX, ...)} - The map can be generated by Config.get_mx_to_domain_policy_map. - - Returns: - The set containing all AcceptableMX policies that list the - provided MX host as viable. - """ - labels = mx_hostname.split(".") - for n in range(1, len(labels)): - parent = "." + ".".join(labels[n:]) - if parent in mx_to_domain_map: - return mx_to_domain_map[parent] - return None - - def get_mx_to_domain_policy_map(self): - """Create mapping of MX hostnames to sets of AcceptableMX policies. - - Generate a dictionary that is typically used in log analysis - (e.g. if your MTA logs interact with beta.innotech.com you use - this mapping to tell you it used the innotech.com AcceptableMX - policy or policies). There are of course complications. - """ - # create reverse mapping dictionary as well for auditing - # and reviewing logs - mx_to_domain_policy = collections.defaultdict(set) - - for mx_host, domain_policy in self.get_all_mx_items(): - existing_mx_policies = mx_to_domain_policy.get(mx_host) - if existing_mx_policies: - existing_domains = [ e.domain for e in existing_mx_policies ] - if domain_policy.domain not in existing_domains: - #TODO plenty of room to enforce a security policy here - # this is also the case of google apps personal domains - msg = ('Attempting to add domain policy (%s) for MX host but MX' - ' host already has a domain policy (%s), appending...') - logger.debug(msg % (domain_policy.domain, - ', '.join(existing_domains))) - mx_to_domain_policy[mx_host].add(domain_policy) - return mx_to_domain_policy - - def get_all_mx_items(self): - """Iterate over (mx_host, mx_policy) - be sure to dedup!""" - all_mx_items = [] - for policy in self.acceptable_mxs.values(): - accepted_mxs = policy.accept_mx_domains - all_mx_items.extend([(mx_host, policy) - for mx_host in accepted_mxs]) - return all_mx_items - - def get_all_mx_hosts(self): - all_mx_hosts = [] - [ all_mx_hosts.extend(domain_policy.acceptable_mxs) - for domain_policy in self.acceptable_mxs.values() ] - return all_mx_hosts - - def is_valid(self): - #TODO implement checks to make sure domains don't overlap - #TODO add debug logging for troubleshooting sake - for mx_config in self.acceptable_mxs.values(): - if not mx_config.is_valid(): - return False - for domain_suffix in mx_config.accept_mx_domains: - # check to make sure every accepted MX has a TLS policy - if not domain_suffix in self.tls_policies: - return False - all_mx_hosts = self.get_all_mx_hosts() - for domain_suffix, tls_config in self.tls_policies.iteritems(): - if not tls_config.is_valid(): - return False - # make sure no unclaimed TLS policies have made their way in - if domain_suffix not in all_mx_hosts: - return False - return True - - -class TLSPolicy(BaseConfig): - - ENFORCE_MODES = ('enforce', 'log-only') - TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3') - - config_properties = ['comment', 'enforce_mode', 'min_tls_version', - 'require_tls', 'require_valid_certificate'] - - def __init__(self, domain_suffix=None): - super(self.__class__, self).__init__() - self.domain_suffix = domain_suffix - #TODO add support for two designed but yet unsupported attrs - # self._data['accept-spki-hashs'] = None - # self._data['error-notification'] = None - - def from_json_dict(self, json_dict): - for key, val in json_dict.iteritems(): - if key == 'comment': - self.comment = val - elif key == 'enforce-mode': - self.enforce_mode = val - elif key == 'min-tls-version': - self.min_tls_version = val - elif key == 'require-tls': - self.require_tls = val - elif key == 'require-valid-certificate': - self.require_valid_certificate = val - else: - logger.warn('Unknown key %s' % key) - - def is_valid(self): - """Do simple check that config contains all required values. - - Should find a way to expose easily which config values - are required, at least place in error messages such that - incomplete configs will expose it. - """ - required_attrs = ('enforce-mode', 'min-tls-version', - 'require-tls') - values_set = [self._data.get(attr) for attr in required_attrs] - if not all(values_set): - return False - else: - return True - - def update(self, newer_policy, **kwargs): - if not kwargs.get('domain_suffix'): - kwargs['domain_suffix'] = self.domain_suffix - fresh_policy = super(self.__class__, self).update(newer_policy, - **kwargs) - logger.debug('from TLS child update %s' % kwargs) - return fresh_policy - - def merge(self, newer_policy, **kwargs): - logger.debug('from TLS child merge: %s' % kwargs) - fresh_policy = super(self.__class__, self).merge(newer_policy, - domain_suffix=self.domain_suffix) - return fresh_policy - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - @property - def enforce_mode(self): - return self._data.get('enforce-mode') - - @enforce_mode.setter - def enforce_mode(self, value): - self._data['enforce-mode'] = verify_member_of(value, self.ENFORCE_MODES, 'enforce-mode') - - @property - def min_tls_version(self): - return self._data.get('min-tls-version') - - @min_tls_version.setter - def min_tls_version(self, value): - """TODO: Should this be dealing only with strings processed by map ... lower()?""" - tls_versions = [ver.lower() for ver in self.TLS_VERSIONS] - tls_versions.extend(self.TLS_VERSIONS) - self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version') - - @property - def require_tls(self): - return self._data.get('require-tls') - - @require_tls.setter - def require_tls(self, value): - self._data['require-tls'] = parse_bool_from_json(value, 'require-tls') - - @property - def require_valid_certificate(self): - return self._data.get('require-valid-certificate') - - @require_valid_certificate.setter - def require_valid_certificate(self, value): - self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate') - - -class AcceptableMX(BaseConfig): - """Holds acceptable MX domain suffixes for a single mail serving domain. - - Such as for gmail.com that single mail serving suffix domain is: - gmail-smtp-in.l.google.com. - - Configuration of the acceptable MX suffix domains must match up with TLS policies - for the suffix domains. - """ - def __init__(self, domain=None): - super(self.__class__, self).__init__() - self.domain = domain - self._data['accept-mx-domains'] = [] - - @property - def accept_mx_domains(self): - return self._data.get('accept-mx-domains') - - def add_acceptable_mx(self, domain_suffix): - unique_domain_suffixes = set(self._data['accept-mx-domains']) - unique_domain_suffixes.add(domain_suffix) - self._data['accept-mx-domains'] = list(unique_domain_suffixes) - - @property - def comment(self): - return self._data.get('comment') - - @comment.setter - def comment(self, value): - self._data['comment'] = verify_string(value, 'comment') - - def is_valid(self): - """Check to make sure there is one acceptable domain suffix. - - This will need to be updated once we can actually test and support - for more than one acceptable domain suffix. - - TODO: could make this object double check the data it is given with - DNS queries. - """ - if len(self._data['accept-mx-domains']) != 1: - return False - else: - return True - - def from_json_dict(self, json_dict): - for key, val in json_dict.iteritems(): - if key == 'accept-mx-domains': - if isinstance(val, list): - for domain_suffix in val: - self.add_acceptable_mx(domain_suffix) - else: - self.add_acceptable_mx(val) - elif key == 'comment': - self.comment = val - else: - logger.warn('warning: unknown key %s' % key) - - def update(self, newer_policy, **kwargs): - logger.debug('from MX child update got %s' % kwargs) - if not kwargs.get('domain'): - kwargs['domain'] = self.domain - fresh_policy = super(self.__class__, self).update(newer_policy, - **kwargs) - if kwargs.get('merge'): - new_accepted_mxs = set(self.accept_mx_domains) - new_accepted_mxs = new_accepted_mxs.union(newer_policy.accept_mx_domains) - else: - new_accepted_mxs = newer_policy.accept_mx_domains - for domain in new_accepted_mxs: - fresh_policy.add_acceptable_mx(domain) - - return fresh_policy - - def merge(self, newer_policy, **kwargs): - logger.debug('from MX child merge: %s' % kwargs) - fresh_policy = super(self.__class__, self).merge(newer_policy, - **kwargs) - return fresh_policy - - -class ConfigError(ValueError): - def __init__(self, message): - super(self.__class__, self).__init__(message) diff --git a/letsencrypt-postfix/PostfixLogSummary.py b/letsencrypt-postfix/PostfixLogSummary.py deleted file mode 100755 index a641a9f31..000000000 --- a/letsencrypt-postfix/PostfixLogSummary.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -import argparse -import collections -import os -import re -import sys -import time - -import Config - -TIME_FORMAT = "%b %d %H:%M:%S" - -# TODO: There's more to be learned from postfix logs! Here's one sample -# observed during failures from the sender vagrant vm: - -# Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] -# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1] -# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1] -# Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix -# Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from= -# Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com> -# Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=, size=576, nrcpt=1 (queue active) -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: -# Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) -# -# Also: -# Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) -def get_counts(input, config, earliest_timestamp): - seen_trusted = False - - counts = collections.defaultdict(lambda: collections.defaultdict(int)) - tls_deferred = collections.defaultdict(int) - # Typical line looks like: - # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) - # indicate a problem that should be alerted on. - # ([^[]*) <--- any group of characters that is not "[" - # Log lines for when a message is deferred for a TLS-related reason. These - deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") - # Log lines for when a TLS connection was successfully established. These can - # indicate the difference between Untrusted, Trusted, and Verified certs. - connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") - mx_to_domain_mapping = config.get_mx_to_domain_policy_map() - - timestamp = 0 - for line in sys.stdin: - timestamp = time.strptime(line[0:15], TIME_FORMAT) - if timestamp < earliest_timestamp: - continue - deferred = deferred_re.search(line) - connected = connected_re.search(line) - if connected: - validation = connected.group(1) - mx_hostname = connected.group(2).lower() - if validation == "Trusted" or validation == "Verified": - seen_trusted = True - address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) - if address_domains: - domains_str = [ a.domain for a in address_domains ] - d = ', '.join(domains_str) - counts[d][validation] += 1 - counts[d]["all"] += 1 - elif deferred: - mx_hostname = deferred.group(1).lower() - tls_deferred[mx_hostname] += 1 - return (counts, tls_deferred, seen_trusted, timestamp) - -def print_summary(counts): - for mx_hostname, validations in counts.items(): - for validation, validation_count in validations.items(): - if validation == "all": - continue - print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(description='Detect delivery problems' - ' in Postfix log files that may be caused by security policies') - arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) - arg_parser.add_argument("policy_file", nargs='?', - default=os.path.join("examples", "starttls-everywhere.json"), - help="STARTTLS Everywhere policy file") - - args = arg_parser.parse_args() - config = Config.Config() - config.load_from_json_file(args.policy_file) - - last_timestamp_processed = 0 - timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' - if os.path.isfile(timestamp_file): - last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) - (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) - with open(timestamp_file, "w") as f: - f.write(time.strftime(TIME_FORMAT, latest_timestamp)) - - # If not running in cron, print an overall summary of log lines seen from known hosts. - if not args.cron: - print_summary(counts) - if not seen_trusted: - print 'No Trusted connections seen! Probably need to install a CAfile.' - - if len(tls_deferred) > 0: - print "Some mail was deferred due to TLS problems:" - for (k, v) in tls_deferred.iteritems(): - print "%s: %s" % (k, v) diff --git a/letsencrypt-postfix/TestConfig.py b/letsencrypt-postfix/TestConfig.py deleted file mode 100755 index ca8e77654..000000000 --- a/letsencrypt-postfix/TestConfig.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -import copy -import itertools -import logging -import unittest - -import Config - -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) - - -class TestTLSPolicy(unittest.TestCase): - - def setUp(self): - self.old_config = Config.TLSPolicy(domain_suffix='.eff.org') - self.old_config.comment = 'Testing EFF.org TLS policy' - self.old_config.require_tls = True - self.old_config.require_valid_certificate = False - self.old_config.min_tls_version = 'TLSv1' - self.old_config.enforce_mode = 'log-only' - - self.new_config = Config.TLSPolicy(domain_suffix='.eff.org') - self.new_config.require_valid_certificate = True - self.new_config.min_tls_version = 'TLSv1.2' - self.new_config.enforce_mode = 'enforce' - - def testUpdateDropsOldSettings(self): - logger.debug('old: %s' % self.old_config) - logger.debug('new: %s' % self.new_config) - tls_policy = self.old_config.update(self.new_config) - logger.debug('just generated: %s' % tls_policy) - self.assertFalse(any([tls_policy.require_tls, tls_policy.comment])) - - def testMergeKeepsOldSettings(self): - logger.debug('old: %s' % self.old_config) - logger.debug('new: %s' % self.new_config) - tls_policy = self.old_config.merge(self.new_config, merge=True) - logger.debug('just generated: %s' % tls_policy) - self.assertTrue(all([tls_policy.require_tls, tls_policy.comment])) - - def testUpdateGetsNameSet(self): - tls_policy = self.old_config.update(self.new_config) - self.assertEquals(tls_policy.domain_suffix, self.old_config.domain_suffix) - - -class TestAcceptableMX(unittest.TestCase): - - def setUp(self): - self.old_config = Config.AcceptableMX(domain='eff.org') - self.old_config.add_acceptable_mx('.eff.org') - - def testUpdateDropsOldMXs(self): - new_bogus_mx = '.testing.eff.org' - new_config = Config.AcceptableMX(domain='eff.org') - new_config.add_acceptable_mx(new_bogus_mx) - updated_config = self.old_config.update(new_config) - self.assertNotIn('.eff.org', updated_config.accept_mx_domains) - - def testMergeKeepsOldMXs(self): - new_bogus_mx = '.testing.eff.org' - new_config = Config.AcceptableMX(domain='eff.org') - new_config.add_acceptable_mx(new_bogus_mx) - updated_config = self.old_config.merge(new_config) - self.assertListEqual(sorted(['.eff.org', '.testing.eff.org']), - sorted(updated_config.accept_mx_domains)) - - def testUpdateGetsNameSet(self): - new_policy = Config.AcceptableMX(domain=self.old_config.domain) - mx_policy = self.old_config.update(new_policy) - self.assertEquals(mx_policy.domain, self.old_config.domain) - - -class TestConfig(unittest.TestCase): - """Test entire configuration. - - Currently lower coverage is being obtained since string sets are - being compared rather than returned objects. Comparison logic for - the config objects isn't clear yet and proof that they function is enough. - """ - - def setUp(self): - self.config = Config.Config() - domain_policies = self.config._data['acceptable-mxs'] - self.mail_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', '123.cn', 'qq.com'] - for domain in self.mail_domains: - new = Config.AcceptableMX(domain=domain) - new.add_acceptable_mx('.' + domain) - domain_policies[domain] = new - - def testGetAllMxItems(self): - """Make sure the basic use case of get_all_mx_items functions.""" - # [ ('.gmail.com', 'gmail.com'), ('.yahoo.com', 'yahoo.com'), ... ] - control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] - test_data = [ (mx, p.domain) for mx, p in self.config.get_all_mx_items() ] - self.assertListEqual(sorted(test_data), sorted(control_data)) - - def testGetAllMxItemsMultiMX(self): - config = copy.deepcopy(self.config) - domain_policy = config.acceptable_mxs.get('gmail.com') - # deal with reality, mail.google.com - domain_policy.add_acceptable_mx('.mail.google.com') - control_data = [ ('.' + domain, domain) for domain in self.mail_domains ] - control_data.append(('.mail.google.com', 'gmail.com')) - test_data = [ (mx, p.domain) for mx, p in config.get_all_mx_items() ] - self.assertListEqual(sorted(test_data), sorted(control_data)) - - def testGetMXtoDomainPolicy(self): - control_data = dict([ ('.' + domain, set([domain])) - for domain in self.mail_domains ]) - test_data = {} - for mx, pset in self.config.get_mx_to_domain_policy_map().items(): - policy_list = [ p.domain for p in pset ] - test_data[mx] = set(policy_list) - self.assertDictEqual(test_data, control_data) - - def testGetMXtoDomainPolicyMultiMX(self): - config = copy.deepcopy(self.config) - domain_policy = config.acceptable_mxs.get('gmail.com') - domain_policy.add_acceptable_mx('.mail.google.com') - control_data = dict([ ('.' + domain, set([domain])) - for domain in self.mail_domains ]) - control_data['.mail.google.com'] = set(['gmail.com']) - test_data = {} - for mx, pset in config.get_mx_to_domain_policy_map().items(): - policy_list = [ p.domain for p in pset ] - test_data[mx] = set(policy_list) - self.assertDictEqual(test_data, control_data) - - -if __name__ == '__main__': - unittest.main() diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5334ba03a..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -dnspython -publicsuffix -m2crypto -dateutils diff --git a/share/golden-domains.txt b/share/golden-domains.txt deleted file mode 100644 index bca6d2f89..000000000 --- a/share/golden-domains.txt +++ /dev/null @@ -1,32 +0,0 @@ -163.com -aol.com -bigpond.com -comcast.net -craigslist.org -facebook.com -gmail.com -gmx.de -hotmail.com -icloud.com -live.com -mac.com -me.com -msn.com -naver.com -outlook.com -qq.com -rocketmail.com -rogers.com -salesforce.com -sbcglobal.net -shaw.ca -sympatico.ca -t-online.de -ukr.net -vtext.com -web.de -wp.pl -yahoo.com -yahoogroups.com -yandex.ru -ymail.com diff --git a/share/google-starttls-domains.csv b/share/google-starttls-domains.csv deleted file mode 100644 index 5d6a00f17..000000000 --- a/share/google-starttls-domains.csv +++ /dev/null @@ -1,6734 +0,0 @@ -Address Suffix,Hostname Suffix,Direction,UN M.49 Region Code,Region Name,Fraction Encrypted -0101.co.jp,0101.co.jp,inbound,001,World,0 -04auto.biz,01auto.biz,inbound,001,World,0 -0bz.biz,hmts.jp,inbound,001,World,0 -1-day.co.nz,1-day.co.nz,inbound,001,World,0 -104.com.tw,104.com.tw,inbound,001,World,0.091112 -1105info.com,1105info.com,inbound,001,World,0 -1111.com.tw,1111.com.tw,inbound,001,World,0 -123.com.tw,123.com.tw,inbound,001,World,0 -12manage.com,netarrest.com,inbound,001,World,0.99998 -160by2.us,160by2.us,inbound,001,World,0 -160by2inbox.com,160by2inbox.com,inbound,001,World,0 -160by2invite.com,160by2invite.com,inbound,001,World,0 -160by2mail.com,160by2mail.com,inbound,001,World,0 -163.com,163.com,inbound,001,World,0.711306 -163.com,netease.com,outbound,001,World,1 -17life.com.tw,17life.com.tw,inbound,001,World,0 -1800flowersinc.com,1800flowersinc.com,inbound,001,World,0 -1800petmeds.com,1800petmeds.com,inbound,001,World,0.00599 -1lejend.com,asumeru.com,inbound,001,World,0 -1lejend.com,asumeru001.com,inbound,001,World,0 -1sale.com,1sale.com,inbound,001,World,0 -1v1y.com,euromsg.net,inbound,001,World,0 -2touchbase.com,infimail.com,inbound,001,World,0 -33go.com.tw,33go.com.tw,inbound,001,World,0 -3suisses.be,3suisses.be,inbound,001,World,0 -3suisses.fr,3suisses.fr,inbound,001,World,0 -4shared.com,4shared.com,inbound,001,World,0.999946 -4wheelparts.com,4wheelparts.com,inbound,001,World,0 -518.com.tw,518.com.tw,inbound,001,World,0 -6pm.com,6pm.com,inbound,001,World,1 -6pm.com,zappos.com,inbound,001,World,0.782581 -7net.com.tw,7net.com.tw,inbound,001,World,0 -99acres.com,99acres.com,inbound,001,World,0 -9dot9digital.in,emce2.in,inbound,001,World,0 -a8.net,a8.net,inbound,001,World,0 -aaa.com,nextjump.com,inbound,001,World,0 -aaas-science.org,aaas-science.org,inbound,001,World,0 -aafes.com,aafes.com,inbound,001,World,0 -aanotifier.nl,aanotifier.nl,inbound,001,World,0 -aarp.org,aarp.org,inbound,001,World,0.006249 -ab0.jp,altovision.co.jp,inbound,001,World,0 -abercrombie-email.com,abercrombie-email.com,inbound,001,World,0 -abercrombiekids-email.com,abercrombie-email.com,inbound,001,World,0 -about.com,about.com,inbound,001,World,2.4e-05 -about.com,sailthru.com,inbound,001,World,0 -academy-enews.com,academy-enews.com,inbound,001,World,0 -accenture.com,outlook.com,inbound,001,World,1 -accountonline.com,accountonline.com,inbound,001,World,0.348991 -acehelpfulemails.com,teradatadmc.com,inbound,001,World,0 -acemserv.com,acemserv.com,inbound,001,World,0 -activesafelist.com,zoothost.com,inbound,001,World,0 -activetrail.com,atmailsvr.net,inbound,001,World,0 -activetrail.com,mymarketing.co.il,inbound,001,World,0 -actorsaccess.com,nonfatmedia.com,inbound,001,World,0 -adchiever.com,kinder-rash-marketing.com,inbound,001,World,0 -adidas.com,neolane.net,inbound,001,World,0 -adidasusnews.com,adidasusnews.com,inbound,001,World,0 -adityabirla.com,adityabirla.com,inbound,001,World,0.006468 -adjockeys.com,thomas-j-brown.com,inbound,001,World,0 -admail.hu,sanomaonline.hu,inbound,001,World,0 -admastersafelist.com,zoothost.com,inbound,001,World,0 -adminforfree.com,adminforfree.com,inbound,001,World,1 -adminforfree.net,adminforfree.com,inbound,001,World,1 -administrativejobinsider.com,administrativejobinsider.com,inbound,001,World,0 -adobe.com,obsmtp.com,inbound,001,World,0.999986 -adobesystems.com,adobesystems.com,inbound,001,World,0 -adorama.com,adorama.com,inbound,001,World,0 -adoreme.com,exacttarget.com,inbound,001,World,0 -adp.com,adp.com,inbound,001,World,1 -adpirate.net,thomas-j-brown.com,inbound,001,World,0 -adsender.us,adsender.us,inbound,001,World,0 -adsolutionline.com,adsolutionline.com,inbound,001,World,0 -adtpulse.com,adtpulse.com,inbound,001,World,0 -adultfriendfinder.com,friendfinder.com,inbound,001,World,0 -advanceauto.com,bigfootinteractive.com,inbound,001,World,0 -advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,001,World,0 -adverts.ie,adverts.ie,inbound,001,World,1 -advfn.com,advfn.com,inbound,001,World,0.635382 -ae.com,ae.com,inbound,001,World,0 -aexp.com,aexp.com,inbound,001,World,1 -af.mil,af.mil,inbound,001,World,0.996166 -affairalert.com,iverificationsystems.com,inbound,001,World,0 -agnitas.de,agnitas.de,inbound,001,World,0.999277 -agoda-emails.com,agoda-emails.com,inbound,001,World,0 -agora.co.il,1host.co.il,inbound,001,World,0 -agorafinancial.com,agorafinancial.com,inbound,001,World,0 -agrupemonos.cl,agrupemonos.cl,inbound,001,World,1 -airbnb.com,airbnb.com,inbound,001,World,0.867975 -airbrake.io,mailgun.net,inbound,001,World,1 -airfarewatchdog.com,smartertravelmedia.com,inbound,001,World,0.011961 -airliquide.com,airliquide.com,inbound,001,World,1 -airmiles.ca,bigfootinteractive.com,inbound,001,World,0 -airtel.com,airtel.in,inbound,001,World,0.064377 -akcijatau.lt,akcijatau.lt,inbound,001,World,0 -alarm.com,alarm.com,inbound,001,World,0 -alarmnet.com,alarmnet.com,inbound,001,World,0 -alaskaair.com,alaskaair.com,inbound,001,World,0 -albertsonsemail.com,email4-mywebgrocer.com,inbound,001,World,0 -alerteimmo.com,alerteimmo.com,inbound,001,World,0 -alertid.com,alertid.com,inbound,001,World,0 -alertsindia.in,alertsindia.in,inbound,001,World,1 -alibaba.com,alibaba.com,inbound,001,World,0 -alice.it,alice.it,inbound,001,World,0 -alice.it,aliceposta.it,outbound,001,World,0 -aliexpress.com,alibaba.com,inbound,001,World,0 -alinea.fr,bp06.net,inbound,001,World,0 -alipay.com,alipay.com,inbound,001,World,1 -allegro.pl,allegro.pl,inbound,001,World,0 -allegrogroup.ua,allegrogroup.ua,inbound,001,World,0 -allegroup.hu,allegroup.hu,inbound,001,World,0 -allheart.com,allheart.com,inbound,001,World,0 -alljob.co.il,alljob.co.il,inbound,001,World,0 -allmodern.com,allmodern.com,inbound,001,World,0 -allout.org,allout.org,inbound,001,World,1 -allrecipes.com,allrecipes.com,inbound,001,World,0 -allsaints.com,allsaints.com,inbound,001,World,0 -allstate.com,rsys1.com,inbound,001,World,0 -alm.com,sailthru.com,inbound,001,World,0 -alumniclass.com,alumniclass.com,inbound,001,World,0 -alumniconnections.com,alumniconnections.com,inbound,001,World,0 -alza.cz,alza.cz,inbound,001,World,0.039449 -alza.sk,alza.cz,inbound,001,World,0.027725 -ama-assn.org,elabs10.com,inbound,001,World,0 -amadeus.com,amadeus.net,inbound,001,World,0 -amazon.{...},amazon.{...},inbound,001,World,0.020886 -amazon.{...},amazonses.com,inbound,001,World,0.999971 -amazon.{...},postini.com,inbound,001,World,0.7325 -amazon.{...},yahoo.{...},inbound,001,World,0.995995 -amazonses.com,amazonses.com,inbound,001,World,0.997919 -amazonses.com,postini.com,inbound,001,World,0.843221 -amctheatres.com,amctheatres.com,inbound,001,World,0 -americanas.com,americanas.com,inbound,001,World,0 -americanbar.org,abanet.org,inbound,001,World,0.00574 -americanexpress.com,americanexpress.com,inbound,001,World,0.000688 -americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,001,World,0 -amubm.com,amubm.com,inbound,001,World,1 -amwayemail.com,mailrouter.net,inbound,001,World,1 -ana.co.jp,ana.co.jp,inbound,001,World,0.534416 -ancestry.com,ancestry.com,inbound,001,World,0 -andrewchristian.com,emv8.com,inbound,001,World,0 -angelbroking.in,infimail.com,inbound,001,World,0 -anghami.com,mailgun.net,inbound,001,World,1 -angieslist.com,angieslist.com,inbound,001,World,0.014064 -anntaylor.com,anntaylor.com,inbound,001,World,0 -anpasia.com,anpasia.com,inbound,001,World,0 -anpdm.com,anpdm.com,inbound,001,World,6e-06 -anthropologie.com,freepeople.com,inbound,001,World,0 -aol.com,aol.com,inbound,001,World,0.999529 -aol.com,aol.com,outbound,001,World,0.999992 -aol.com,sailthru.com,inbound,001,World,0 -aol.net,aol.com,inbound,001,World,1 -apache.org,apache.org,inbound,001,World,0 -apnacomplex.com,apnacomplex.com,inbound,001,World,0.999981 -apple.com,apple.com,inbound,001,World,0.974422 -apply-4-jobs.com,apply-4-jobs.com,inbound,001,World,0 -aprovaconcursos.com.br,eadunicid.com.br,inbound,001,World,0 -aptmail.in,mailurja.com,inbound,001,World,0 -ara.cat,ara.cat,inbound,001,World,0.997709 -arcamax.com,arcamax.com,inbound,001,World,1.4e-05 -argos.co.uk,argos.co.uk,inbound,001,World,1 -argos.co.uk,exacttarget.com,inbound,001,World,1 -aritzia.com,aritzia.com,inbound,001,World,0 -armaniexchange.com,bronto.com,inbound,001,World,0 -artists-hub.com,artists-hub.com,inbound,001,World,0 -artscow.com,dyxnet.com,inbound,001,World,0.00062 -aruba.it,aruba.it,inbound,001,World,0.055666 -asadventure.com,asadventure.com,inbound,001,World,0 -asana.com,asana.com,inbound,001,World,1 -asda.com,ec-cluster.com,inbound,001,World,0 -ashampoo.com,ashampoo.com,inbound,001,World,1 -ashleymadison.com,ashleymadison.com,inbound,001,World,1 -ask.fm,ask.fm,inbound,001,World,1e-05 -askmen.com,askmen.com,inbound,001,World,0 -asos.com,asos.com,inbound,001,World,0 -assembla.com,assembla.com,inbound,001,World,1 -astrocenter.com,center.com,inbound,001,World,0 -astrology.com,astrology.com,inbound,001,World,0 -astrology.com,hsnlmailsvc.com,inbound,001,World,0 -astrology.com,webstakes.com,inbound,001,World,0 -astrology.com,wsafmailsvc.com,inbound,001,World,0 -asus.com,asus.com,inbound,001,World,0 -aswatson.com,emarsys.net,inbound,001,World,0 -athleta.com,athleta.com,inbound,001,World,0 -atlassian.net,uc-inf.net,inbound,001,World,1 -atrapalo.cl,atrapalo.com,inbound,001,World,0 -atrapalo.com,atrapalo.com,inbound,001,World,0 -att-mail.com,att-mail.com,inbound,001,World,2.2e-05 -att-mail.com,att.com,inbound,001,World,0.999966 -att.net,att.net,outbound,001,World,0.204629 -att.net,mycingular.net,inbound,001,World,0.00021 -att.net,yahoo.{...},inbound,001,World,0.99997 -auctionzip-email.com,email-auctionholdings.com,inbound,001,World,0 -auinmeio.com.br,fnac.com.br,inbound,001,World,0 -australiagsm.net,australiagsm.net,inbound,001,World,0 -authorize.net,authorize.net,inbound,001,World,0 -authorize.net,visa.com,inbound,001,World,0.993558 -autoloop.us,loop28.com,inbound,001,World,0 -autoreply.com,autoreply.com,inbound,001,World,0 -avaaz.org,avaaz.org,inbound,001,World,0 -avalanchesafelist.com,zoothost.com,inbound,001,World,0 -avast.com,avast.com,inbound,001,World,0.007867 -aveda.com,esteelauder.com,inbound,001,World,0 -avenue.com,avenue.com,inbound,001,World,0 -avg.com,avg.com,inbound,001,World,0.015703 -avira.com,avira.com,inbound,001,World,0.042528 -avito.ru,avito.ru,inbound,001,World,0.002677 -avomail.com,avomail.com,inbound,001,World,0 -avon.com,email-avonglobal.com,inbound,001,World,0 -avon.com,postdirect.com,inbound,001,World,0 -aweber.com,aweber.com,inbound,001,World,3e-06 -ayi.com,ayi.com,inbound,001,World,0 -b2b-mail.net,b2b-mail.net,inbound,001,World,0 -b2b-mail.net,contact-list.net,inbound,001,World,0 -babycenter.com,rsys3.com,inbound,001,World,0 -babyoye.com,babyoye.com,inbound,001,World,0.011564 -backcountry.com,backcountry.com,inbound,001,World,0 -backlog.jp,backlog.jp,inbound,001,World,0 -badoo.com,monopost.com,inbound,001,World,1 -bagitgetitmailer.in,emce2.in,inbound,001,World,0 -baligam.co.il,baligam.co.il,inbound,001,World,1 -balsamik.fr,balsamik.fr,inbound,001,World,0 -banamex.com,citi.com,inbound,001,World,0.999958 -banamex.com,ibrands.es,inbound,001,World,0 -bananarepublic.com,bananarepublic.com,inbound,001,World,3.48869418874254e-07 -bancoahorrofamsa.com,avantel.net.mx,inbound,001,World,1 -bancochile.cl,bancochile.cl,inbound,001,World,0.999504 -bancofalabella.com,bancofalabella.com,inbound,001,World,0 -bancomer.com,postini.com,inbound,001,World,5.9e-05 -bancomercorreo.com,bancomercorreo.com,inbound,001,World,0 -bandsintown.com,bandsintown.com,inbound,001,World,1 -banesco.com,banesco.com,inbound,001,World,0 -bankofamerica.com,bankofamerica.com,inbound,001,World,0.97133 -banorte.com,gfnorte.com.mx,inbound,001,World,0.994999 -barclaycard.co.uk,barclays.co.uk,inbound,001,World,0 -barclaycardus.com,bigfootinteractive.com,inbound,001,World,0 -barenecessities.com,barenecessities.com,inbound,001,World,0 -barleyment.ca,barleyment.ca,inbound,001,World,0 -barneys.com,barneys.com,inbound,001,World,0 -baseballsavings.com,baseballsavings.com,inbound,001,World,0 -basecamp.com,basecamp.com,inbound,001,World,1 -basecamphq.com,basecamphq.com,inbound,001,World,1 -baskinrobbins.com,baskinrobbins.com,inbound,001,World,0 -basspronews.com,basspronews.com,inbound,001,World,0 -bathandbodyworks.com,bathandbodyworks.com,inbound,001,World,0 -baublebar.com,baublebar.com,inbound,001,World,0.011219 -baycrews.co.jp,webcas.net,inbound,001,World,0 -bayt.com,bayt.com,inbound,001,World,2e-06 -bazarchic-invitations.com,bazarchic-emstech.com,inbound,001,World,0 -bbvacompass.com,postini.com,inbound,001,World,0.996402 -bcbg.com,bcbg.com,inbound,001,World,0 -bci.cl,bci.cl,inbound,001,World,0.999963 -bcp.com.pe,bcp.com.pe,inbound,001,World,1 -be2.com,nmp1.net,inbound,001,World,0 -beamtele.com,beamtele.com,inbound,001,World,0 -beanfun.com,beanfun.com,inbound,001,World,1 -beatport-email.com,beatport-email.com,inbound,001,World,0 -beautylish.com,beautylish.com,inbound,001,World,1 -bebe.com,ed10.com,inbound,001,World,0 -befrugal.com,befrugal.com,inbound,001,World,0.050462 -belkemail.com,belkemail.com,inbound,001,World,0 -bellsouth.net,att.net,outbound,001,World,0 -bellsouth.net,yahoo.{...},inbound,001,World,0.999967 -belluna.net,belluna.net,inbound,001,World,0 -benihana-news.com,benihana-news.com,inbound,001,World,0 -bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,001,World,0 -bespokeoffers.co.uk,chtah.net,inbound,001,World,0 -bestbuy.ca,bestbuy.ca,inbound,001,World,0 -bestbuy.com,bestbuy.com,inbound,001,World,0.003289 -bestdealsforyou.in,elabs5.com,inbound,001,World,0 -beta.lt,mailersend3.com,inbound,001,World,0 -betrend.com,betrend.com,inbound,001,World,0 -bevmo.com,bevmo.com,inbound,001,World,0.000967 -beyondtherack.com,beyondtherack.com,inbound,001,World,0 -bharatmatrimony.com,bharatmatrimony.com,inbound,001,World,1 -bhcosmetics.com,bronto.com,inbound,001,World,0 -bhg.com,meredith.com,inbound,001,World,0 -bigfishgames.com,bigfishgames.com,inbound,001,World,0 -biglion.ru,biglion.ru,inbound,001,World,0.999775 -biglist.com,biglist.com,inbound,001,World,0 -biglots.com,biglots.com,inbound,001,World,0.00029 -bigmailsender.com,bigmailsender.com,inbound,001,World,0 -bigpond.com,bigpond.com,inbound,001,World,0 -bigpond.com,bigpond.com,outbound,001,World,1 -bigtent.com,carezen.net,inbound,001,World,0 -bioagri.com.br,postini.com,inbound,001,World,0.991765 -biomedcentral.com,emv5.com,inbound,001,World,0 -bionexo.com,bionexo.com.br,inbound,001,World,0.999594 -birthdayalarm.com,monkeyinferno.net,inbound,001,World,0 -bitbucket.org,bitbucket.org,inbound,001,World,0 -bitlysupport.com,mailgun.info,inbound,001,World,1 -bitlysupport.com,mailgun.us,inbound,001,World,1 -bitslane.email,bitslane.email,inbound,001,World,0 -bitstatement.org,bitstatement.org,inbound,001,World,1 -bizjournals.com,bizjournals.com,inbound,001,World,0 -bizmailtoday.com,bizmailtoday.com,inbound,001,World,0 -bjs.com,bjs.com,inbound,001,World,0 -bjsrestaurants.com,bjsrestaurants.com,inbound,001,World,0 -bk.ru,mail.ru,inbound,001,World,0.992498 -blablacar.com,blablacar.com,inbound,001,World,1 -blackberry.com,blackberry.com,inbound,001,World,0 -blackboard.com,blackboard.com,inbound,001,World,0.998206 -blackboard.com,notification.com,inbound,001,World,0 -blackpeoplemeet.com,blackpeoplemeet.com,inbound,001,World,0 -blayn.jp,bserver.jp,inbound,001,World,0 -blinkboxmusic.com,mediagraft.com,inbound,001,World,1 -blissworld.com,lstrk.net,inbound,001,World,1 -blizzard.com,battle.net,inbound,001,World,0.11977 -bloglovin.com,bloglovin.com,inbound,001,World,0.000154 -blogtrottr.com,blogtrottr.com,inbound,001,World,0 -bloomberg.com,bloomberg.com,inbound,001,World,0.00501 -bloomberg.net,bloomberg.net,inbound,001,World,1 -bloomingdales.com,bloomingdales.com,inbound,001,World,0 -bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,001,World,0 -blue-compass.com,blue-compass.com,inbound,001,World,0 -bluediamondhost3.com,web-hosting.com,inbound,001,World,1 -bluehornet.com,bluehornet.com,inbound,001,World,0 -bluehost.com,bluehost.com,inbound,001,World,0.000943 -bluehost.com,hostmonster.com,inbound,001,World,0 -bluehost.com,unifiedlayer.com,inbound,001,World,1.6e-05 -bluenile.com,bluenile.com,inbound,001,World,0 -blueshellgames.com,blueshellgames.com,inbound,001,World,0 -bluestatedigital.com,bluestatedigital.com,inbound,001,World,0 -bluestonemx.com,bluestonemx.com,inbound,001,World,1 -bm05.net,bm05.net,inbound,001,World,0 -bm23.com,bronto.com,inbound,001,World,0 -bm324.com,bronto.com,inbound,001,World,0 -bmdeda99.com,bmdeda99.com,inbound,001,World,0 -bme.jp,bserver.jp,inbound,001,World,0 -bmnt.jp,bmnt.jp,inbound,001,World,0 -bmsend.com,bmsend.com,inbound,001,World,0 -bn.com,bn.com,inbound,001,World,0 -bncollegemail.com,bncollegemail.com,inbound,001,World,0 -bnetmail.com,bnetmail.com,inbound,001,World,0 -bol.com.br,bol.com.br,inbound,001,World,0 -bol.com.br,bol.com.br,outbound,001,World,0 -boletinrenuevo.com,boletinrenuevo.com,inbound,001,World,0 -bolsfr.fr,colt.net,inbound,001,World,0 -bomnegocio.com,bomnegocio.com,inbound,001,World,0.687113 -bonobos.com,bronto.com,inbound,001,World,0 -bonuszbrigad.hu,bonuszbrigad.hu,inbound,001,World,0 -boohooemail.com,smartfocusdigital.net,inbound,001,World,0 -bookbub.com,bookbub.com,inbound,001,World,1 -booking.com,booking.com,inbound,001,World,1 -bookingbuddy.com,smartertravelmedia.com,inbound,001,World,0.000486 -bookmyshow.com,eccluster.com,inbound,001,World,0 -bookoffonline.co.jp,bookoffonline.co.jp,inbound,001,World,0 -boomtownroi.com,boomtownroi.com,inbound,001,World,0 -boots.com,boots.com,inbound,001,World,0 -boscovs.com,boscovs.com,inbound,001,World,0 -bostonproper.com,bostonproper.com,inbound,001,World,0 -bouncemanager.it,musvc.com,inbound,001,World,0.362026 -boutiquesecret.com,chtah.net,inbound,001,World,0 -box.com,box.com,inbound,001,World,0.955607 -br.com,cmailsys.com,inbound,001,World,0 -bradfordexchange.com,bradfordexchange.com,inbound,001,World,0 -bradsdeals.com,bradsdeals.com,inbound,001,World,0 -brandalley.com,brandalley.com,inbound,001,World,0 -brands4friends.de,emv5.com,inbound,001,World,0 -brands4friends.jp,webcas.net,inbound,001,World,0 -brandsfever.com,mailgun.net,inbound,001,World,1 -brandsvillage.net,brandsvillage.net,inbound,001,World,0 -brassring.com,brassring.com,inbound,001,World,0.999989 -briantracyintl.com,briantracyintl.com,inbound,001,World,0 -brierleycrm.com,brierleycrm.com,inbound,001,World,0 -brijj.com,brijj.com,inbound,001,World,0 -brincltd.com,brincltd.com,inbound,001,World,0 -bronto.com,bronto.com,inbound,001,World,0 -brooksbrothers.com,brooksbrothers.com,inbound,001,World,0 -bsf01.com,bsftransmit33.com,inbound,001,World,0 -bt.com,bt.com,inbound,001,World,0.409404 -btinternet.com,cpcloud.co.uk,inbound,001,World,0 -btinternet.com,cpcloud.co.uk,outbound,001,World,0 -btinternet.com,yahoo.{...},inbound,001,World,0.99998 -budgettravel.com,email-budgettravel.com,inbound,001,World,0 -buffalo.edu,buffalo.edu,inbound,001,World,0.001059 -bumeran.com,bumeran.com,inbound,001,World,0 -burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,001,World,0 -burton.co.uk,burton.co.uk,inbound,001,World,0 -buscojobs.com,amazonaws.com,inbound,001,World,0 -buy123.com.tw,buy123.com.tw,inbound,001,World,1 -buyinvite.com.au,buyinvite.com.au,inbound,001,World,0 -buyma.com,buyma.com,inbound,001,World,0 -bv.com.br,bv.com.br,inbound,001,World,0 -bweeble.com,adlabsinc.com,inbound,001,World,0 -byway.it,byway.it,inbound,001,World,0 -bzm.mobi,nmsrv.com,inbound,001,World,1 -c21stores.com,c21stores.com,inbound,001,World,0 -ca.gov,ca.gov,inbound,001,World,0.694242 -cabelas.com,cabelas.com,inbound,001,World,0 -cabestan.com,cab07.net,inbound,001,World,0 -cadremploi.fr,cadremploi.fr,inbound,001,World,0 -cafepress.com,cafepress.com,inbound,001,World,0.000778 -caixa.gov.br,caixa.gov.br,inbound,001,World,0 -californiajobdepartment.com,californiajobdepartment.com,inbound,001,World,0 -californiapsychicsemail.com,californiapsychicsemail.com,inbound,001,World,0 -callcommand.com,callcommand.com,inbound,001,World,0 -calottery.com,calottery.com,inbound,001,World,0.999517 -cam2life.com,hinet.net,inbound,001,World,0 -camel.com,rjrsignup.com,inbound,001,World,0 -camsonline.com,camsonline.com,inbound,001,World,0.136261 -canadiantire.ca,canadiantire.ca,inbound,001,World,0 -canadianvisaexpert.net,canadianvisaexpert.net,inbound,001,World,0 -canalplus.es,canalplus.es,inbound,001,World,0 -cancer.org,delivery.net,inbound,001,World,0 -capillary.co.in,capillary.co.in,inbound,001,World,1 -capitalone.com,bigfootinteractive.com,inbound,001,World,0 -capitalone360.com,ingdirect.com,inbound,001,World,0 -capitaloneemail.com,capitaloneemail.com,inbound,001,World,0 -cardsys.at,cardsys.at,inbound,001,World,1 -care.com,care.com,inbound,001,World,0 -care2.com,care2.com,inbound,001,World,0 -career-hub.net,career-hub.net,inbound,001,World,1 -careerage.com,careerage.com,inbound,001,World,0 -careerbuilder-email.com,careerbuilder-email.com,inbound,001,World,0 -careerbuilder.com,careerbuilder.com,inbound,001,World,0.000449 -careerflash.net,careerflash.net,inbound,001,World,1 -careers24.com,careers24.com,inbound,001,World,0 -careesma.in,careesma.in,inbound,001,World,0 -carmamail.com,carmamail.com,inbound,001,World,0 -carnivalfunmail.com,carnivalfunmail.com,inbound,001,World,0 -carolsdaughter.com,carolsdaughter.com,inbound,001,World,0 -carrefour.fr,carrefour.fr,inbound,001,World,0 -carters.com,carters.com,inbound,001,World,0.003931 -cartrade.com,cartrade.com,inbound,001,World,0.006389 -carwale.com,carwale.com,inbound,001,World,0 -casasbahia.com.br,casasbahia.com.br,inbound,001,World,0 -case.edu,cwru.edu,inbound,001,World,0.999942 -caseyresearch.com,caseyresearch.com,inbound,001,World,1 -castingnetworks.com,castingnetworks.com,inbound,001,World,0.000102 -catchoftheday.com.au,inxserver.de,inbound,001,World,1 -catchyfreebies.net,mmsend53.com,inbound,001,World,0 -catererglobal.com,madgexjb.com,inbound,001,World,0 -caterermail.com,totaljobsmail.co.uk,inbound,001,World,0 -cathkidston.com,cathkidston.co.uk,inbound,001,World,0 -causes.com,causes.com,inbound,001,World,1 -cb2.com,cb2.com,inbound,001,World,0 -cbsig.net,cbsig.net,inbound,001,World,1 -ccavenue.com,avenues.info,inbound,001,World,1 -ccbchurch.com,ccbchurch.com,inbound,001,World,1 -cccampaigns.com,emv5.com,inbound,001,World,0 -cccampaigns.com,emv8.com,inbound,001,World,0 -cccampaigns.net,01net.com,inbound,001,World,0 -cccampaigns.net,cccampaigns.net,inbound,001,World,0 -cccampaigns.net,emv4.net,inbound,001,World,0 -cccampaigns.net,emv9.net,inbound,001,World,0 -ccialerts.com,ccialerts.com,inbound,001,World,0 -ccmbg.com,benchmark.fr,inbound,001,World,0 -ccs.com,footlocker.com,inbound,001,World,2.5e-05 -cdongroup.com,cdongroup.com,inbound,001,World,0.000147 -cecentertainment.com,cecentertainment.com,inbound,001,World,0 -celebritycruises.com,celebritycruises.com,inbound,001,World,0 -cenlat.com,cenlat.com,inbound,001,World,0.064459 -centaur.co.uk,centaur.co.uk,inbound,001,World,0 -centauro.com.br,centauro.com.br,inbound,001,World,0 -centerparcs.co.uk,ec-cluster.com,inbound,001,World,0 -cerberusapp.com,cerberusapp.com,inbound,001,World,1 -cfmailer.com,elabs11.com,inbound,001,World,0 -cfmvmail.com,cfmvmail.com,inbound,001,World,0 -chabad.org,chabad.org,inbound,001,World,0 -champssports.com,footlocker.com,inbound,001,World,0.000131 -chance.com,data-hotel.net,inbound,001,World,0 -change.org,change.org,inbound,001,World,1 -channel4.com,channel4.com,inbound,001,World,0.000613 -charlestyrwhitt.com,charlestyrwhitt.com,inbound,001,World,0 -charter.net,charter.net,inbound,001,World,0 -charter.net,charter.net,outbound,001,World,0 -chase.com,bigfootinteractive.com,inbound,001,World,0 -chase.com,jpmchase.com,inbound,001,World,0.999999 -chatcitynotifications.com,chatcitynotifications.com,inbound,001,World,0 -chaturbate.com,chaturbate.com,inbound,001,World,1 -cheapairmailer.com,cheapairmailer.com,inbound,001,World,0 -cheaperthandirt.com,cheaperthandirt.com,inbound,001,World,2.2e-05 -cheapflights.co.uk,cheapflights.co.uk,inbound,001,World,0 -cheapflights.com,cheapflights.com,inbound,001,World,0 -check.me,check.me,inbound,001,World,0 -cheekylovers.com,ropot.net,inbound,001,World,0 -chefscatalog.com,chefscatalog.com,inbound,001,World,0 -chelseafc.com,chelseafc.com,inbound,001,World,0.003566 -chemistdirect.co.uk,ec-cluster.com,inbound,001,World,0 -chemistry.com,chemistry.com,inbound,001,World,0 -chess.com,chess.com,inbound,001,World,1 -chiangcn.com,chiangcn.com,inbound,001,World,0 -chicagotribune.com,latimes.com,inbound,001,World,0 -chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,001,World,0 -chicos.com,chicos.com,inbound,001,World,0.000156 -childrensplace.com,childrensplace.com,inbound,001,World,0 -chinatrust.com.tw,chinatrust.com.tw,inbound,001,World,0.001272 -chopra.com,chopra.com,inbound,001,World,1 -christianbook.com,christianbook.com,inbound,001,World,1 -christianmingle.com,christianmingle.com,inbound,001,World,0 -christianmingle.com,postdirect.com,inbound,001,World,0 -chtah.com,chtah.net,inbound,001,World,0 -chtah.net,chtah.net,inbound,001,World,0 -cincghq.com,searchhomesingta.com,inbound,001,World,1 -cinesa.es,cccampaigns.com,inbound,001,World,0 -cipherzone.com,infimail.com,inbound,001,World,0 -cir.ca,cir.ca,inbound,001,World,1 -circleofmomsmail.com,circleofmomsmail.com,inbound,001,World,0 -citi.com,citi.com,inbound,001,World,0.999941 -citibank.com,bigfootinteractive.com,inbound,001,World,0 -citibank.com,citi.com,inbound,001,World,0.999997 -citicorp.com,citi.com,inbound,001,World,0.999999 -citruslane.com,citruslane.com,inbound,001,World,5e-06 -citybrands.hu,webinform.hu,inbound,001,World,1 -cityheaven.net,cityheaven.net,inbound,001,World,0 -ck.com,ck.com,inbound,001,World,0 -clarisonic.com,clarisonic.com,inbound,001,World,0 -clarks.com,clarks.com,inbound,001,World,0 -classmates.com,classmates.com,inbound,001,World,0 -clickdimensions.com,clickdimensions.com,inbound,001,World,0 -clickexperts.net,clickexperts.net,inbound,001,World,0 -clickmailer.jp,clickmailer.jp,inbound,001,World,9e-05 -clickon.com.ar,clickon.com.ar,inbound,001,World,0 -clickon.com.br,clickon.com.br,inbound,001,World,0 -clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,001,World,0 -clicplan.com,dmdelivery.com,inbound,001,World,0 -climber.com,climber.com,inbound,001,World,0 -clinique.com,esteelauder.com,inbound,001,World,0 -clubcupon.com.ar,clubcupon.com.ar,inbound,001,World,0 -cmail1.com,createsend.com,inbound,001,World,0 -cmail2.com,createsend.com,inbound,001,World,0 -cmm01.com,coremotivesmarketing.com,inbound,001,World,0 -cmrfalabella.com,cmrfalabella.com,inbound,001,World,0 -coach.com,delivery.net,inbound,001,World,0 -cobone.com,emarsys.net,inbound,001,World,0 -cocacola.co.jp,cocacola.co.jp,inbound,001,World,0 -codebreak.info,codebreak.info,inbound,001,World,1 -codeproject.com,codeproject.com,inbound,001,World,0 -coldwatercreek.com,coldwatercreek.com,inbound,001,World,0 -collectionsetc.com,collectionsetc.com,inbound,001,World,0 -columbia.edu,columbia.edu,inbound,001,World,0.762355 -combzmail.jp,combzmail.jp,inbound,001,World,0 -comcast.net,comcast.net,inbound,001,World,0.888399 -comcast.net,comcast.net,outbound,001,World,0.999999 -comenity.net,alldata.net,inbound,001,World,1 -comenity.net,bigfootinteractive.com,inbound,001,World,0 -commonfloor.com,commonfloor.com,inbound,001,World,1 -communicatoremail.com,communicatoremail.com,inbound,001,World,0 -communitymatrimony.com,communitymatrimony.com,inbound,001,World,1 -compute.internal,amazonaws.com,inbound,001,World,0.858276 -computerworld.com,computerworld.com,inbound,001,World,0 -comunicacaodemkt.com,locaweb.com.br,inbound,001,World,0 -confirmedoptin.com,confirmedoptin.com,inbound,001,World,0 -confirmsignup.com,mmsend53.com,inbound,001,World,0 -conrepmail.com,conrepmail.com,inbound,001,World,0 -constantcontact.com,confirmedcc.com,inbound,001,World,0 -constantcontact.com,constantcontact.com,inbound,001,World,5.3e-05 -constantcontact.com,postini.com,inbound,001,World,0.078144 -constantcontact.com,yahoo.{...},inbound,001,World,0.999724 -contact-darty.com,mm-send.com,inbound,001,World,0 -contactlab.it,contactlab.it,inbound,001,World,0 -containerstore.com,containerstore.com,inbound,001,World,0 -continente.pt,1-hostingservice.com,inbound,001,World,0 -converse.com,converse.com,inbound,001,World,1 -convio.net,convio.net,inbound,001,World,0 -cookingchanneltv.com,cookingchanneltv.com,inbound,001,World,0 -cookpad.com,cookpad.com,inbound,001,World,0 -copernica.nl,picsrv.net,inbound,001,World,0.011507 -copernica.nl,vicinity.nl,inbound,001,World,0.011753 -coppel.com,coppel.com,inbound,001,World,0 -coremotivesmarketing.com,coremotivesmarketing.com,inbound,001,World,0 -cornell.edu,cornell.edu,inbound,001,World,0.195104 -corporateperks.com,nextjump.com,inbound,001,World,0 -correosocc.com,correosocc.com,inbound,001,World,1 -costco.co.uk,costco.com,inbound,001,World,0 -costco.com,costco.com,inbound,001,World,7e-06 -costcophotocenter.com,wc09.net,inbound,001,World,0 -costcoservices.com,costco.com,inbound,001,World,0 -cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,001,World,0 -couchsurfing.org,couchsurfing.com,inbound,001,World,0 -countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,001,World,0 -couponamama.com,couponamama.com,inbound,001,World,1 -coupondunia.in,coupondunia.in,inbound,001,World,1 -cox.com,cox.com,inbound,001,World,0.001665 -cox.net,cox.net,inbound,001,World,0.009187 -cox.net,cox.net,outbound,001,World,0 -coyotelogistics.com,postini.com,inbound,001,World,0 -cp20.com,cp20.com,inbound,001,World,0 -cpbnc.com,cpbnc.com,inbound,001,World,0 -cpbnc.com,fye.com,inbound,001,World,0 -cpc.gov.in,cpc.gov.in,inbound,001,World,0 -cpm.co.ma,cpm.co.ma,inbound,001,World,0 -crabtree-evelyn.com,crabtree-evelyn.com,inbound,001,World,0.000566 -crackle.com,crackle.com,inbound,001,World,0 -craigslist.org,craigslist.org,inbound,001,World,0 -craigslist.org,craigslist.org,outbound,001,World,1 -crainnewsalerts.com,crainnewsalerts.com,inbound,001,World,0 -crashlytics.com,crashlytics.com,inbound,001,World,1 -crashlytics.com,sendgrid.net,inbound,001,World,1 -crateandbarrel.com,crateandbarrel.com,inbound,001,World,0 -cratusservices.in,ramcorp.in,inbound,001,World,0 -creationsrewards.net,creationsrewards.net,inbound,001,World,0 -creditkarma.com,creditkarma.com,inbound,001,World,1 -credoaction.com,credoaction.com,inbound,001,World,1 -cricinfo.com,cricinfo.com,inbound,001,World,0 -cricut.com,elabs12.com,inbound,001,World,0 -criticalimpactinc.com,criticalimpactinc.com,inbound,001,World,0 -critsend.com,critsend.com,inbound,001,World,0 -crmstyle.com,crmstyle.com,inbound,001,World,0 -crocos.jp,crocos.jp,inbound,001,World,0 -crocs-email.com,crocs-email.com,inbound,001,World,0 -crosswalkmail.com,crosswalkmail.com,inbound,001,World,0 -crowdcut.com,crowdcut.com,inbound,001,World,1 -crsend.com,crsend.com,inbound,001,World,0.008688 -crunchyroll.com,crunchyroll.com,inbound,001,World,0 -csas.cz,csas.cz,inbound,001,World,0.999971 -ctrip.com,ctrip.com,inbound,001,World,0.01917 -cudo.com.au,exacttarget.com,inbound,001,World,0 -cuenote.jp,cuenote.jp,inbound,001,World,0 -cumulusdist.net,cumulusdist.net,inbound,001,World,0 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,001,World,1 -cuponatic.com.pe,cuponatic.com.pe,inbound,001,World,1 -cuponicamail.com,fnbox.com,inbound,001,World,0 -cuppon.pl,cuppon.pl,inbound,001,World,0 -curbednetwork.com,curbednetwork.com,inbound,001,World,1 -curriculum.com.br,curriculum.com.br,inbound,001,World,0 -currys.co.uk,currys.co.uk,inbound,001,World,0 -cuspemail.com,neimanmarcusemail.com,inbound,001,World,0 -custom-emailing.com,elabs12.com,inbound,001,World,0 -custombriefings.com,custombriefings.com,inbound,001,World,0 -customercenter.net,customercenter.net,inbound,001,World,0.996453 -customeriomail.com,customeriomail.com,inbound,001,World,1 -cv-library.co.uk,cv-library.co.uk,inbound,001,World,0 -cvbankas.lt,efadm.eu,inbound,001,World,0 -cvent-planner.com,cvent-planner.com,inbound,001,World,0 -cw.com.tw,cw.com.tw,inbound,001,World,0.002535 -cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 -cxomedia.com,cxomedia.com,inbound,001,World,0 -cybercoders.com,cybercoders.com,inbound,001,World,0 -cyberdiet.com.br,allinmedia.com.br,inbound,001,World,0 -cyberlinkmember.com,cyberlinkmember.com,inbound,001,World,0 -d-reizen.nl,dmdelivery.com,inbound,001,World,0 -dabmail.com,iaires.com,inbound,001,World,0 -dabmail.com,mailurja.com,inbound,001,World,0 -dafiti.cl,dafiti.cl,inbound,001,World,0 -dafiti.com.br,fagms.de,inbound,001,World,0 -dailyhoroscope.com,tarot.com,inbound,001,World,0 -dailyom.com,dailyom.com,inbound,001,World,1 -dairyqueen.com,dairyqueen.com,inbound,001,World,0 -datadrivenemail.com,datadrivenemail.com,inbound,001,World,0 -datehookup.com,datehookup.com,inbound,001,World,0 -datingfactory.com,caerussolutions.net,inbound,001,World,0 -datingvipnotifications.com,datingvipnotifications.com,inbound,001,World,0 -daveramsey.com,daveramsey.com,inbound,001,World,0.025924 -daviacalendar.com,daviacalendar.com,inbound,001,World,1 -davidsbridal.com,davidsbridal.com,inbound,001,World,0 -davidstea.com,bronto.com,inbound,001,World,0 -daz3d.com,bronto.com,inbound,001,World,0 -dbgi.co.uk,emc1.co.uk,inbound,001,World,0 -ddc-emails.com,ddc-emails.com,inbound,001,World,0 -deal.com.sg,emarsys.net,inbound,001,World,0 -dealchicken.com,dealchicken.com,inbound,001,World,0 -dealchicken.com,exacttarget.com,inbound,001,World,0 -dealersocket.com,dealersocket.com,inbound,001,World,4e-06 -dealfind.com,dealfind.com,inbound,001,World,0 -dealnews.com,dealnews.com,inbound,001,World,0 -dealsaver.com,secondstreetmedia.com,inbound,001,World,0.999823 -dealsdirect.com.au,dealsdirect.com.au,inbound,001,World,0 -dealspl.us,dealspl.us,inbound,001,World,0 -debian.org,debian.org,inbound,001,World,1 -debshops.com,lstrk.net,inbound,001,World,1 -deezer.com,dms30.com,inbound,001,World,0 -deliasshopemail.com,deliasshopemail.com,inbound,001,World,0 -delivery.net,delivery.net,inbound,001,World,0 -delivery.net,m0.net,inbound,001,World,0 -dell.com,bfi0.com,inbound,001,World,0 -dell.com,dell.com,inbound,001,World,0.969277 -delta.com,delta.com,inbound,001,World,0.092496 -dena.ne.jp,dena.ne.jp,inbound,001,World,0.000249 -dentalsenders.com,dentalsenders.com,inbound,001,World,0 -dermstore.com,exacttarget.com,inbound,001,World,0 -descontos.pt,descontos.pt,inbound,001,World,0 -designerapparel.com,myperfectsale.com,inbound,001,World,1 -despegar.com,despegar.com,inbound,001,World,0 -dhgate.com,chtah.net,inbound,001,World,0 -dhl.com,dhl.com,inbound,001,World,0.994107 -dice.com,dice.com,inbound,001,World,0 -dietaesaude.com.br,dietaesaude.com.br,inbound,001,World,0 -dietnavi.com,data-hotel.net,inbound,001,World,0 -digitalmailer.com,digitalmailer.com,inbound,001,World,0 -digitalmedia-comunicacion.es,chtah.net,inbound,001,World,0 -digitalromanceinc.com,digitalromanceinc.com,inbound,001,World,1 -dinda.com.br,dinda.com.br,inbound,001,World,0.017061 -dip-net.co.jp,dip-net.co.jp,inbound,001,World,0 -directcrm.ru,directcrm.ru,inbound,001,World,0 -directresponsemanager.com,wide.ne.jp,inbound,001,World,0 -directv.com,directv.com,inbound,001,World,0.050604 -directvla.com,directvla.com,inbound,001,World,0 -disc.co.jp,disc.co.jp,inbound,001,World,0.001051 -discover.com,discover.com,inbound,001,World,0 -discover.com,discoverfinancial.com,inbound,001,World,1 -dishtv.co.in,dishtv.co.in,inbound,001,World,0.047664 -disney.co.uk,emv9.com,inbound,001,World,0 -disneydestinations.com,disneyparks.com,inbound,001,World,0 -disneydestinations.com,disneyworld.com,inbound,001,World,0 -disparadordeemails.com,locaweb.com.br,inbound,001,World,0 -disqus.net,disqus.net,inbound,001,World,0 -diynetwork.com,diynetwork.com,inbound,001,World,0 -dks.com.tw,dks.com.tw,inbound,001,World,0.018134 -dmm.com,dmm.com,inbound,001,World,0 -dn.net,naukri.com,inbound,001,World,0 -docomo.ne.jp,docomo.ne.jp,inbound,001,World,0 -docomo.ne.jp,docomo.ne.jp,outbound,001,World,0 -doctoroz.com,email-sharecare2.com,inbound,001,World,0 -docusign.net,docusign.net,inbound,001,World,0.985435 -dollartree.com,email-dollartree.com,inbound,001,World,0 -dominos.com,dominos.com,inbound,001,World,0.011684 -dominos.com.au,dominos.com.au,inbound,001,World,0 -dominosemail.co.uk,dominosemail.co.uk,inbound,001,World,0 -donationnet.net,donationnet.net,inbound,001,World,0 -donuts.ne.jp,dnuts.jp,inbound,001,World,0 -doodle.com,doodle.com,inbound,001,World,1 -dorothyperkins.com,dorothyperkins.com,inbound,001,World,0 -dotmailer-email.com,dotmailer.com,inbound,001,World,0 -dotmailer.co.uk,dotmailer.com,inbound,001,World,0 -dotz.com.br,dotz.com.br,inbound,001,World,0 -doubletakeoffers.com,doubletakeoffers.com,inbound,001,World,0 -dowjones.info,dowjones.info,inbound,001,World,0 -downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,001,World,0 -dpapp.nl,prikbordmailer.nl,inbound,001,World,0 -dpapp.nl,sslsecuref.nl,inbound,001,World,0 -dptagent.biz,dptagent.biz,inbound,001,World,0 -dptagent.net,dptagent.net,inbound,001,World,0 -draftkings.com,draftkings.com,inbound,001,World,0 -dreamhost.com,dreamhost.com,inbound,001,World,0 -dreammail.ne.jp,dreammail.jp,inbound,001,World,0 -dreamwidth.org,dreamwidth.org,inbound,001,World,0 -dreivip.com,dreivip.com,inbound,001,World,0.000301 -dress-for-less.de,privalia.com,inbound,001,World,0 -drhinternet.net,drhinternet.net,inbound,001,World,0 -driftem.com,emce2.in,inbound,001,World,0 -driftem.com,mailurja.com,inbound,001,World,0 -drjays-mail.com,drjays-mail.com,inbound,001,World,0 -dromadaire-news.com,ecmcluster.com,inbound,001,World,0 -dropbox.com,dropbox.com,inbound,001,World,1 -dropboxmail.com,dropbox.com,inbound,001,World,1 -drushim.co.il,drushim.co.il,inbound,001,World,0 -drweb.com,drweb.com,inbound,001,World,0.969315 -dstyleweb.com,dstyleweb.com,inbound,001,World,0.001099 -dsw.com,dsw.com,inbound,001,World,0 -ducks.org,uptilt.com,inbound,001,World,0 -duke.edu,duke.edu,inbound,001,World,0.308101 -dukecareers.com,dukecareers.com,inbound,001,World,1 -duluthtradingemail.com,email-duluthtrading.com,inbound,001,World,0 -dvor.com,dvor.com,inbound,001,World,0 -dynamite-safelist.com,thomas-j-brown.com,inbound,001,World,0 -dynect-mailer.net,dynect.net,inbound,001,World,0 -dynect-mailer.net,sendlabs.com,inbound,001,World,0 -e-activist.com,e-activist.com,inbound,001,World,0 -e-beallsonline.com,e-stagestores.com,inbound,001,World,0 -e-bodyc.com,email-bodycentral.com,inbound,001,World,0 -e-boks.dk,e-boks.dk,inbound,001,World,1 -e-costco.mx,costco.com,inbound,001,World,0 -e-ebuyer.com,e-ebuyer.com,inbound,001,World,0 -e-goodysonline.com,e-stagestores.com,inbound,001,World,0 -e-jobs-ville.com,e-jobs-ville.com,inbound,001,World,1 -e-leclerc.com,e-leclerc.com,inbound,001,World,0.000248 -e-mark.nl,e-mark.nl,inbound,001,World,0 -e-ngine.nl,e-ngine.nl,inbound,001,World,0 -e-peebles.com,e-stagestores.com,inbound,001,World,0 -e-rewards.net,e-rewards.net,inbound,001,World,1 -e-stagestores.com,e-stagestores.com,inbound,001,World,0 -e-travelclub.es,e-travelclub.es,inbound,001,World,0 -e2ma.net,e2ma.net,inbound,001,World,1 -ea.com,ea.com,inbound,001,World,0.001314 -eaccess.net,postini.com,inbound,001,World,0 -earn-e-miles.com,earn-e-miles.com,inbound,001,World,0 -earnerslist.com,traxweb.net,inbound,001,World,9e-06 -earthfare-email.com,edclient2.com,inbound,001,World,0 -earthlink.net,earthlink.net,inbound,001,World,0.031678 -earthlink.net,earthlink.net,outbound,001,World,0 -eastbay.com,footlocker.com,inbound,001,World,0 -easycanvasprints.com,easycanvasprints.com,inbound,001,World,0 -easyhealthoptions.com,easyhealthoptions.com,inbound,001,World,1 -easyhits4u.com,easyhits4u.com,inbound,001,World,1 -easyhits4u.com,relmax.net,inbound,001,World,0 -easyroommate.com,easyroommate.com,inbound,001,World,0.108483 -ebags.com,ebags.com,inbound,001,World,0 -ebates.com,bfi0.com,inbound,001,World,0 -ebay-kleinanzeigen.de,mobile.de,inbound,001,World,1 -ebay.{...},ebay.{...},inbound,001,World,0.99953 -ebay.{...},emarsys.net,inbound,001,World,0 -ebay.{...},postdirect.com,inbound,001,World,0 -ebizac2.com,ebizac2.com,inbound,001,World,0 -ebizac3.com,ebizac3.com,inbound,001,World,0 -eblastengine.com,secondstreetmedia.com,inbound,001,World,0.999827 -ebuildabear.com,ebuildabear.com,inbound,001,World,8e-06 -ec2.internal,amazonaws.com,inbound,001,World,0.769117 -ec21.com,ec21.com,inbound,001,World,0.002261 -ecasend.com,ecasend.com,inbound,001,World,0 -ecnavi.jp,ecnavi.jp,inbound,001,World,0 -ecommzone.com,ecommzone.com,inbound,001,World,0 -ed.gov,leepfrog.com,inbound,001,World,0.106381 -ed10.net,ed10.com,inbound,001,World,0 -ed10.net,postini.com,inbound,001,World,0.083658 -edarling.fr,fagms.de,inbound,001,World,0 -eddiebauer.com,eddiebauer.com,inbound,001,World,0 -edima.hu,edima.hu,inbound,001,World,0 -edirect1.com,ivytech.edu,inbound,001,World,0 -edmodo.com,edmodo.com,inbound,001,World,1.1e-05 -educationzone.co.in,iaires.com,inbound,001,World,0 -eduk.com,eduk.com,inbound,001,World,0 -efamilydollar.com,efamilydollar.com,inbound,001,World,0 -effectivesafelist.com,zoothost.com,inbound,001,World,0 -efox-shop.com,dmdelivery.com,inbound,001,World,0 -eharmony.com,eharmony.com,inbound,001,World,1e-06 -eigbox.net,eigbox.net,inbound,001,World,0 -ejobs.ro,ejobs.ro,inbound,001,World,8.4e-05 -elabs10.com,elabs10.com,inbound,001,World,0 -elabs12.com,elabs12.com,inbound,001,World,0 -elabs3.com,elabs3.com,inbound,001,World,0 -elabs3.com,meritline.com,inbound,001,World,0 -elabs5.com,elabs5.com,inbound,001,World,0 -elabs6.com,elabs6.com,inbound,001,World,0 -elaine-asp.de,artegic.net,inbound,001,World,0.999997 -elanceonline.com,elanceonline.com,inbound,001,World,0 -elcorteingles.es,elcorteingles.es,inbound,001,World,0 -eleadtrack.net,eleadtrack.net,inbound,001,World,0 -elektronskaposta.si,eprvak.si,inbound,001,World,0 -elettershop.de,servicemail24.de,inbound,001,World,1 -elistas.net,elistas.net,inbound,001,World,0 -elitesafelist.com,elitesafelist.com,inbound,001,World,0 -elkjop.no,ec-cluster.com,inbound,001,World,0 -elkjop.no,eccluster.com,inbound,001,World,0 -elo7.com.br,elo7.com.br,inbound,001,World,0 -emag.ro,emag.ro,inbound,001,World,0.005013 -email-1800contacts.com,email-1800contacts.com,inbound,001,World,0 -email-aaa.com,email-aaa.com,inbound,001,World,0 -email-aeriagames.com,email-aeriagames.com,inbound,001,World,0 -email-comparethemarket.com,smartfocusdigital.net,inbound,001,World,0 -email-cooking.com,email-cooking.com,inbound,001,World,0 -email-dressbarn.com,email-dressbarn.com,inbound,001,World,0 -email-firestone.com,reminder-firestone.com,inbound,001,World,0 -email-galls.com,email-galls.com,inbound,001,World,1 -email-honest.com,email-honest.com,inbound,001,World,0 -email-od.com,email-od.com,inbound,001,World,0.999652 -email-od.com,smtprelayserver.com,inbound,001,World,0.999779 -email-petsmart.com,email-petsmart.com,inbound,001,World,0 -email-sportchalet.com,email-sportchalet.com,inbound,001,World,0 -email-telekom.de,ecm-cluster.com,inbound,001,World,0 -email-ticketdada.com,email-ticketdada.com,inbound,001,World,1 -email-totalwine.com,email-totalwine.com,inbound,001,World,0 -email-wildstar-online.com,email-carbine.com,inbound,001,World,0 -email2-beyond.com,messagebus.com,inbound,001,World,0 -email360api.com,email360api.com,inbound,001,World,0 -email365inc.com,email365inc.com,inbound,001,World,0 -email3m.com,email3m.com,inbound,001,World,0 -email4-beyond.com,email4-beyond.com,inbound,001,World,0 -emailcounts.com,secureserver.net,inbound,001,World,0 -emaildir2.com,emaildirect.net,inbound,001,World,0 -emaildir2.com,espsnd.com,inbound,001,World,0 -emailnotify.net,emailnotify.net,inbound,001,World,0.961424 -emailrestaurant.com,emailrestaurant.com,inbound,001,World,0 -emailsbancoestado.cl,emailsbancoestado.cl,inbound,001,World,0 -emailsripley.cl,etarget.cl,inbound,001,World,0 -emailtoryburch.com,emailtoryburch.com,inbound,001,World,0 -emarsys.net,emarsys.net,inbound,001,World,0.092041 -embarqmail.com,centurylink.net,inbound,001,World,0.999918 -embluejet.com,embluejet.com,inbound,001,World,0 -embluejet.com,emblueuser.com,inbound,001,World,0 -emcsend.com,emcsend.com,inbound,001,World,0 -emergencyemail.org,emergencyemail.org,inbound,001,World,0 -eminentinc.com,eminentinc.com,inbound,001,World,0 -emktsender.net,locaweb.com.br,inbound,001,World,0 -emktviajarbarato.com.br,splio.com.br,inbound,001,World,0.875902 -emma.cl,emma.cl,inbound,001,World,0.996895 -emobile.ad.jp,postini.com,inbound,001,World,0 -employboard.com,employboard.com,inbound,001,World,1 -empoweredcomms.com.au,empoweredcomms.com.au,inbound,001,World,0 -emsecure.net,emsecure.net,inbound,001,World,0 -emsmtp.com,emsmtp.com,inbound,001,World,0.175797 -en-japan.com,en-japan.com,inbound,001,World,0.000204 -en25.com,kqed.org,inbound,001,World,0 -enewscartes.net,bp06.net,inbound,001,World,0 -enewsletter.pl,enewsletter.pl,inbound,001,World,0.005395 -enewsletter.pl,mydeal.pl,inbound,001,World,0 -enewsletter.pl,sare25.com,inbound,001,World,0 -enplenitud.com,enplenitud.com,inbound,001,World,0 -entregadeemails.com,locaweb.com.br,inbound,001,World,0 -entregadordecampanhas.net,locaweb.com.br,inbound,001,World,0 -entrepreneur.com,entrepreneur.com,inbound,001,World,0 -enviodecampanhas.net,locaweb.com.br,inbound,001,World,0 -enviodemkt.com.br,locaweb.com.br,inbound,001,World,0 -eonet.ne.jp,eonet.ne.jp,inbound,001,World,0.99955 -epaper.com.tw,epaper.com.tw,inbound,001,World,0 -eplus.jp,eplus.jp,inbound,001,World,0 -epriority.com,epriority.com,inbound,001,World,0 -equifax.com,equifax.com,inbound,001,World,0.936013 -equussafelist.com,equussafelist.com,inbound,001,World,0.000265 -eslitebooks.com,eslitebooks.com,inbound,001,World,0 -espmp-agfr.net,bp06.net,inbound,001,World,0 -esprit-friends.com,esprit-friends.com,inbound,001,World,0 -esri.com,esri.com,inbound,001,World,0.983104 -esteelauder.com,esteelauder.com,inbound,001,World,0 -ethingsremembered.com,ethingsremembered.com,inbound,001,World,0 -etrade.com,etrade.com,inbound,001,World,0.021422 -etransmail.com,etransmail.com,inbound,001,World,0 -etransmail.com,ptransmail.com,inbound,001,World,0 -etrmailbox.com,etrmailbox.com,inbound,001,World,0 -etsy.com,etsy.com,inbound,001,World,0.020844 -euromsg.net,euromsg.net,inbound,001,World,0 -evaair.com,evaair.com,inbound,001,World,0.007363 -evanguard.com,evanguard.com,inbound,001,World,0 -evanscycles.com,msgfocus.com,inbound,001,World,0 -eventbrite.com,eventbrite.com,inbound,001,World,0 -evernote.com,evernote.com,inbound,001,World,1 -eversavelocal.com,eversavelocal.com,inbound,001,World,0 -everydayfamily.com,everydayfamily.com,inbound,001,World,0 -everydayhealthinc.com,waterfrontmedia.net,inbound,001,World,0 -everyjobforme.com,everyjobforme.com,inbound,001,World,0 -everytown.org,everytown.org,inbound,001,World,1 -exacttarget.com,bazaarvoice.com,inbound,001,World,0 -exacttarget.com,booksamillion.com,inbound,001,World,0 -exacttarget.com,exacttarget.com,inbound,001,World,0.000325 -exacttarget.com,msg.com,inbound,001,World,0 -exacttarget.com,redboxinstant.com,inbound,001,World,0 -exacttarget.com,skylinetechnologies.com,inbound,001,World,0 -exchangesolutions.com,exchangesolutions.com,inbound,001,World,0.000143 -exec-u-net-mail.com,exec-u-net-mail.com,inbound,001,World,0 -expediamail.com,airasiago.com,inbound,001,World,1 -expediamail.com,exacttarget.com,inbound,001,World,1 -expediamail.com,expediamail.com,inbound,001,World,0.839662 -expediamail.com,quotitmail.com,inbound,001,World,0 -experteer.com,experteer.com,inbound,001,World,0 -express.com,expressfashion.com,inbound,001,World,0 -exprpt.com,exprpt.com,inbound,001,World,0 -expvtinboxhub.net,expvtinboxhub.net,inbound,001,World,0 -extra.com.br,emv8.com,inbound,001,World,0 -eyepin.com,eyepin.com,inbound,001,World,0 -ezweb.ne.jp,ezweb.ne.jp,inbound,001,World,0.282076 -ezweb.ne.jp,ezweb.ne.jp,outbound,001,World,0 -fabfurnish.com,fagms.de,inbound,001,World,0 -fabletics.com,bronto.com,inbound,001,World,0 -facebook.com,facebook.com,inbound,001,World,0.553197 -facebook.com,facebook.com,outbound,001,World,1 -facebookappmail.com,facebook.com,inbound,001,World,1 -facebookmail.com,facebook.com,inbound,001,World,1 -facebookmail.com,postini.com,inbound,001,World,0.726315 -facebookmail.com,yahoo.{...},inbound,001,World,0.999904 -facilisimo.com,facilisimo.com,inbound,001,World,1 -fagms.net,fagms.de,inbound,001,World,0 -falabella.com,falabella.com,inbound,001,World,0 -familychristianmail.com,familychristianmail.com,inbound,001,World,0 -famousfootwear.com,famousfootwear.com,inbound,001,World,0 -fanatics.com,fanatics.com,inbound,001,World,0 -fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,001,World,0 -fanbridge.com,fanbridge.com,inbound,001,World,0.000198 -fanfiction.com,fictionpress.com,inbound,001,World,1 -fanofannas.com,fanofannas.com,inbound,001,World,0 -fansedge.com,fansedge.com,inbound,001,World,0 -farmers.com,farmers.com,inbound,001,World,0.999984 -farmersonly.com,mailgun.net,inbound,001,World,1 -farmersonly.com,mailgun.us,inbound,001,World,1 -fashion2hub.in,mgenie.in,inbound,001,World,0 -fastcompany.com,fastcompany.com,inbound,001,World,0 -fastgb.com,fastgb.com,inbound,001,World,0 -fastlistmailer.com,zoothost.com,inbound,001,World,0 -fastweb.com,fastweb.com,inbound,001,World,0 -fbi.gov,fbi.gov,inbound,001,World,0 -fbmta.com,fbmta.com,inbound,001,World,0 -fc2.com,fc2.com,inbound,001,World,0.000798 -fedex.com,fedex.com,inbound,001,World,0.921011 -fedoraproject.org,fedoraproject.org,inbound,001,World,5e-06 -feedblitz.com,feedblitz.com,inbound,001,World,0 -feld-ent.com,postdirect.com,inbound,001,World,0 -felissimo.jp,felissimo.jp,inbound,001,World,0 -fellowshiponemail.com,fellowshiponemail.com,inbound,001,World,0 -fetlifemail.com,fetlifemail.com,inbound,001,World,0 -fibertel.com.ar,fibertel.com.ar,inbound,001,World,0.003897 -fidelity.com,fidelity.com,inbound,001,World,1 -fidelizador.org,fidelizador.org,inbound,001,World,0 -financialfreedommail.com,financialfreedommail.com,inbound,001,World,0 -finansbank.com.tr,finansbank.com.tr,inbound,001,World,0.927 -findexpvtinbox.com,findexpvtinbox.com,inbound,001,World,0 -fingerhut.com,fingerhut.com,inbound,001,World,0.020664 -finishline.com,finishline.com,inbound,001,World,0 -finn.no,schibsted-it.no,inbound,001,World,0.002893 -firemountaingems.com,firemountaingems.com,inbound,001,World,0 -fiscosoft.com.br,fiscosoft.com.br,inbound,001,World,0 -fisher-price.com,fisher-price.com,inbound,001,World,0 -fitbit.com,fitbit.com,inbound,001,World,1 -fitnessmagazine.com,meredith.com,inbound,001,World,0 -fiverr.com,fiverr.com,inbound,001,World,0 -fixeads.com,fixeads.com,inbound,001,World,0 -flets.com,flets.com,inbound,001,World,0 -flexmls.com,flexmls.com,inbound,001,World,0.999992 -flightaware.com,flightaware.com,inbound,001,World,0.020698 -flipkart.com,flipkart.com,inbound,001,World,1 -flirchi.com,flirchi.com,inbound,001,World,1.0 -flirt.com,ropot.net,inbound,001,World,0 -flirthookup.com,flirthookup.com,inbound,001,World,1 -flirtlocal.com,flirtlocal.com,inbound,001,World,1 -flmsecure.com,fling.com,inbound,001,World,0 -flmsecure.com,flmsecure.com,inbound,001,World,0 -floridajobdepartment.com,floridajobdepartment.com,inbound,001,World,0 -flyceb.com,flyceb.com,inbound,001,World,0 -flyfrontier.com,flyfrontier.com,inbound,001,World,0 -flymonarchemail.com,flymonarchemail.com,inbound,001,World,0 -fmworld.net,fmworld.net,inbound,001,World,0 -fnac.com,fnac.com,inbound,001,World,0.021985 -fnb.co.za,fnb.co.za,inbound,001,World,0.104983 -fofa.jp,mpme.jp,inbound,001,World,0 -follow-up.se,follow-up.se,inbound,001,World,1 -foodnetwork.com,foodnetwork.com,inbound,001,World,0 -foolsubs.com,foolcs.com,inbound,001,World,0 -foolsubs.com,foolsubs.com,inbound,001,World,0 -footaction.com,footlocker.com,inbound,001,World,0 -footlocker.com,footlocker.com,inbound,001,World,0.000605 -forcemail.in,iaires.com,inbound,001,World,0 -foreseegame.com,iaires.com,inbound,001,World,0 -forever21.com,forever21.com,inbound,001,World,0 -fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,001,World,0 -fotffamily.com,fotffamily.com,inbound,001,World,0 -fotocasa.es,fotocasa.es,inbound,001,World,0 -fotolivro.com.br,fotolivro.com.br,inbound,001,World,0 -fotostrana.ru,fotocdn.net,inbound,001,World,3.4e-05 -foursquare.com,foursquare.com,inbound,001,World,1 -foxnews.com,foxnews.com,inbound,001,World,0.0139 -fpmailerbr.com,fpmailerbr.com,inbound,001,World,0 -fragrancenet.com,fragrancenet.com,inbound,001,World,0.000427 -francescas.com,bronto.com,inbound,001,World,0 -free-lance.ru,free-lance.ru,inbound,001,World,0 -free.fr,free.fr,inbound,001,World,0.984012 -free.fr,free.fr,outbound,001,World,6.9e-05 -freeadsmailer.com,zoothost.com,inbound,001,World,0 -freebeesafelist.com,zoothost.com,inbound,001,World,0 -freebizmag.com,delivery.net,inbound,001,World,0 -freebsd.org,freebsd.org,inbound,001,World,0.999835 -freecycle.org,freecycle.org,inbound,001,World,0.999927 -freedesktop.org,freedesktop.org,inbound,001,World,0 -freeflys.com,freeflys.com,inbound,001,World,0 -freelancer.com,freelancer.com,inbound,001,World,0 -freelancer.com,freelancernotify.com,inbound,001,World,0 -freelancer.com,getafreelancer.com,inbound,001,World,0 -freelists.org,iquest.net,inbound,001,World,0 -freelotto.com,plasmanetinc.com,inbound,001,World,0 -freemail.hu,freemail.hu,outbound,001,World,0 -freeml.com,gmo-media.jp,inbound,001,World,0 -freepeople.com,freepeople.com,inbound,001,World,0 -freesafelistking.com,zoothost.com,inbound,001,World,0 -freesafelistmailer.com,waters-advertising.com,inbound,001,World,0 -freshdesk.com,freshdesk.com,inbound,001,World,1 -freshers2015.com,secureserver.net,inbound,001,World,0 -freshlatesave.com,freshlatesave.com,inbound,001,World,1 -freshmail.pl,freshmail.pl,inbound,001,World,0 -fridays.com,fridays.com,inbound,001,World,0 -friskone.com,mailurja.com,inbound,001,World,0 -frk.com,frk.com,inbound,001,World,0.999995 -frontdoor.com,frontdoor.com,inbound,001,World,0 -frontgate-email.com,frontgate-email.com,inbound,001,World,0 -frontsight.com,frontsight.com,inbound,001,World,0 -frys.com,frys.com,inbound,001,World,0.00388 -frysmail.com,frysmail.com,inbound,001,World,0 -fspeletters.com,agorapub.co.uk,inbound,001,World,0 -ftchinese.com,ftchinese.com,inbound,001,World,0 -fubonshop.com,fubonshop.com,inbound,001,World,0 -fuckbooknet.net,infinitypersonals.com,inbound,001,World,0 -fuelrewards.com,britecast.com,inbound,001,World,0 -fundplaza.co.in,arrowsignindia.com,inbound,001,World,0 -fundplaza.in,fundplaza.in,inbound,001,World,0 -funonthenet.in,funonthenet.in,inbound,001,World,1 -futureshop.com,futureshop.com,inbound,001,World,0 -futurmailer.pt,futurmailer.pt,inbound,001,World,0 -gabbar.info,gabbar.info,inbound,001,World,1 -gaiaonline.com,gaiaonline.com,inbound,001,World,0 -gamecity.ne.jp,gamecity.ne.jp,inbound,001,World,0 -gamefly.com,gamefly.com,inbound,001,World,0.013381 -gamehouse.com,gamehouse.com,inbound,001,World,0 -gamingmails.com,gamingmails.com,inbound,001,World,0 -gap.com,gap.com,inbound,001,World,0 -gap.eu,gap.eu,inbound,001,World,0 -gapcanada.ca,gapcanada.ca,inbound,001,World,0 -garanti.com.tr,euromsg.net,inbound,001,World,0 -garanti.com.tr,garanti.com.tr,inbound,001,World,0.421936 -gardeningclubmail.co.uk,msgfocus.com,inbound,001,World,0 -garnethill-email.com,garnethill-email.com,inbound,001,World,0 -gaylordalert.com,gaylordalert.com,inbound,001,World,0 -gbyguess.com,guess.com,inbound,001,World,0.001787 -gcast.com.au,systemsserver.net,inbound,001,World,0 -gdtsuccess.com,groupdealtools.com,inbound,001,World,1 -geico.com,geico.com,inbound,001,World,0.096879 -gemoney.com,rsys1.com,inbound,001,World,0 -gene.com,roche.com,inbound,001,World,1 -generalmills.com,boxtops4education.com,inbound,001,World,0 -generalmills.com,pillsbury.com,inbound,001,World,0 -gentoo.org,gentoo.org,inbound,001,World,1 -geocaching.com,groundspeak.com,inbound,001,World,1 -geojit.com,geojit.com,inbound,001,World,0.203748 -get-me-jobs.com,get-me-jobs.com,inbound,001,World,0 -gethired.com,gethired.com,inbound,001,World,1 -getinbox.net,getinbox.net,inbound,001,World,0 -getitfree.us,getitfree.us,inbound,001,World,0 -getkeepsafe.com,getkeepsafe.com,inbound,001,World,1 -getmein.com,getmein.com,inbound,001,World,0 -getpaidsolutions.com,getpaidsolutions.com,inbound,001,World,1 -getpocket.com,bronto.com,inbound,001,World,0 -getresponse.com,getresponse.com,inbound,001,World,0 -gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,001,World,0 -ghin.com,ghinconnect.com,inbound,001,World,0 -ghup.in,mgenie.in,inbound,001,World,0 -giffgaff.com,giffgaff.com,inbound,001,World,0 -gillyhicks-email.com,abercrombie-email.com,inbound,001,World,0 -gilt.com,gilt.com,inbound,001,World,1e-06 -gilt.jp,gilt.jp,inbound,001,World,0 -github.com,github.com,inbound,001,World,1 -github.com,github.net,inbound,001,World,1 -github.com,postini.com,inbound,001,World,0.872186 -glassdoor.com,glassdoor.com,inbound,001,World,0.662834 -glasses.com,glasses.com,inbound,001,World,0 -gliq.com,gliq.com,inbound,001,World,0.99852 -globalmembersupport.com,globalmembersupport.com,inbound,001,World,0 -globalsafelist.com,globalsafelist.com,inbound,001,World,0 -globalsources.com,globalsources.com,inbound,001,World,0.007546 -globalspec.com,globalspec.com,inbound,001,World,0 -globaltestmarket.com,globaltestmarket.com,inbound,001,World,0 -globasemail.com,globasemail.com,inbound,001,World,0.951088 -globetel.com.ph,globetel.com.ph,inbound,001,World,1 -gmail.com,02.net,inbound,001,World,0.998007 -gmail.com,amazonaws.com,inbound,001,World,0.988441 -gmail.com,anteldata.net.uy,inbound,001,World,0.998653 -gmail.com,as13285.net,inbound,001,World,0.999943 -gmail.com,asianet.co.th,inbound,001,World,0.998473 -gmail.com,au-net.ne.jp,inbound,001,World,1 -gmail.com,bbox.fr,inbound,001,World,0.919868 -gmail.com,bbtec.net,inbound,001,World,1 -gmail.com,belgacom.be,inbound,001,World,0.822281 -gmail.com,bell.ca,inbound,001,World,0.992482 -gmail.com,bellsouth.net,inbound,001,World,0.9996 -gmail.com,bezeqint.net,inbound,001,World,0.974444 -gmail.com,bigpond.net.au,inbound,001,World,0.999907 -gmail.com,blackberry.com,inbound,001,World,0.996337 -gmail.com,bluewin.ch,inbound,001,World,0.9337 -gmail.com,brasiltelecom.net.br,inbound,001,World,0.99995 -gmail.com,btcentralplus.com,inbound,001,World,0.999969 -gmail.com,centurytel.net,inbound,001,World,0.999415 -gmail.com,cgocable.net,inbound,001,World,0.998291 -gmail.com,charter.com,inbound,001,World,0.999111 -gmail.com,chello.nl,inbound,001,World,0.999951 -gmail.com,claro.net.br,inbound,001,World,1 -gmail.com,comcast.net,inbound,001,World,0.999616 -gmail.com,comcastbusiness.net,inbound,001,World,0.985464 -gmail.com,cox.net,inbound,001,World,0.962636 -gmail.com,data-hotel.net,inbound,001,World,0.000605 -gmail.com,emailsrvr.com,inbound,001,World,1 -gmail.com,embarqhsd.net,inbound,001,World,0.999554 -gmail.com,fastwebnet.it,inbound,001,World,0.984708 -gmail.com,franchiseindia.com,inbound,001,World,1 -gmail.com,frontiernet.net,inbound,001,World,0.995233 -gmail.com,gvt.net.br,inbound,001,World,0.999513 -gmail.com,hinet.net,inbound,001,World,0.97312 -gmail.com,iinet.net.au,inbound,001,World,0.966984 -gmail.com,jazztel.es,inbound,001,World,0.999701 -gmail.com,lorexddns.net,inbound,001,World,0 -gmail.com,majesticmoneymailer.com,inbound,001,World,1 -gmail.com,mchsi.com,inbound,001,World,0.999951 -gmail.com,movistar.cl,inbound,001,World,0.999361 -gmail.com,mtnl.net.in,inbound,001,World,0.999527 -gmail.com,mycingular.net,inbound,001,World,0.999918 -gmail.com,myvzw.com,inbound,001,World,0.999889 -gmail.com,naukri.com,inbound,001,World,0.000998 -gmail.com,net24.it,inbound,001,World,0.999964 -gmail.com,netcabo.pt,inbound,001,World,0.998164 -gmail.com,netvigator.com,inbound,001,World,0.818637 -gmail.com,numericable.fr,inbound,001,World,0.999726 -gmail.com,ocn.ne.jp,inbound,001,World,0.99466 -gmail.com,ono.com,inbound,001,World,0.995991 -gmail.com,optonline.net,inbound,001,World,0.999776 -gmail.com,optusnet.com.au,inbound,001,World,0.992856 -gmail.com,orange.es,inbound,001,World,0.998743 -gmail.com,orange.fr,inbound,001,World,0 -gmail.com,otenet.gr,inbound,001,World,0.957964 -gmail.com,panda-world.ne.jp,inbound,001,World,1 -gmail.com,postini.com,inbound,001,World,0.776029 -gmail.com,proxad.net,inbound,001,World,0.998094 -gmail.com,qwest.net,inbound,001,World,0.997575 -gmail.com,rcn.com,inbound,001,World,0.986768 -gmail.com,rima-tde.net,inbound,001,World,0.99915 -gmail.com,rogers.com,inbound,001,World,0.999917 -gmail.com,rr.com,inbound,001,World,0.967896 -gmail.com,sbcglobal.net,inbound,001,World,0.998817 -gmail.com,secureserver.net,inbound,001,World,0.272513 -gmail.com,seed.net.tw,inbound,001,World,0.992252 -gmail.com,sfr.net,inbound,001,World,0.999878 -gmail.com,shawcable.net,inbound,001,World,0.999998 -gmail.com,singnet.com.sg,inbound,001,World,0.955461 -gmail.com,skybroadband.com,inbound,001,World,0.999854 -gmail.com,spcsdns.net,inbound,001,World,0.999998 -gmail.com,suddenlink.net,inbound,001,World,0.961594 -gmail.com,t-ipconnect.de,inbound,001,World,0.999841 -gmail.com,tdc.net,inbound,001,World,0.999591 -gmail.com,telecom.net.ar,inbound,001,World,0.999664 -gmail.com,telecomitalia.it,inbound,001,World,0.996998 -gmail.com,telekom.hu,inbound,001,World,0.999977 -gmail.com,telenet.be,inbound,001,World,1 -gmail.com,telepac.pt,inbound,001,World,0.999584 -gmail.com,telesp.net.br,inbound,001,World,0.999743 -gmail.com,telia.com,inbound,001,World,1 -gmail.com,telkomadsl.co.za,inbound,001,World,0.999989 -gmail.com,telus.com,inbound,001,World,1 -gmail.com,telus.net,inbound,001,World,0.974677 -gmail.com,threembb.co.uk,inbound,001,World,1 -gmail.com,tmodns.net,inbound,001,World,0.999979 -gmail.com,totbb.net,inbound,001,World,0.999876 -gmail.com,tpgi.com.au,inbound,001,World,0.999649 -gmail.com,tpnet.pl,inbound,001,World,0.999761 -gmail.com,veloxzone.com.br,inbound,001,World,0.999969 -gmail.com,verizon.net,inbound,001,World,0.990214 -gmail.com,videotron.ca,inbound,001,World,0.967213 -gmail.com,virginm.net,inbound,001,World,0.996611 -gmail.com,vodacom.co.za,inbound,001,World,1 -gmail.com,vodafone-ip.de,inbound,001,World,1 -gmail.com,vodafone.pt,inbound,001,World,0.999563 -gmail.com,vodafonedsl.it,inbound,001,World,0.999006 -gmail.com,vtr.net,inbound,001,World,0.999072 -gmail.com,wanadoo.fr,inbound,001,World,0.999753 -gmail.com,websitewelcome.com,inbound,001,World,1 -gmail.com,wideopenwest.com,inbound,001,World,0.999729 -gmail.com,windstream.net,inbound,001,World,0.951847 -gmail.com,yahoo.{...},inbound,001,World,0.999147 -gmail.com,ziggo.nl,inbound,001,World,1 -gmail.com,zoothost.com,inbound,001,World,0.033858 -gmo.jp,gmo-media.jp,inbound,001,World,0 -gmoes.jp,gmoes.jp,inbound,001,World,0 -gmsend.com,gmsend.com,inbound,001,World,0 -gmt.ne.jp,gmt.ne.jp,inbound,001,World,0 -gmx.de,gmx.net,inbound,001,World,1 -gmx.de,gmx.net,outbound,001,World,1 -gmx.net,gmx.net,inbound,001,World,1 -gnavi.co.jp,gnavi.co.jp,inbound,001,World,0.002441 -go.com,starwave.com,inbound,001,World,0.006276 -go4worldbusiness.com,go4worldbusiness.com,inbound,001,World,1 -goalunited.org,ccmdcampaigns.net,inbound,001,World,0 -gob.ar,gob.ar,inbound,001,World,0.159673 -gob.ec,gob.ec,inbound,001,World,0.618006 -godaddy.com,secureserver.net,inbound,001,World,0 -godtubemail.com,godtubemail.com,inbound,001,World,0 -godvinemail.com,godvinemail.com,inbound,001,World,0 -gog.com,gog.com,inbound,001,World,0 -gogecapital.com,rsys1.com,inbound,001,World,0 -gogroopie.com,gogroopie.com,inbound,001,World,0.000283 -gohappy.com.tw,gohappy.com.tw,inbound,001,World,0.000104 -goldenbrands.gr,goldenbrands.gr,inbound,001,World,1 -goldenline.pl,goldenline.pl,inbound,001,World,1 -goldenopsafelist.com,zoothost.com,inbound,001,World,0 -goldstar.com,goldstar.com,inbound,001,World,1 -golfmnb.com,golfmnb.com,inbound,001,World,0 -golfnow.com,email-golfnow.com,inbound,001,World,0 -gomaji.com,gomaji.com,inbound,001,World,0 -goodgame.com,emsmtp.com,inbound,001,World,0 -goodlife.pt,emv8.com,inbound,001,World,0 -google.com,postini.com,inbound,001,World,0.703706 -googlegroups.com,postini.com,inbound,001,World,0.674075 -googlemail.com,t-ipconnect.de,inbound,001,World,0.999957 -gop.com,gop.com,inbound,001,World,0 -gopusamedia.com,gopusamedia.com,inbound,001,World,0 -govdelivery.com,govdelivery.com,inbound,001,World,0 -govdelivery.com,postini.com,inbound,001,World,0.089736 -governmentjobs.com,governmentjobs.com,inbound,001,World,0 -gpmailer.com.br,parperfeito.com,inbound,001,World,0 -grabone-mail-ie.com,grabone-mail-ie.com,inbound,001,World,0 -grabone-mail.com,grabone-mail.com,inbound,001,World,0 -grassrootsaction.com,grassfire.net,inbound,001,World,0 -gratka.pl,gratka.pl,inbound,001,World,0 -greatergood.com,greatergood.com,inbound,001,World,0 -gree.jp,gree.jp,inbound,001,World,0 -grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,001,World,0 -groopdealz.com,groopdealz.com,inbound,001,World,1 -groupalia.es,groupalia.es,inbound,001,World,0 -groupalia.it,groupalia.it,inbound,001,World,0 -groupon.jp,data-hotel.net,inbound,001,World,1e-06 -groupon.{...},chtah.net,inbound,001,World,0 -groupon.{...},groupon.{...},inbound,001,World,0.989844 -groupon.{...},postini.com,inbound,001,World,0.887392 -grouponmail.{...},grouponmail.{...},inbound,001,World,0 -grubhubmail.com,grubhubmail.com,inbound,001,World,0 -grupanya.com,euromsg.net,inbound,001,World,0 -grupos.com.br,grupos.com.br,inbound,001,World,0 -gtbank.com,gtbank.com,inbound,001,World,0.056121 -guess.ca,guess.com,inbound,001,World,0.000807 -guess.com,guess.com,inbound,001,World,0.003805 -guessfactory.com,guess.com,inbound,001,World,0.00164 -gumtree.com,marktplaats.nl,inbound,001,World,0 -gumtree.com.au,kijiji.com,inbound,001,World,0 -gunosy.com,gunosy.com,inbound,001,World,0.998682 -guruin.info,guru.net.in,inbound,001,World,1 -gurunavi.jp,gurunavi.jp,inbound,001,World,0 -gustazos.com,cityoferta.com,inbound,001,World,1 -gymglish.com,gymglish.com,inbound,001,World,0.000788 -habitaclia.com,splio.es,inbound,001,World,0.951406 -hallmark.com,hallmark.com,inbound,001,World,0 -hannaandersson.com,hannaandersson.com,inbound,001,World,0.013533 -harborfreightemail.com,harborfreightemail.com,inbound,001,World,0 -harristeetermail.com,harristeetermail.com,inbound,001,World,0.99673 -harvard.edu,harvard.edu,inbound,001,World,0.1751 -haskell.org,haskell.org,inbound,001,World,0.000703 -hautelook.com,hautelook.com,inbound,001,World,0.000459 -hayneedle.com,hayneedle.com,inbound,001,World,0 -hazteoir.org,hazteoir.org,inbound,001,World,0.31478 -hdfcbank.com,powerelay.com,inbound,001,World,1 -hdfcbank.net,powerelay.com,inbound,001,World,1 -hdfcbank.net,quickvmail.com,inbound,001,World,0 -helpareporter.net,helpareporter.com,inbound,001,World,0 -hepsiburada.com,euromsg.net,inbound,001,World,0 -herbalifemail.com,herbalifemail.com,inbound,001,World,0 -herculist.com,herculist.com,inbound,001,World,0 -heteml.jp,heteml.jp,inbound,001,World,0.950766 -hgtv.com,hgtv.com,inbound,001,World,0 -hh.ru,hh.ru,inbound,001,World,0.996236 -hhgreggemail.com,hhgreggemail.com,inbound,001,World,0 -hilton.com,hiltonemail.com,inbound,001,World,0 -hinet.net,hinet.net,inbound,001,World,0.007093 -hinet.net,hinet.net,outbound,001,World,0.00565 -hipchat.com,hipchat.com,inbound,001,World,1 -hipmunk.com,hipmunk.com,inbound,001,World,0 -hispavista.com,hispavista.com,inbound,001,World,0 -hln.be,persgroep-ops.net,inbound,001,World,0 -hm-f.jp,hm-f.jp,inbound,001,World,0 -hobsonsmail.com,hobsonsmail.com,inbound,001,World,0 -hollister-email.com,abercrombie-email.com,inbound,001,World,0 -home.ne.jp,zaq.ne.jp,inbound,001,World,9e-06 -homeaway.com,haspf.com,inbound,001,World,0 -homebaselife.com,ec-cluster.com,inbound,001,World,0 -homechoice.co.za,homechoice.co.za,inbound,001,World,0 -homedecorators.com,homedecorators.com,inbound,001,World,0 -homedepot.com,homedepot.com,inbound,001,World,1 -homedepotemail.com,homedepotemail.com,inbound,001,World,0 -honto.jp,honto.jp,inbound,001,World,0 -hootsuite.com,hootsuite.com,inbound,001,World,1 -horchowemail.com,horchowemail.com,inbound,001,World,0 -horoscope.com,center.com,inbound,001,World,2e-06 -hostelworld.com,bronto.com,inbound,001,World,0 -hostgator.com,hostgator.com,inbound,001,World,0.976131 -hostgator.com,websitewelcome.com,inbound,001,World,0.999822 -hotel.de,emp-mail.de,inbound,001,World,0 -hotels.com,hotels.com,inbound,001,World,0 -hotelurbano.com.br,allin.com.br,inbound,001,World,0 -hotmail.{...},hotmail.{...},inbound,001,World,0.999968 -hotmail.{...},hotmail.{...},outbound,001,World,1 -hotmail.{...},postini.com,inbound,001,World,0.837967 -hotornot.com,monopost.com,inbound,001,World,0.99472 -hotschedules.com,hotschedules.com,inbound,001,World,0 -hotspotmailer.com,hotspotmailer.com,inbound,001,World,1 -hotukdeals.com,hotukdeals.com,inbound,001,World,1 -hotwire.com,hotwire.com,inbound,001,World,0 -house.gov,house.gov,inbound,001,World,0.999966 -houseoffraser.co.uk,houseoffraser.co.uk,inbound,001,World,0 -houzz.com,houzz.com,inbound,001,World,1 -hp.com,hp.com,inbound,001,World,0.202841 -hpnotifier.nl,hpnotifier.nl,inbound,001,World,0 -hsbc.co.in,hsbc.com.hk,inbound,001,World,1 -hsbc.com.hk,hsbc.com.hk,inbound,001,World,1 -hsn.com,hsn.com,inbound,001,World,0 -htcampusmailer.com,eccluster.com,inbound,001,World,0 -hubspot.com,hubspot.com,inbound,001,World,1 -huinforma.com.br,huinforma.com.br,inbound,001,World,0 -hulumail.com,hulumail.com,inbound,001,World,0 -hungry-girl.com,hungry-girl.com,inbound,001,World,0 -hungryhouse.co.uk,mxmfb.com,inbound,001,World,0 -huntington.com,huntington.com,inbound,001,World,0.987351 -i-part.com.tw,i-part.com.tw,inbound,001,World,0.001865 -i-say.com,ipsos-interactive.com,inbound,001,World,1 -iamlgnd2.com,iamlgnd2.com,inbound,001,World,1 -ibm.com,ibm.com,inbound,001,World,0.94413 -ibps.in,sify.net,inbound,001,World,0 -ibpsorg.org,sify.net,inbound,001,World,0 -ibsys.com,ibsys.com,inbound,001,World,9e-05 -icbc.com.ar,clickexperts.net,inbound,001,World,0 -icbc.com.ar,standardbank.com.ar,inbound,001,World,0 -icelandmail.co.uk,emsg-live.co.uk,inbound,001,World,0 -icicibank.com,icicibank.com,inbound,001,World,0.009835 -icicisecurities.com,icicibank.com,inbound,001,World,0.000803 -icims.com,icims.com,inbound,001,World,0.999939 -icloud.com,apple.com,inbound,001,World,1 -icloud.com,icloud.com,outbound,001,World,1 -icloud.com,mac.com,inbound,001,World,1 -icloud.com,me.com,inbound,001,World,0.999995 -icors.org,lsoft.us,inbound,001,World,0 -icpbounce.com,icpbounce.com,inbound,001,World,0 -idc.email,nmsrv.com,inbound,001,World,1 -idealista.com,idealista.com,inbound,001,World,0.001627 -ideascost.com,ramcorp.in,inbound,001,World,0 -idgconnect-resources.com,idgconnect-resources.com,inbound,001,World,0 -ieee.org,ieee.org,inbound,001,World,0.999912 -ifttt.com,ifttt.com,inbound,001,World,1 -ig.com.br,ig.com.br,inbound,001,World,0 -ig.com.br,ig.com.br,outbound,001,World,0 -ign.com,ign.com,inbound,001,World,0 -ignitionsender.com,ignitionsender.com,inbound,001,World,0 -igot-mails.com,zoothost.com,inbound,001,World,0 -iheart.com,iheart.com,inbound,001,World,0 -iimjobs.com,iimjobs.com,inbound,001,World,1 -ikmultimedianews.com,ikmultimedianews.com,inbound,001,World,0.993319 -illinois.edu,illinois.edu,inbound,001,World,0.866481 -imageshost.ca,imageshost.ca,inbound,001,World,0 -imakenews.net,imakenews.com,inbound,001,World,0 -imi.ne.jp,lifemedia.jp,inbound,001,World,0 -immobilienscout24.de,immobilienscout24.de,inbound,001,World,1 -imo.im,imo.im,inbound,001,World,1 -imodules.com,imodules.com,inbound,001,World,0 -imvu.com,imvu.com,inbound,001,World,7e-06 -in-boxpays.com,in-boxpays.com,inbound,001,World,0 -inboxair.com,inboxair.com,inbound,001,World,0 -inboxdollars.com,inboxdollars.com,inbound,001,World,0 -inboxfirst.com,inboxfirst.com,inbound,001,World,0 -inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,001,World,0.999877 -inboxpays.com,inboxpays.com,inbound,001,World,0 -inboxpounds.co.uk,inboxpounds.co.uk,inbound,001,World,0 -indeed.com,indeed.com,inbound,001,World,0.000121 -indeedemail.com,indeedemail.com,inbound,001,World,0 -independentlivingbullion.com,independentlivingbullion.com,inbound,001,World,0 -indiamart.com,indiamart.com,inbound,001,World,1 -indiaproperty.com,indiaproperty.com,inbound,001,World,0 -indiatimes.com,speakingtree.in,inbound,001,World,0 -indiatimeshop.com,sendpal.in,inbound,001,World,0 -indieroyale.com,desura.com,inbound,001,World,1 -infibeam.com,eccluster.com,inbound,001,World,0 -infobradesco.com.br,infobradesco.com.br,inbound,001,World,0 -infoempleo.com,infoempleo.com,inbound,001,World,0 -infojobs.com.br,anuntis.com,inbound,001,World,0 -infojobs.it,infojobs.it,inbound,001,World,0 -infojobs.net,infojobs.net,inbound,001,World,0 -infomoney.com.br,infomoney.com.br,inbound,001,World,1 -infopanel.jp,mailds.jp,inbound,001,World,0 -infopraca.pl,careesma.com,inbound,001,World,0 -informz.net,informz.net,inbound,001,World,0 -infos-micromania.com,infos-micromania.com,inbound,001,World,0 -infosephora.com,splio.com,inbound,001,World,0.962167 -infoworld.com,infoworld.com,inbound,001,World,0 -infradead.org,infradead.org,inbound,001,World,1 -infusionmail.com,infusionmail.com,inbound,001,World,0 -ingdirect.es,ingdirect.es,inbound,001,World,1 -inman.com,inman.com,inbound,001,World,0 -inmotionhosting.com,inmotionhosting.com,inbound,001,World,0.990051 -innovyx.net,innovyx.net,inbound,001,World,0 -ino.com,ino.com,inbound,001,World,0.999981 -insidehook.com,sailthru.com,inbound,001,World,0 -instagram.com,facebook.com,inbound,001,World,1 -instantprofitlist.com,screenshotads.com,inbound,001,World,0 -intage.co.jp,intage.co.jp,inbound,001,World,0.00557 -inter-chat.com,inter-chat.com,inbound,001,World,0 -interac.ca,certapay.com,inbound,001,World,0 -interactivebrokers.com,interactivebrokers.com,inbound,001,World,1 -interactiverealtyservices.com,interactiverealtyservices.com,inbound,001,World,0 -intercom.io,mailgun.info,inbound,001,World,1 -interdatesa.com,fagms.net,inbound,001,World,0 -interealty.net,interealty.net,inbound,001,World,1 -internations.org,internations.org,inbound,001,World,1 -interweave.com,interweave.com,inbound,001,World,0 -interwell.gr,interwell.gr,inbound,001,World,0 -intliv2.net,internationalliving.com,inbound,001,World,0 -intuit.com,intuit.com,inbound,001,World,0.89114 -invalidemail.com,taleo.net,inbound,001,World,1 -investopedia.com,vclk.net,inbound,001,World,0.999999 -investorplace.com,investorplace.com,inbound,001,World,0.995093 -inx1and1.de,1and1.com,inbound,001,World,1 -inxserver.com,inxserver.de,inbound,001,World,0.952863 -inxserver.de,inxserver.de,inbound,001,World,0.980838 -ipcmedia.co.uk,ipcmedia.co.uk,inbound,001,World,0 -iqelite.com,iqelite.com,inbound,001,World,0 -irctcshopping.com,chtah.net,inbound,001,World,0 -iridium.com,iridium.com,inbound,001,World,0.997882 -isbank.com.tr,isbank.com.tr,inbound,001,World,0.009655 -isendservice.com.br,isendservice.com.br,inbound,001,World,0 -itau-unibanco.com.br,itau.com.br,inbound,001,World,0 -itms.in.ua,itms.in.ua,inbound,001,World,0 -itsmyascent.com,itsmyascent.com,inbound,001,World,0 -ittoolbox.com,ittoolbox.com,inbound,001,World,0 -ittoolbox.com,toolbox.com,inbound,001,World,0 -itunes.com,apple.com,inbound,001,World,0.082921 -itwhitepapers.com,itwhitepapers.com,inbound,001,World,0 -iwantoneofthose.com,thehut.com,inbound,001,World,0 -ixs1.net,ixs1.net,inbound,001,World,0.005131 -jackwills.com,jackwills.com,inbound,001,World,0 -jalag.de,jalag.de,inbound,001,World,1 -jane.com,jane.com,inbound,001,World,1 -jango.com,jango.com,inbound,001,World,0 -jared.com,jared.com,inbound,001,World,0 -jcity.com,jcity.com,inbound,001,World,0 -jcpenney.com,jcpenney.com,inbound,001,World,9e-06 -jdate.com,postdirect.com,inbound,001,World,0 -jeevansathi.com,jeevansathi.com,inbound,001,World,0 -jetprivilege.com,jetprivilege.com,inbound,001,World,0 -jetsetter.com,smartertravelmedia.com,inbound,001,World,0.041362 -jeuxvideo.com,jeuxvideo.com,inbound,001,World,0.148921 -jeweloscoemail.com,email-mywebgrocer2.com,inbound,001,World,0 -jibjab.com,storybots.com,inbound,001,World,0 -jira.com,uc-inf.net,inbound,001,World,1 -jiscmail.ac.uk,lsoft.se,inbound,001,World,0 -joann-mail.com,joann-mail.com,inbound,001,World,0 -jobinsider.com,jobinsider.com,inbound,001,World,0 -jobinthailand.com,jobinthailand.com,inbound,001,World,1 -jobisjob.com,jobisjob.com,inbound,001,World,0 -jobmaster.co.il,jobmaster1.co.il,inbound,001,World,0 -jobomas.com,jobomas.com,inbound,001,World,1 -jobrapidoalert.com,jobrapidoalert.com,inbound,001,World,0 -jobs2web.com,ondemand.com,inbound,001,World,1 -jobscentral.com.sg,mailgun.net,inbound,001,World,1 -jobsdbalert.co.id,jobsdbalert.co.id,inbound,001,World,0 -jobsdbalert.com,jobsdbalert.com,inbound,001,World,0 -jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,001,World,0 -jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,001,World,0 -jobserve.com,jobserve.com,inbound,001,World,0 -jobsindubai.com,jobsindubai.ca,inbound,001,World,0 -jobsite.co.uk,jobsite.co.uk,inbound,001,World,0.000509 -jobson.com,jobsonmail.com,inbound,001,World,0 -jobsradar.com,jobsradar.com,inbound,001,World,0 -jobstreet.com,jobstreet.com,inbound,001,World,0 -jockeycomfort.com,jockeycomfort.com,inbound,001,World,0 -johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,001,World,0 -jomashop.com,lstrk.net,inbound,001,World,1 -joobmailer.com,joobmailer.com,inbound,001,World,0 -josbank.com,josbank.com,inbound,001,World,0 -joshin.co.jp,joshin.co.jp,inbound,001,World,8e-06 -jossandmain.com,jossandmain.com,inbound,001,World,0 -jpcycles.com,jpcycles.com,inbound,001,World,0.001716 -jtv.com,jtv.com,inbound,001,World,0 -jumia.com.ng,fagms.de,inbound,001,World,0 -jungleerummy.com,jungleerummy.com,inbound,001,World,1 -juno.com,untd.com,inbound,001,World,0 -juno.com,untd.com,outbound,001,World,0 -jusbrasil.com.br,jusbrasil.com.br,inbound,001,World,0 -just-eat.co.uk,ec-cluster.com,inbound,001,World,0 -justclick.ru,justclick.ru,inbound,001,World,0 -justdial.com,iaires.com,inbound,001,World,0 -justdial.com,mailurja.com,inbound,001,World,0 -justfab.com,bronto.com,inbound,001,World,0 -justfab.fr,bronto.com,inbound,001,World,0 -k1speed.com,k1speed.com,inbound,001,World,1 -kagoya.net,kagoya.net,inbound,001,World,0.013097 -kalunga.com.br,kalunga.com.br,inbound,001,World,0 -karvy.com,karvy.com,inbound,001,World,0.045186 -kasikornbank.com,kasikornbank.com,inbound,001,World,0 -kaskusnetworks.com,kaskus.com,inbound,001,World,0 -kay.com,kay.com,inbound,001,World,0 -keek.com,keek.com,inbound,001,World,1 -kernel.org,kernel.org,inbound,001,World,0 -kgbdeals.co.uk,email1-kgbdeals.com,inbound,001,World,0 -kgstores.com,kgstores.com,inbound,001,World,0 -kiabi.com,dms-02.net,inbound,001,World,0 -kickstarter.com,kickstarter.com,inbound,001,World,1 -kidsfootlocker.com,footlocker.com,inbound,001,World,0 -kijiji.ca,kijiji.com,inbound,001,World,0 -kik.com,kik.com,inbound,001,World,1 -kimblegroup.com,kimblegroup.com,inbound,001,World,1 -kintera.com,kintera.com,inbound,001,World,0 -kismia.com,kismia.com,inbound,001,World,1 -kiwari.com,kiwari.com,inbound,001,World,9e-06 -klaviyomail.com,klaviyomail.com,inbound,001,World,1 -kliksa.net,euromsg.net,inbound,001,World,0 -kliktoday.com,kliktoday.com,inbound,001,World,0 -klm-mail.com,klm-mail.com,inbound,001,World,0 -klove.com,emfbroadcasting.com,inbound,001,World,0 -kohls.com,kohls.com,inbound,001,World,0 -komando.com,komando.com,inbound,001,World,0 -kongregate.com,kongregate.com,inbound,001,World,0 -kotak.com,kotak.com,inbound,001,World,0.115651 -kp.org,kp.org,inbound,001,World,0.999975 -krogermail.com,bigfootinteractive.com,inbound,001,World,0 -krs.bz,tricorn.net,inbound,001,World,0 -kubra.com,kubra.com,inbound,001,World,1.8e-05 -kundenserver.de,kundenserver.de,inbound,001,World,1 -kvbmail.com,kvbmail.com,inbound,001,World,0 -la-meteo-mail.fr,splio.com,inbound,001,World,1 -laaptuemail.com,laaptuemail.com,inbound,001,World,0 -lakewoodchurch.com,lakewoodchurch.com,inbound,001,World,0 -lancers.jp,lancers.jp,inbound,001,World,0 -landmarketingmailer.com,zoothost.com,inbound,001,World,0 -landofnod.com,landofnod.com,inbound,001,World,0.002525 -landsend.com,email-landsend.com,inbound,001,World,0 -landsend.com,postdirect.com,inbound,001,World,0 -languagepod101.com,eclient10.com,inbound,001,World,0 -languagepod101.com,eddlvr.com,inbound,001,World,0 -languagepod101.com,ednwsltr3.com,inbound,001,World,0 -languagepod101.com,ednwsltr8.com,inbound,001,World,0 -languagepod101.com,emaildirect.net,inbound,001,World,0 -laposte.net,laposte.net,inbound,001,World,0.271722 -laposte.net,laposte.net,outbound,001,World,0 -laptuinvite.com,laptuinvite.com,inbound,001,World,0 -laredoute.fr,laredoute.fr,inbound,001,World,0 -lasenza.com,lasenza.com,inbound,001,World,0 -lastcallemail.com,lastcallemail.com,inbound,001,World,0 -lastminute.com,lastminute.com,inbound,001,World,0.00831 -laterooms.com,laterooms.com,inbound,001,World,0.014746 -latimes.com,latimes.com,inbound,001,World,0 -lauraashley.com,lauraashley.com,inbound,001,World,0 -lazerhits.com,lazerhits.com,inbound,001,World,1 -leadercontato.com.br,leadercontato.com.br,inbound,001,World,0 -leboncoin.fr,leboncoin.fr,inbound,001,World,0 -lefigaro.fr,splio.com,inbound,001,World,1 -leftlanesports.com,auspient.com,inbound,001,World,0.00705 -leftlanesports.com,leftlanesports.com,inbound,001,World,0 -legalshieldassociate.com,legalshield.com,inbound,001,World,0 -lelong.my,lelong.com.my,inbound,001,World,1 -lelong.my,lelong.net.my,inbound,001,World,1 -lemonde.fr,lemonde.fr,inbound,001,World,0.000111 -leparisien.fr,leparisien.fr,inbound,001,World,0 -lexico.com,lexico.com,inbound,001,World,0 -lexpress.fr,bp06.net,inbound,001,World,0 -libero.it,libero.it,inbound,001,World,0.00024 -libero.it,libero.it,outbound,001,World,0 -life360.com,life360.com,inbound,001,World,1 -lifecare-news.com,email-lifecare.com,inbound,001,World,0 -lifecooler.com,1-hostingservice.com,inbound,001,World,0 -lifemiles.com,bigfootinteractive.com,inbound,001,World,0 -lifescript.com,ilinkmd.com,inbound,001,World,0 -lindenlab.com,lindenlab.com,inbound,001,World,0.999444 -line.me,naver.com,inbound,001,World,1 -line6.com,line6.com,inbound,001,World,2.7e-05 -linkedin.com,linkedin.com,inbound,001,World,0.998882 -linkedin.com,postini.com,inbound,001,World,0.835872 -linkedin.com,yahoo.{...},inbound,001,World,0.999927 -linkshare.com,linksynergy.com,inbound,001,World,0 -liquidation.com,liquidation.com,inbound,001,World,6e-06 -listadventure.com,adlabsinc.com,inbound,001,World,0 -listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,001,World,0.00023 -listeneremail.net,listeneremail.net,inbound,001,World,0 -listia.com,listia.com,inbound,001,World,1 -listjoe.com,adlabsinc.com,inbound,001,World,0 -listnerds.com,listnerds.com,inbound,001,World,0 -listreturn.com,zoothost.com,inbound,001,World,0 -listserve.com,listserve.com,inbound,001,World,0 -listvolta.com,listvolta.com,inbound,001,World,0 -listwire.com,listwire.com,inbound,001,World,0 -litres.ru,litres.ru,inbound,001,World,1 -live.{...},hotmail.{...},inbound,001,World,0.999954 -live.{...},hotmail.{...},outbound,001,World,1 -livedoor.com,livedoor.com,inbound,001,World,0 -livefyre.com,andbit.net,inbound,001,World,1 -livejournal.com,livejournal.com,inbound,001,World,0 -livemailservice.com,livemailservice.com,inbound,001,World,0 -livenation.com,exacttarget.com,inbound,001,World,0 -livescribe.com,bronto.com,inbound,001,World,0 -livingsocial.com,livingsocial.com,inbound,001,World,0 -livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,001,World,0 -lmlmgv.com.br,gvarev.com.br,inbound,001,World,0 -localhires.com,localhires.com,inbound,001,World,1 -loccitane.com,neolane.net,inbound,001,World,0 -loft.com,anntaylor.com,inbound,001,World,0 -logentries.com,logentries.com,inbound,001,World,0 -logitech.com,dvsops.com,inbound,001,World,0 -logmein.com,logmein.com,inbound,001,World,0.031467 -lojasmarisa.com.br,lojasmarisa.com.br,inbound,001,World,0 -lolsolos.com,ultimateadsites.net,inbound,001,World,0.999996 -lombardipublishing.com,lombardipublishing.com,inbound,001,World,0 -lonelywifehookup.com,iverificationsystems.com,inbound,001,World,0 -lookout.com,lookout.com,inbound,001,World,1 -lordandtaylor.com,lordandtaylor.com,inbound,001,World,0 -loveaholics.com,ropot.net,inbound,001,World,0 -lovelywholesale.com,lovelywholesale.com,inbound,001,World,1 -loveplanet.ru,pochta.ru,inbound,001,World,0.640035 -lowcostholidays.co.uk,communicatoremail.com,inbound,001,World,0 -lrsmail.com,lrsmail.com,inbound,001,World,0 -lsi.com,postini.com,inbound,001,World,0.981121 -lt02.net,listrak.com,inbound,001,World,1 -lt02.net,lstrk.net,inbound,001,World,1 -ltdcommodities.com,ltdcomm.net,inbound,001,World,0 -lua.org,pepperfish.net,inbound,001,World,1 -luckymag.com,mkt4500.com,inbound,001,World,0 -ludokados.com,ludokado.com,inbound,001,World,0 -lulu.com,bronto.com,inbound,001,World,0 -lulus.com,lstrk.net,inbound,001,World,1 -lumosity.com,lumosity.com,inbound,001,World,1 -luxa.jp,luxa.jp,inbound,001,World,0 -lynxmail.in,iaires.com,inbound,001,World,0 -lyris.net,lyris.net,inbound,001,World,0 -lyst.com,lyst.com,inbound,001,World,1 -m1e.net,m1e.net,inbound,001,World,0.000342 -m3.com,m3.com,inbound,001,World,0 -mac.com,icloud.com,outbound,001,World,1 -mac.com,mac.com,inbound,001,World,1 -maccosmetics.com,esteelauder.com,inbound,001,World,0 -macromill.com,macromill.com,inbound,001,World,1e-06 -macupdate.com,mailgun.info,inbound,001,World,1 -macys.com,macys.com,inbound,001,World,0 -madmels.info,ultimateadsites.net,inbound,001,World,1 -madmimi.com,madmimi.com,inbound,001,World,0 -mag2.com,tandem-m.com,inbound,001,World,0 -magicbricks.com,tbsl.in,inbound,001,World,0 -magicjack.com,magicjack.com,inbound,001,World,1 -magix.net,magix.net,inbound,001,World,0.673176 -magnetdev.com,magnetmail.net,inbound,001,World,0 -mail-backcountry.com,email-bcmarketing.com,inbound,001,World,0 -mail-boss.com,mail-boss.com,inbound,001,World,0 -mail-cdiscount.com,mail-cdiscount.com,inbound,001,World,0 -mail-mbank.pl,mail-mbank.pl,inbound,001,World,0 -mail-route.com,mail-route.com,inbound,001,World,0 -mail-thestreet.com,mail-thestreet.com,inbound,001,World,0 -mail.mil,mail.mil,inbound,001,World,0 -mail.ru,mail.ru,inbound,001,World,0.987506 -mail.ru,mail.ru,outbound,001,World,0.00674 -mailaccurate.com,mgenie.in,inbound,001,World,0 -mailchimp.com,mailchimp.com,inbound,001,World,0.967415 -maileclipse.com,emce2.in,inbound,001,World,0 -mailengine1.com,mailengine1.com,inbound,001,World,0 -mailer-service.de,mailer-service.de,inbound,001,World,4.9e-05 -mailer4u.in,elabs10.com,inbound,001,World,0 -mailersend.com,mailersend.com,inbound,001,World,0 -mailfacil.com.br,md02.com,inbound,001,World,0 -mailfeast.com,mgenie.in,inbound,001,World,0 -mailgun.org,mailgun.info,inbound,001,World,1 -mailgun.org,mailgun.net,inbound,001,World,1 -mailgun.org,mailgun.us,inbound,001,World,1 -mailing-list.it,mailing-list.it,inbound,001,World,0 -mailingathome.net,mailingathome.net,inbound,001,World,0.999995 -mailjayde.com,mailjayde.com,inbound,001,World,0 -mailjet.com,mailjet.com,inbound,001,World,0.436025 -mailmachine1050.com,mailmachine1050.com,inbound,001,World,0 -mailmailmail.net,mailmailmail.net,inbound,001,World,0 -mailoct.in,tcmailer14.in,inbound,001,World,0 -mailoct1.in,mailoct1.in,inbound,001,World,0 -mailoct1.in,myntramail2.in,inbound,001,World,0 -mailorama.fr,mailorama.fr,inbound,001,World,0 -mailplus.nl,brightbase.net,inbound,001,World,1 -mailpost.in,iaires.com,inbound,001,World,0 -mailpv.net,pvmailer.net,inbound,001,World,1 -mailquant.com,iaires.com,inbound,001,World,0 -mailsend1.com,mailsend6.com,inbound,001,World,0 -mailsender.com.br,mailsender.com.br,inbound,001,World,0 -maisonsdumonde.com,bp06.net,inbound,001,World,0 -makro.nl,srv2.de,inbound,001,World,0.88256 -manager.com.br,manager.com.br,inbound,001,World,0 -mandrillapp.com,backpage.com,inbound,001,World,1 -mandrillapp.com,mandrillapp.com,inbound,001,World,1 -mandrillapp.com,mcsignup.com,inbound,001,World,1 -mandrillapp.com,myjobhelperalerts.com,inbound,001,World,1 -mango.com,emstechnology2.net,inbound,001,World,0 -manipal.edu,iaires.com,inbound,001,World,0 -manta.com,exacttarget.com,inbound,001,World,0 -mapfre.com,emv5.com,inbound,001,World,0 -mar0.net,mar0.net,inbound,001,World,0.976409 -marcustheatres.com,movio.co,inbound,001,World,0 -markandgraham.com,markandgraham.com,inbound,001,World,0 -markavip.com,markavip.com,inbound,001,World,0 -marketer-safelist.com,jsalfianmarketing.com,inbound,001,World,1 -marketinghq.net,elabs8.com,inbound,001,World,0 -marketingprofs.com,marketingprofs.com,inbound,001,World,0.004412 -marketingstudio.com,marketingstudio.com,inbound,001,World,0 -marksandspencer.com,marksandspencer.com,inbound,001,World,0 -marktplaats.nl,marktplaats.nl,inbound,001,World,0 -marlboro.com,marlboro.com,inbound,001,World,0 -maropost.com,biotrustnews.com,inbound,001,World,0 -maropost.com,mailing-truthaboutabs.com,inbound,001,World,0 -maropost.com,maropost.com,inbound,001,World,0 -maropost.com,mp2200.com,inbound,001,World,0 -maropost.com,mp2201.com,inbound,001,World,0 -maropost.com,survivallife.com,inbound,001,World,0 -marykay.com,marykay.com,inbound,001,World,0 -masivapp.com,masivapp.com,inbound,001,World,1 -massageenvyclinics.com,massageenvyclinics.com,inbound,001,World,0 -masterbase.com,masterbase.com,inbound,001,World,0 -mastercard-email.com,mastercard-email.com,inbound,001,World,0 -match.com,match.com,inbound,001,World,0 -matchwereld.nl,matchwereld.nl,inbound,001,World,0 -mate1.net,mate1.net,inbound,001,World,0 -matrixemailer.com,matrixemailer.com,inbound,001,World,0 -maxpark.com,gidepark.ru,inbound,001,World,1 -mbga.jp,mbga.jp,inbound,001,World,0 -mbna.co.uk,ec-cluster.com,inbound,001,World,0 -mbounces.com,emdbms.com,inbound,001,World,0 -mbstrm.com,mobilestorm.com,inbound,001,World,0 -mcafee.com,mcafee.com,inbound,001,World,0.963535 -mcarthurglen.com,mcarthurglen.com,inbound,001,World,0 -mcdlv.net,mcdlv.net,inbound,001,World,0 -mcdlv.net,postini.com,inbound,001,World,0.082163 -mckinsey.com,bigfootinteractive.com,inbound,001,World,0 -mcsv.net,mcsv.net,inbound,001,World,0 -mcsv.net,postini.com,inbound,001,World,0.101883 -mdirector.com,mdrctr.com,inbound,001,World,0 -mdlinx.com,mdlinx.com,inbound,001,World,0 -me.com,icloud.com,outbound,001,World,1 -me.com,mac.com,inbound,001,World,1 -mec.gov.br,mec.gov.br,inbound,001,World,0 -mecumauction.com,mecumauction.com,inbound,001,World,0 -medallia.com,medallia.com,inbound,001,World,0.999682 -mediabistro.com,iworld.com,inbound,001,World,0.006428 -mediapost.com,mediapost.com,inbound,001,World,0 -medium.com,messagebus.com,inbound,001,World,0 -medpagetoday.com,wc09.net,inbound,001,World,0 -medscape.com,medscape.com,inbound,001,World,0 -meetic.com,meetic.com,inbound,001,World,0 -meetmemail.com,meetmemail.com,inbound,001,World,0 -meetup.com,meetup.com,inbound,001,World,5e-06 -megasenders.com,megasenders.com,inbound,001,World,0.07662 -melaleuca.com,melaleuca.com,inbound,001,World,0.003493 -memberdealsusa.com,memberdealsusa.com,inbound,001,World,0 -menswearhouse.com,menswearhouse.com,inbound,001,World,0 -mequedouno.com,mequedouno.com,inbound,001,World,0 -mercadojobs.com,sendgrid.net,inbound,001,World,1 -mercadolibre.com,mercadolibre.com,inbound,001,World,0 -mercadolivre.com,mercadolibre.com,inbound,001,World,0 -merceworld.com,merceworld.com,inbound,001,World,0.996738 -mercola.com,mercola.com,inbound,001,World,0.000369 -merodea.me,sendgrid.net,inbound,001,World,1 -messagegears.net,messagegears.net,inbound,001,World,0 -met-art.com,hydentra.com,inbound,001,World,1 -metro.co.in,srv2.de,inbound,001,World,0.966947 -metrodeal.com,fagms.de,inbound,001,World,0 -mgmresorts.com,mgmresorts.com,inbound,001,World,0 -mgo.com,bronto.com,inbound,001,World,0 -michaels.com,chtah.net,inbound,001,World,0 -michaels.com,michaels.com,inbound,001,World,0 -microcentermedia.com,bfi0.com,inbound,001,World,0 -microsoft.com,hotmail.{...},inbound,001,World,1 -microsoft.com,msn.com,inbound,001,World,1 -microsoftemail.com,microsoftemail.com,inbound,001,World,0 -microsoftemail.com,microsoftstoreemail.com,inbound,001,World,0 -midnightsunsafelist.com,zoothost.com,inbound,001,World,0 -mightydeals.co.uk,mightydeals.co.uk,inbound,001,World,0 -mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,001,World,0 -milfaholic.com,iverificationsystems.com,inbound,001,World,0 -miltnews.com,miltnews.com,inbound,001,World,0 -mindbodyonline.com,mindbodyonline.com,inbound,001,World,1 -mindfieldonline.com,mindfieldonline.com,inbound,001,World,0 -mindmoviesmail.com,mindmoviesmail.com,inbound,001,World,0.004239 -mindvalleymail3.com,mindvalleymail3.com,inbound,001,World,0 -minhavida.com.br,minhavida.com.br,inbound,001,World,0 -mint.com,mint.com,inbound,001,World,0 -minted.com,messagelabs.com,inbound,001,World,0.999669 -mirtesen.ru,mtml.ru,inbound,001,World,0 -missselfridge.com,wallis-fashion.com,inbound,001,World,0 -mistersafelist.com,zoothost.com,inbound,001,World,0 -mit.edu,mit.edu,inbound,001,World,0.868705 -mitula.net,mitula.org,inbound,001,World,0 -mitula.org,mitula.org,inbound,001,World,0 -mixcloudmail.com,mixcloudmail.com,inbound,001,World,0.995482 -mixi.jp,mixi.jp,inbound,001,World,0 -mjinn.com,mailurja.com,inbound,001,World,0 -mkt015.com,mkt015.com,inbound,001,World,0 -mkt022.com,mkt022.com,inbound,001,World,0 -mkt063.com,mkt063.com,inbound,001,World,0 -mkt1136.com,mkt1136.com,inbound,001,World,0 -mkt1985.com,fmlinks.net,inbound,001,World,0 -mkt2010.com,mkt2010.com,inbound,001,World,0 -mkt2106.com,mkt2106.com,inbound,001,World,0 -mkt2170.com,mkt2170.com,inbound,001,World,0 -mkt2181.com,mkt2181.com,inbound,001,World,0 -mkt2615.com,mkt2615.com,inbound,001,World,0 -mkt2813.com,mkt2813.com,inbound,001,World,0 -mkt2944.com,mkt2944.com,inbound,001,World,0 -mkt3134.com,mkt3134.com,inbound,001,World,0 -mkt3142.com,mkt3142.com,inbound,001,World,0 -mkt3156.com,mkt3156.com,inbound,001,World,0 -mkt3203.com,mkt3203.com,inbound,001,World,0 -mkt346.com,mkt346.com,inbound,001,World,0 -mkt3544.com,mkt3544.com,inbound,001,World,0 -mkt3622.com,mkt3622.com,inbound,001,World,0 -mkt3682.com,mkt3682.com,inbound,001,World,0 -mkt3690.com,mkt3690.com,inbound,001,World,0 -mkt3695.com,mkt3695.com,inbound,001,World,0 -mkt3804.com,mkt3804.com,inbound,001,World,0 -mkt3815.com,mkt3815.com,inbound,001,World,0 -mkt3952.com,xoom.com,inbound,001,World,0 -mkt4355.com,mkt4355.com,inbound,001,World,0 -mkt4364.com,mkt4364.com,inbound,001,World,0 -mkt459.com,mkt459.com,inbound,001,World,0 -mkt4701.com,mkt4701.com,inbound,001,World,0 -mkt4728.com,mkt4728.com,inbound,001,World,0 -mkt4731.com,mkt4731.com,inbound,001,World,0 -mkt4738.com,mkt4738.com,inbound,001,World,0 -mkt5071.com,mkt5071.com,inbound,001,World,0 -mkt5098.com,mkt5098.com,inbound,001,World,0 -mkt5131.com,mkt5131.com,inbound,001,World,0 -mkt5144.com,mkt5144.com,inbound,001,World,0 -mkt5144.com,mkt5980.com,inbound,001,World,0 -mkt5144.com,mkt5981.com,inbound,001,World,0 -mkt5181.com,mkt5181.com,inbound,001,World,0 -mkt5269.com,mkt5269.com,inbound,001,World,0 -mkt529.com,mkt529.com,inbound,001,World,0 -mkt5297.com,mkt5297.com,inbound,001,World,0 -mkt5297.com,mkt5309.com,inbound,001,World,0 -mkt5371.com,mkt5371.com,inbound,001,World,0 -mkt5806.com,mkt5806.com,inbound,001,World,0 -mkt5934.com,mkt5934.com,inbound,001,World,0 -mkt5937.com,mkt5937.com,inbound,001,World,0 -mkt5970.com,mkt5970.com,inbound,001,World,0 -mkt6100.com,mkt6098.com,inbound,001,World,0 -mkt6276.com,mkt6276.com,inbound,001,World,0 -mkt6323.com,mkt6323.com,inbound,001,World,0 -mkt746.com,mkt746.com,inbound,001,World,0 -mkt824.com,mkt869.com,inbound,001,World,0 -mktdillards.com,mktdillards.com,inbound,001,World,0 -mktid10.com,1-hostingservice.com,inbound,001,World,0 -mktomail.com,mktdns.com,inbound,001,World,0 -mktomail.com,mktomail.com,inbound,001,World,0 -mktomail.com,mktroute.com,inbound,001,World,0 -ml.com,bankofamerica.com,inbound,001,World,1 -mlgns.com,mlgns.com,inbound,001,World,0 -mlgnserv.com,mlgnserv.com,inbound,001,World,0 -mlsend.com,mlsend.com,inbound,001,World,0 -mlsend2.com,mlsend2.com,inbound,001,World,0 -mlssoccer.com,mlssoccer.com,inbound,001,World,0 -mmaco.net,mmaco.net,inbound,001,World,0.999998 -mmagic.jp,mmagic.jp,inbound,001,World,0.000531 -mmks.it,mail-maker.it,inbound,001,World,0 -mmorpg.com,mmorpg.com,inbound,001,World,0.002979 -mmsecure.nl,donenad.nl,inbound,001,World,0 -mo1send.com,mo1send.com,inbound,001,World,0 -mobile01.com,mobile01.com,inbound,001,World,0 -mobly.com.br,mobly.com.br,inbound,001,World,0 -mocospace.com,mocospace.com,inbound,001,World,0 -modellsemail.com,n-email.net,inbound,001,World,0 -modnakasta.ua,emv5.com,inbound,001,World,0 -monex.co.jp,monex.co.jp,inbound,001,World,0.019424 -moneycontrol.com,active18.com,inbound,001,World,0 -moneyforward.com,moneyforward.com,inbound,001,World,0 -moneymorning.com,moneymappress.com,inbound,001,World,0 -moneysupermarketmail.com,moneysupermarketmail.com,inbound,001,World,0 -monipla.jp,aainc.co.jp,inbound,001,World,0 -monografias.com,elistas.net,inbound,001,World,0 -monster.co.in,monster.co.in,inbound,001,World,0 -monster.com,monster.com,inbound,001,World,0.000231 -monster.com,tmpw.net,inbound,001,World,0.000503 -monsterindia.com,monster.co.in,inbound,001,World,0 -moon-ray.com,moon-ray.com,inbound,001,World,0.001892 -mooply.co,mailendo.com,inbound,001,World,0 -mooresclothing.com,mooresclothing.com,inbound,001,World,0 -morhipo.com,euromsg.net,inbound,001,World,0 -morningstar.net,morningstar.net,inbound,001,World,0 -mothercaregroup.com,neolane.net,inbound,001,World,0 -motosnap.com,motosnap.com,inbound,001,World,0.994795 -moveon.org,moveon.org,inbound,001,World,0.99577 -moviestarplanet.com,moviestarplanet.com,inbound,001,World,0 -mozilla.org,mozilla.com,inbound,001,World,0.633237 -mpme.jp,mpme.jp,inbound,001,World,0 -mpse.jp,emsaqua.jp,inbound,001,World,0 -mpse.jp,emsbeige.jp,inbound,001,World,0 -mpse.jp,emsbrown.jp,inbound,001,World,0 -mpse.jp,emscyan.jp,inbound,001,World,0 -mpse.jp,emsgold.jp,inbound,001,World,0 -mpse.jp,emslime.jp,inbound,001,World,0 -mpse.jp,emsnavy.jp,inbound,001,World,0 -mpse.jp,emspink.jp,inbound,001,World,0 -mpse.jp,emssnow.jp,inbound,001,World,0 -mpse.jp,mpme.jp,inbound,001,World,0 -mpse.jp,yahoo.co.jp,inbound,001,World,0 -mpsnd.ch,agenceweb.net,inbound,001,World,0 -mrc.org,msgfocus.com,inbound,001,World,0 -mrmlsmatrix.com,mrmlsmatrix.com,inbound,001,World,0 -ms.com,ms.com,inbound,001,World,1 -ms00.net,ms00.net,inbound,001,World,0 -msdp1.com,msdp1.com,inbound,001,World,0 -msgfocus.com,msgfocus.com,inbound,001,World,0.011658 -msn.com,hotmail.{...},inbound,001,World,0.999964 -msn.com,hotmail.{...},outbound,001,World,1 -mta.info,ealert.com,inbound,001,World,0 -mtasv.net,mtasv.net,inbound,001,World,0.999999 -musiciansfriend.com,musiciansfriend.com,inbound,001,World,0 -musicnotes-alerts.com,mybuys.com,inbound,001,World,0 -mustanglist.com,mustanglist.com,inbound,001,World,0 -mxmfb.com,mxmfb.com,inbound,001,World,0 -mycheapoair.com,mycheapoair.com,inbound,001,World,0.957931 -mycolorscreen.com,mta4.net,inbound,001,World,0 -mydailymoment.biz,mydailymoment.biz,inbound,001,World,0 -mydailymoment.info,mydailymoment.info,inbound,001,World,0 -mydailymoment.net,mydailymoment.net,inbound,001,World,0 -mydailymoment.us,mydailymoment.us,inbound,001,World,0 -myfedloan.org,aessuccess.org,inbound,001,World,0.328917 -myfitnesspal.com,messagebus.com,inbound,001,World,0 -myfxbook.com,myfxbook.com,inbound,001,World,0 -mygreatlakes.org,glhec.org,inbound,001,World,0.000247 -mygroupon.co.th,grouponmail.{...},inbound,001,World,0 -myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,001,World,0 -myheritage.com,myheritage.com,inbound,001,World,0 -myideeli.com,myideeli.com,inbound,001,World,0 -mymeijer.com,mymeijer.com,inbound,001,World,0 -mymms.com,fagms.de,inbound,001,World,0 -mynavi.jp,mynavi.jp,inbound,001,World,3.5e-05 -myngp.com,ngpweb.com,inbound,001,World,0 -myntramail.com,iaires.com,inbound,001,World,0 -myntramail.com,myntramail.com,inbound,001,World,0 -myntramails.in,icubes.in,inbound,001,World,0 -myoutlets.in,trustmailer.com,inbound,001,World,0 -myperfectsale.com,myperfectsale.com,inbound,001,World,0.999988 -mypoints.com,mypoints.com,inbound,001,World,0 -myprotein.com,thehut.com,inbound,001,World,0 -mysafelistmailer.com,mysafelistmailer.com,inbound,001,World,0.00033 -mysale.my,mysale.my,inbound,001,World,0 -mysale.ph,mysale.ph,inbound,001,World,0 -mysmartprice.com,itzwow.com,inbound,001,World,1 -mysupermarket.co.uk,mysupermarket.co.uk,inbound,001,World,0 -mysurvey.com,mysurvey.com,inbound,001,World,0 -mysurvey.eu,mysurvey.com,inbound,001,World,0 -myvegas.com,myvegas.com,inbound,001,World,1 -myzamanamail.com,myzamanamail.com,inbound,001,World,0 -n-email.net,n-email.net,inbound,001,World,0 -n-email1.net,n-email1.net,inbound,001,World,0 -n-email4.net,n-email4.net,inbound,001,World,0 -naaptoldeals.com,eccluster.com,inbound,001,World,0 -namorico.me,namorico.me,inbound,001,World,1 -nanomail.com.br,araie.com.br,inbound,001,World,1 -napitipp.hu,napitipp.hu,inbound,001,World,0 -nasa.gov,nasa.gov,inbound,001,World,0.082443 -nascar.com,nascar.com,inbound,001,World,0 -nastygal.com,bronto.com,inbound,001,World,0 -nasza-klasa.pl,nasza-klasa.pl,inbound,001,World,0 -nationalexpress.com,nationalexpress.com,inbound,001,World,0 -nationbuilder.com,nationbuilder.com,inbound,001,World,1 -nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,001,World,0 -nature.com,nature.com,inbound,001,World,0 -naukri.com,naukri.com,inbound,001,World,0.002488 -nauta.cu,etecsa.net,inbound,001,World,0 -naver.com,naver.com,inbound,001,World,1 -naver.com,naver.com,outbound,001,World,1 -navy.mil,navy.mil,inbound,001,World,0.152107 -nba.com,nba.com,inbound,001,World,0.005828 -nbaa.org,nbaa.org,inbound,001,World,0 -ncl.com,ncl.com,inbound,001,World,0 -neimanmarcusemail.com,neimanmarcusemail.com,inbound,001,World,0 -nend.net,postini.com,inbound,001,World,0 -neolane.net,neolane.net,inbound,001,World,0.01682 -nesinemail.com,euromsg.net,inbound,001,World,0 -net-a-porter.com,net-a-porter.com,inbound,001,World,0 -net-empregos.com,net-empregos.com,inbound,001,World,0 -net-survey.jp,net-survey.jp,inbound,001,World,0 -netatlantic.com,netatlantic.com,inbound,001,World,0.001085 -netbk.co.jp,netbk.co.jp,inbound,001,World,0.010994 -netcommunity1.com,blackbaud.com,inbound,001,World,0 -netflix.com,amazonses.com,inbound,001,World,0.999999 -netflix.com,netflix.com,inbound,001,World,1 -netlogmail.com,netlogmail.com,inbound,001,World,0 -netopia.pt,netopia.pt,inbound,001,World,0.017294 -netprosoftmail.com,netprosoftmail.com,inbound,001,World,0 -netshoes.com.br,netshoes.com.br,inbound,001,World,0.078768 -netsuite.com,netsuite.com,inbound,001,World,0.57723 -networkworld.com,networkworld.com,inbound,001,World,0 -newegg.com,newegg.com,inbound,001,World,1e-06 -newgrounds.com,newgrounds.com,inbound,001,World,0 -newmarkethealth.com,newmarkethealth.com,inbound,001,World,0 -newrelic.com,sendlabs.com,inbound,001,World,0 -news-h5g.com,news-h5g.com,inbound,001,World,0 -newsletter-verychic.com,splio.es,inbound,001,World,0.9804 -newsmax.com,newsmax.com,inbound,001,World,0.004105 -newspaperdirect.com,newspaperdirect.com,inbound,001,World,0.000177 -newyorktimesinfo.com,newyorktimesinfo.com,inbound,001,World,0 -nexcess.net,nexcess.net,inbound,001,World,0.214803 -next-engine.org,next-engine.org,inbound,001,World,1 -nextdoor.com,mailgun.info,inbound,001,World,1 -nextdoor.com,mailgun.net,inbound,001,World,1 -nextdoor.com,nextdoor.com,inbound,001,World,1 -nfl.com,bfi0.com,inbound,001,World,0 -nflshop.com,nflshop.com,inbound,001,World,0 -nhs.jobs,nhscareersjobs.co.uk,inbound,001,World,0 -nic.in,relayout.nic.in,inbound,001,World,0 -nicovideo.jp,nicovideo.jp,inbound,001,World,0 -nieuwsblad.be,vummail.be,inbound,001,World,0 -nifty.com,nifty.com,inbound,001,World,0.762068 -nih.gov,nih.gov,inbound,001,World,8.8e-05 -nike.com,nike.com,inbound,001,World,0 -nikkei.com,nikkei.co.jp,inbound,001,World,0 -nikkeibp.co.jp,nikkeibp.co.jp,inbound,001,World,4.9e-05 -ninewestmail.com,ninewestmail.com,inbound,001,World,0 -ning.com,ning.com,inbound,001,World,0 -nissen.jp,nissen.jp,inbound,001,World,0 -nixle.com,nixle.com,inbound,001,World,0 -nl00.net,netline.com,inbound,001,World,5e-06 -nl00.net,nl00.net,inbound,001,World,0 -nmp1.com,nmp1.net,inbound,001,World,0 -nokia.com,nokia.com,inbound,001,World,0.001256 -nongnu.org,gnu.org,inbound,001,World,1 -nordstrom.com,taleo.net,inbound,001,World,1 -nortonfromsymantec.com,rsys1.com,inbound,001,World,0 -nos.pt,netcabo.pt,inbound,001,World,6.3e-05 -noticiasaominuto.com,ccmdcampaigns.net,inbound,001,World,0 -noticiasaominuto.com,noticiasaominuto.com,inbound,001,World,0 -novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,001,World,0 -npr.org,npr.org,inbound,001,World,0 -nrholding.net,nrholding.net,inbound,001,World,0 -ns.nl,tripolis.com,inbound,001,World,0 -nsandi.com,mxmfb.com,inbound,001,World,0 -numbersusa.com,numbersusa.com,inbound,001,World,0 -nyandcompany.com,nyandcompany.com,inbound,001,World,0 -nytimes.com,nytimes.com,inbound,001,World,0.00472 -nzsale.co.nz,nzsale.co.nz,inbound,001,World,0 -oakley.com,oakley.com,inbound,001,World,0.000904 -ocadomail.com,ocadomail.com,inbound,001,World,0 -ocmail1.in,tcmail.in,inbound,001,World,0 -ocmail14.in,tcmailer5.in,inbound,001,World,0 -ocmail22.in,tcmailer15.in,inbound,001,World,0 -ocmail22.in,tcmailer4.in,inbound,001,World,0 -ocmail40.in,tcmailer15.in,inbound,001,World,0 -ocmail40.in,tcmailer4.in,inbound,001,World,0 -ocn.ad.jp,ocn.ad.jp,inbound,001,World,0 -ocn.ne.jp,ocn.ad.jp,inbound,001,World,0 -ocnmail.in,ocmail6.in,inbound,001,World,0 -ocnmail.in,tcmail3.in,inbound,001,World,0 -odisseias.com,emv4.net,inbound,001,World,0 -odnoklassniki.ru,odnoklassniki.ru,inbound,001,World,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,001,World,0 -ofertasefacil.com.br,ofertasefacil.com.br,inbound,001,World,0 -ofertix.com,ofertix.com,inbound,001,World,0 -ofertop.pe,icommarketing.com,inbound,001,World,0 -offers.com,offers.com,inbound,001,World,1 -offerum.com,cccampaigns.com,inbound,001,World,0 -offerum.com,ccemails.com,inbound,001,World,0 -officedepot.com,officedepot.com,inbound,001,World,0.012483 -officemax.com,officemax.com,inbound,001,World,0 -officemax.com,officemaxworkplace.com,inbound,001,World,0 -ofsys.com,bulletin-metro.ca,inbound,001,World,0 -oknotify2.com,oknotify2.com,inbound,001,World,0 -oldnavy.ca,oldnavy.ca,inbound,001,World,0 -oldnavy.com,oldnavy.com,inbound,001,World,0 -olx.pt,fixeads.com,inbound,001,World,0 -olympiaedge.net,olympiaedge.net,inbound,001,World,0 -omahasteaks.com,omahasteaks.com,inbound,001,World,0.006139 -oneindia.in,infimail.com,inbound,001,World,0 -oneindia.in,mailurja.com,inbound,001,World,0 -onekingslane.com,onekingslane.com,inbound,001,World,0 -onepatriotplace.com,britecast.com,inbound,001,World,0 -onestopplus.com,neolane.net,inbound,001,World,0 -onetravelspecials.com,onetravelspecials.com,inbound,001,World,0.365386 -online.com,cnet.com,inbound,001,World,0 -onlive.com,ipost.com,inbound,001,World,0 -onmicrosoft.com,outlook.com,inbound,001,World,1 -onthecitymail.org,onthecitymail.org,inbound,001,World,1 -oo155.com,bsftransmit7.com,inbound,001,World,0 -oo155.com,oo155.com,inbound,001,World,0 -openstack.org,openstack.org,inbound,001,World,0.996884 -openstackmail.com,infimail.com,inbound,001,World,0 -opentable.com,opentable.com,inbound,001,World,0.000662 -opinionoutpost.com,opinionoutpost.com,inbound,001,World,0 -opinionsquare.com,opinionsquare.com,inbound,001,World,0 -oprah.com,oprah.com,inbound,001,World,0 -opticsplanet.com,opticsplanet.com,inbound,001,World,0 -optimusmail.in,iaires.com,inbound,001,World,0 -optonline.net,cv.net,inbound,001,World,0 -optonline.net,optonline.net,outbound,001,World,0 -orange.fr,orange.fr,inbound,001,World,0 -orange.fr,orange.fr,outbound,001,World,0 -orderscatalog.com,orderscatalog.com,inbound,001,World,0 -oriental-trading.com,oriental-trading.com,inbound,001,World,0 -oroscopofree.com,adsender.us,inbound,001,World,0 -oroscopofree.com,oroscopofree.com,inbound,001,World,0 -orsay.com,emp-mail.de,inbound,001,World,0 -os-email.com,os-email.com,inbound,001,World,0 -oshkoshbgosh.com,oshkoshbgosh.com,inbound,001,World,6e-06 -osu.edu,outlook.com,inbound,001,World,1 -otto.de,eccluster.com,inbound,001,World,1 -ouffer.com,ouffer.com,inbound,001,World,0.01022 -ourtime.com,seniorpeoplemeet.com,inbound,001,World,0 -outback.com,outback.com,inbound,001,World,0 -outlook.com,hotmail.{...},inbound,001,World,0.999929 -outlook.com,hotmail.{...},outbound,001,World,1 -outspot.be,teneo.be,inbound,001,World,0 -outspot.nl,teneo.be,inbound,001,World,0 -ovenmail.com,iaires.com,inbound,001,World,0 -overnightprints.com,chtah.net,inbound,001,World,0 -overstock.com,overstock.com,inbound,001,World,0 -ovh.net,ovh.net,inbound,001,World,0.207527 -ovuline.com,ovuline.com,inbound,001,World,1 -oxfam.org.uk,msgfocus.com,inbound,001,World,0 -ozsale.com.au,ozsale.com.au,inbound,001,World,0.000192 -p-world.co.jp,p-world.co.jp,inbound,001,World,0 -pagoda.com,zales.com,inbound,001,World,0 -pagseguro.com.br,uol.com.br,inbound,001,World,0 -pair.com,pair.com,inbound,001,World,0.880779 -palmscasinoresort.com,palmscasinoresort.com,inbound,001,World,0 -pampers.com,bfi0.com,inbound,001,World,0 -panasonic.jp,panasonic.jp,inbound,001,World,0 -pandaresearch.com,pandaresearch.com,inbound,001,World,0 -pandora.com,pandora.com,inbound,001,World,1 -pandora.net,pandora.net,inbound,001,World,0.999666 -panelplace.com,smtp.com,inbound,001,World,0 -panerabreadnews.com,panerabreadnews.com,inbound,001,World,0 -pantaloondirect.net,iaires.com,inbound,001,World,0 -papajohns-specials.com,papajohns-specials.com,inbound,001,World,0 -paradisepublishers.com,paradisepublishers.com,inbound,001,World,0.999626 -parents.com,meredith.com,inbound,001,World,0 -parkmobileglobal.com,parkmobile.us,inbound,001,World,0 -path.com,path.com,inbound,001,World,1 -patriotupdate.com,inboxfirst.com,inbound,001,World,0 -payback.de,artegic.net,inbound,001,World,0.999986 -payback.in,chtah.net,inbound,001,World,0 -payback.in,eccluster.com,inbound,001,World,0 -payback.in,ecm-cluster.com,inbound,001,World,0 -payback.in,ecmcluster.com,inbound,001,World,0 -paypal.co.uk,paypal.com,inbound,001,World,1 -paypal.com,paypal.com,inbound,001,World,0.608856 -paypal.com.au,paypal.com,inbound,001,World,1 -paypal.de,paypal.com,inbound,001,World,1 -paytm.com,paytm.com,inbound,001,World,1 -pbteen.com,pbteen.com,inbound,001,World,0 -pccomponentes.com,pccomponentes.com,inbound,001,World,0 -pch.com,ed10.com,inbound,001,World,0 -pchfrontpage.com,ed10.com,inbound,001,World,0 -pchlotto.com,ed10.com,inbound,001,World,0 -pchplayandwin.com,ed10.com,inbound,001,World,0 -pchsearch.com,ed10.com,inbound,001,World,0 -pcmag.com,ittoolbox.com,inbound,001,World,0 -pcworld.com,pcworld.com,inbound,001,World,0 -pd25.com,pd25.com,inbound,001,World,1 -pd25.com,pd27.com,inbound,001,World,1 -peanuthome.info,adopterc.info,inbound,001,World,0 -peanuthome.info,aguitytr.info,inbound,001,World,0 -peanuthome.info,bevest.info,inbound,001,World,0 -peanuthome.info,bluester.info,inbound,001,World,0 -peanuthome.info,burror.info,inbound,001,World,0 -peanuthome.info,bursion.info,inbound,001,World,0 -peanuthome.info,cantexi.info,inbound,001,World,0 -peanuthome.info,caserhi.info,inbound,001,World,0 -peanuthome.info,celect.info,inbound,001,World,0 -peanuthome.info,chintone.info,inbound,001,World,0 -peanuthome.info,citery.info,inbound,001,World,0 -peanuthome.info,cleathal.info,inbound,001,World,0 -peanuthome.info,coherentrequittal.info,inbound,001,World,0 -peanuthome.info,colicom.info,inbound,001,World,0 -peanuthome.info,complec.info,inbound,001,World,0 -peanuthome.info,cyprinoidkaiserdom.info,inbound,001,World,0 -peanuthome.info,deciarc.info,inbound,001,World,0 -peanuthome.info,declaws.info,inbound,001,World,0 -peanuthome.info,dewest.info,inbound,001,World,0 -peanuthome.info,epconce.info,inbound,001,World,0 -peanuthome.info,fnotec.info,inbound,001,World,0 -peanuthome.info,folkswor.info,inbound,001,World,0 -peanuthome.info,forepert.info,inbound,001,World,0 -peanuthome.info,gurgaro.info,inbound,001,World,0 -peanuthome.info,heallyps.info,inbound,001,World,0 -peanuthome.info,holeph.info,inbound,001,World,0 -peanuthome.info,homewor.info,inbound,001,World,0 -peanuthome.info,hydroni.info,inbound,001,World,0 -peanuthome.info,ingenbu.info,inbound,001,World,0 -peanuthome.info,kinklybotaurus.info,inbound,001,World,0 -peanuthome.info,ninetiethwhiffet.info,inbound,001,World,0 -peanuthome.info,unglibshudder.info,inbound,001,World,0 -peanutwebmaster.info,adcrent.info,inbound,001,World,0 -peanutwebmaster.info,addmiel.info,inbound,001,World,0 -peanutwebmaster.info,agilhe.info,inbound,001,World,0 -peanutwebmaster.info,allegap.info,inbound,001,World,0 -peanutwebmaster.info,andvore.info,inbound,001,World,0 -peanutwebmaster.info,angogl.info,inbound,001,World,0 -peanutwebmaster.info,animass.info,inbound,001,World,0 -peanutwebmaster.info,arettery.info,inbound,001,World,0 -peanutwebmaster.info,aribank.info,inbound,001,World,0 -peanutwebmaster.info,arkefoc.info,inbound,001,World,0 -peanutwebmaster.info,avenog.info,inbound,001,World,0 -peanutwebmaster.info,barrave.info,inbound,001,World,0 -peanutwebmaster.info,bindowmo.info,inbound,001,World,0 -peanutwebmaster.info,bitravit.info,inbound,001,World,0 -peanutwebmaster.info,borsand.info,inbound,001,World,0 -peanutwebmaster.info,branti.info,inbound,001,World,0 -peanutwebmaster.info,breaserp.info,inbound,001,World,0 -peanutwebmaster.info,bredogly.info,inbound,001,World,0 -peanutwebmaster.info,briantra.info,inbound,001,World,0 -peanutwebmaster.info,bridea.info,inbound,001,World,0 -peanutwebmaster.info,carial.info,inbound,001,World,0 -peanutwebmaster.info,castac.info,inbound,001,World,0 -peanutwebmaster.info,chedoner.info,inbound,001,World,0 -peanutwebmaster.info,chiquent.info,inbound,001,World,0 -peanutwebmaster.info,cinchoi.info,inbound,001,World,0 -peanutwebmaster.info,cliate.info,inbound,001,World,0 -peanutwebmaster.info,cognn.info,inbound,001,World,0 -peanutwebsite.info,abjibbin.info,inbound,001,World,0 -peanutwebsite.info,audiette.info,inbound,001,World,0 -peanutwebsite.info,blancer.info,inbound,001,World,0 -peanutwebsite.info,bluester.info,inbound,001,World,0 -peanutwebsite.info,burror.info,inbound,001,World,0 -peanutwebsite.info,bursion.info,inbound,001,World,0 -peanutwebsite.info,caserhi.info,inbound,001,World,0 -peanutwebsite.info,celect.info,inbound,001,World,0 -peanutwebsite.info,cleathal.info,inbound,001,World,0 -peanutwebsite.info,coherentrequittal.info,inbound,001,World,0 -peanutwebsite.info,complec.info,inbound,001,World,0 -peanutwebsite.info,condost.info,inbound,001,World,0 -peanutwebsite.info,cyprinoidkaiserdom.info,inbound,001,World,0 -peanutwebsite.info,deciarc.info,inbound,001,World,0 -peanutwebsite.info,declaws.info,inbound,001,World,0 -peanutwebsite.info,epconce.info,inbound,001,World,0 -peanutwebsite.info,ferrayer.info,inbound,001,World,0 -peanutwebsite.info,fnotec.info,inbound,001,World,0 -peanutwebsite.info,folkswor.info,inbound,001,World,0 -peanutwebsite.info,forepert.info,inbound,001,World,0 -peanutwebsite.info,gurgaro.info,inbound,001,World,0 -peanutwebsite.info,holeph.info,inbound,001,World,0 -peanutwebsite.info,homewor.info,inbound,001,World,0 -peanutwebsite.info,hydroni.info,inbound,001,World,0 -peanutwebsite.info,ingenbu.info,inbound,001,World,0 -peanutwebsite.info,kinklybotaurus.info,inbound,001,World,0 -peanutwebsite.info,ninetiethwhiffet.info,inbound,001,World,0 -pearlsofwealth.com,pearlsofwealth.com,inbound,001,World,1 -peartreegreetings.com,rexcraft.com,inbound,001,World,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,001,World,0 -pennwell.com,pennwell.com,inbound,001,World,0 -pepabo.com,pepabo.com,inbound,001,World,0 -pepboys.com,pepboys.com,inbound,001,World,0 -peperoni.de,peperoni.de,inbound,001,World,1 -pepperfry.com,epidm.net,inbound,001,World,0 -perfectpriceindia.com,infimail.com,inbound,001,World,0 -perfectworld.com,perfectworld.com,inbound,001,World,0.000162 -perfora.net,perfora.net,inbound,001,World,0.920896 -permissionresearch.com,permissionresearch.com,inbound,001,World,0 -personalliberty.com,personalliberty.com,inbound,001,World,1 -personare.com.br,personare.com.br,inbound,001,World,0 -peytz.dk,peytz.dk,inbound,001,World,4e-06 -pga.com,pga.com,inbound,001,World,0 -pge.com,pge.com,inbound,001,World,0.149792 -pgeveryday.com,bfi0.com,inbound,001,World,0 -philosophy.com,philosophy.com,inbound,001,World,0 -phoenix.edu,phoenix.edu,inbound,001,World,0 -photobox.com,photobox.com,inbound,001,World,0 -photoprintit.com,photoprintit.com,inbound,001,World,0 -phpclasses.org,phpclasses.org,inbound,001,World,0 -phsmtpbox.com,phsmtpbox.com,inbound,001,World,0 -pia.jp,pia.jp,inbound,001,World,0 -pinger.com,pinger.com,inbound,001,World,0 -pinterest.com,pinterest.com,inbound,001,World,1 -pivotaltracker.com,pivotaltracker.com,inbound,001,World,1 -pixable.com,pixable.com,inbound,001,World,1 -pixum.com,pixum.com,inbound,001,World,0 -pizzahut.com,quikorder.com,inbound,001,World,0 -pizzahutoffers.com,pizzahutoffers.com,inbound,001,World,0 -placedestendances.com,placedestendances.com,inbound,001,World,0 -plaisio.gr,fagms.de,inbound,001,World,0 -planeo.com,planeo.com,inbound,001,World,0 -planeo.pt,planeo.pt,inbound,001,World,0 -playstation.com,playstation.com,inbound,001,World,0 -playstationmail.net,playstationmail.net,inbound,001,World,0 -playtika.com,emv8.com,inbound,001,World,0 -plexapp.com,plex.tv,inbound,001,World,1 -plumdistrict.com,plumdistrict.com,inbound,001,World,1 -pmailus.com,patrontechnology.com,inbound,001,World,0 -pnc.com,messagelabs.com,inbound,001,World,0.997194 -pnetweb.co.za,hosting.co.za,inbound,001,World,0 -pnetweb.co.za,salesnet.co.za,inbound,001,World,0 -pobox.com,pobox.com,inbound,001,World,0.975372 -pof.com,plentyoffish.co.uk,inbound,001,World,0 -pogo.com,pogo.com,inbound,001,World,0 -pointtown.com,gmo-media.jp,inbound,001,World,0 -poinx.com,poinx.com,inbound,001,World,0 -pokerstars.com,pokerstars.eu,inbound,001,World,0 -pokerstars.eu,pokerstars.eu,inbound,001,World,0 -pokupon.by,mailersend.com,inbound,001,World,0 -pokupon.ua,mailersend.com,inbound,001,World,0 -politicoemail.com,politicoemail.com,inbound,001,World,0 -polyvore.com,polyvore.com,inbound,001,World,1 -pontofrio.com.br,emv8.com,inbound,001,World,0 -popsugar.com,popsugar.com,inbound,001,World,0 -postcardfromhell.com,cyberthugs.com,inbound,001,World,1 -postgresql.org,postgresql.org,inbound,001,World,0.996805 -potterybarn.com,potterybarn.com,inbound,001,World,0 -potterybarnkids.com,potterybarnkids.com,inbound,001,World,0 -praca.pl,praca.pl,inbound,001,World,1 -pracuj.pl,pracuj.pl,inbound,001,World,0 -preferredpetclub.com,preferredpetclub.com,inbound,001,World,0 -presslaff.net,dat-e-baseonline.com,inbound,001,World,0 -pressmartmail.com,pressmartmail.com,inbound,001,World,0 -priceline.com,priceline.com,inbound,001,World,1 -princess.com,princess.com,inbound,001,World,0 -printvenue.com,fagms.de,inbound,001,World,0 -priorityoneemail.com,priorityoneemail.com,inbound,001,World,0 -privalia.com,privalia.com,inbound,001,World,3.94913202027326e-07 -private-elist.com,private-elist.com,inbound,001,World,0 -privoscite.si,privoscite.si,inbound,001,World,0 -profitcenteronline.com,groupdealtools.com,inbound,001,World,1 -progressive.com,progressive.com,inbound,001,World,0.897291 -progressiveagent.com,progressive.com,inbound,001,World,1 -promedmail.org,childrenshospital.org,inbound,001,World,1 -promod-news.fr,promod-news.com,inbound,001,World,0 -propertysolutions.com,propertysolutions.com,inbound,001,World,1 -prospectgeysercoop.com,prospectgeysercoop.com,inbound,001,World,1 -protopmail.com,protopmail.com,inbound,001,World,0 -providesupport.com,providesupport.com,inbound,001,World,0.000103 -proxyvote.com,adp-ics.com,inbound,001,World,0.908635 -psu.edu,psu.edu,inbound,001,World,0.073843 -pttplc.com,pttgrp.com,inbound,001,World,0 -publicators.com,publicators.com,inbound,001,World,0 -publix.com,publix.com,inbound,001,World,0.002567 -pucp.edu.pe,pucp.edu.pe,inbound,001,World,0.225541 -puffinmailer.com,zoothost.com,inbound,001,World,0 -pur3.net,pur3.net,inbound,001,World,0.001111 -purewow.com,purewow.com,inbound,001,World,0 -puritan.com,email-nbtyinc.com,inbound,001,World,0 -purlsmail.com,purlsmail.com,inbound,001,World,0 -pxsmail.com,pxsmail.com,inbound,001,World,0 -python.org,python.org,inbound,001,World,1 -q.com,synacor.com,inbound,001,World,0.999729 -qemailserver.com,qemailserver.com,inbound,001,World,0 -qoinpro.com,qoinpro.com,inbound,001,World,1 -qoo10.jp,qoo10.jp,inbound,001,World,2e-06 -qoo10.sg,qoo10.co.id,inbound,001,World,1.9e-05 -qoo10.sg,qoo10.com,inbound,001,World,0 -qoo10.sg,qoo10.my,inbound,001,World,2.2e-05 -qoo10.sg,qoo10.sg,inbound,001,World,8e-06 -qq.com,qq.com,inbound,001,World,0.999978 -qq.com,qq.com,outbound,001,World,1 -qtropnews.com,qtropnews.com,inbound,001,World,1.7e-05 -qualicorp.com.br,qualicorp.com.br,inbound,001,World,1 -quality.net.ua,quality.net.ua,inbound,001,World,0 -qualitysafelist.com,zoothost.com,inbound,001,World,0 -queopinas.com,confirmit.com,inbound,001,World,1 -quickbooks.com,intuit.com,inbound,001,World,0.99908 -quickrewards.net,quickrewards.net,inbound,001,World,0 -quikr.com,quikr.com,inbound,001,World,0.026599 -quinstreet.com,neoquin.com,inbound,001,World,0 -quora.com,quora.com,inbound,001,World,1 -quoramail.com,quoramail.com,inbound,001,World,1 -qvcemail.com,qvcemail.com,inbound,001,World,0 -r-project.org,ethz.ch,inbound,001,World,0.99999 -r51.it,musvc.com,inbound,001,World,0.092498 -r52.it,musvc.com,inbound,001,World,0.000221 -r57.it,musvc.com,inbound,001,World,0.139425 -r67.it,musvc.com,inbound,001,World,0.199845 -r70.it,musvc.com,inbound,001,World,0.311983 -rabota.ua,rabota.ua,inbound,001,World,0 -rackroom-email.com,rackroom-email.com,inbound,001,World,0 -radarsystems.net,radarsystems.net,inbound,001,World,1 -radiantretailapps.com,radiantretailapps.com,inbound,001,World,0 -radioshack.com,radioshack.com,inbound,001,World,0 -railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,001,World,0 -rakuten.co.jp,rakuten.co.jp,inbound,001,World,0 -rakuten.co.jp,shareee.jp,inbound,001,World,0 -rakuten.co.jp,yahoo.co.jp,inbound,001,World,0 -rakuten.com,rakuten.com,inbound,001,World,0 -rakuten.ne.jp,rakuten.co.jp,inbound,001,World,0 -rambler.ru,rambler.ru,inbound,001,World,0.08173 -randomhouse.com,randomhouse.com,inbound,001,World,0 -rapattoni.com,rapmls.com,inbound,001,World,0 -ratedpeople.com,ratedpeople.com,inbound,001,World,0.000979 -rax.ru,rax.ru,inbound,001,World,0.000258 -razerzone.com,chtah.net,inbound,001,World,0 -rbc.com,rbc.com,inbound,001,World,0.001897 -rdio.com,rdio.com,inbound,001,World,1 -reactiveadz.com,downlinebuilderdirect.com,inbound,001,World,0 -realage-mail.com,postdirect.com,inbound,001,World,0 -realestate.com.au,realestate.com.au,inbound,001,World,0 -realtor.org,realtor.org,inbound,001,World,0 -realtytrac.com,realtytrac.com,inbound,001,World,0.001681 -realus.co.jp,realus.co.jp,inbound,001,World,0 -recipe.com,meredith.com,inbound,001,World,0 -recochoku.jp,recochoku.jp,inbound,001,World,0 -recruit.net,recruit.net,inbound,001,World,0 -redbox.com,exacttarget.com,inbound,001,World,0 -redbox.com,redbox.com,inbound,001,World,4e-06 -redcross.org.uk,redcross.org.uk,inbound,001,World,0 -redfin.com,redfin.com,inbound,001,World,0 -rediffmail.com,akadns.net,outbound,001,World,0 -rediffmail.com,rediffmail.com,inbound,001,World,0 -redtri.com,redtri.com,inbound,001,World,0 -reebokusnews.com,reebokusnews.com,inbound,001,World,0 -reebonz.com,ed10.com,inbound,001,World,0 -reebonz.com,reebonz.com,inbound,001,World,0.857615 -reed.co.uk,reed.co.uk,inbound,001,World,0 -regie11.net,odiso.net,inbound,001,World,0 -regionalhelpwanted.com,regionalhelpwanted.com,inbound,001,World,1 -registrar-servers.com,registrar-servers.com,inbound,001,World,0.053619 -registro.br,registro.br,inbound,001,World,0 -relax7.hu,gruppi.hu,inbound,001,World,0 -relianceada.com,relianceada.com,inbound,001,World,0.276515 -rent.com,rent.com,inbound,001,World,0.000397 -rentalcars.com,rentalcars.com,inbound,001,World,1 -renweb.com,renweb.com,inbound,001,World,0 -repica.jp,kakaku.com,inbound,001,World,0 -repica.jp,repica.jp,inbound,001,World,0 -republicwireless.com,republicwireless.com,inbound,001,World,0.033564 -research-panel.jp,research-panel.jp,inbound,001,World,0 -researchgate.net,researchgate.net,inbound,001,World,0 -responder.co.il,responder.co.il,inbound,001,World,1 -restorationhardware.com,restorationhardware.com,inbound,001,World,0 -retailjobinsider.com,retailjobinsider.com,inbound,001,World,0 -retailmenot.com,retailmenot.com,inbound,001,World,3.54930373308668e-07 -reverbnation.com,reverbnation.com,inbound,001,World,0 -revolutiongolf.com,revolutiongolf.com,inbound,001,World,0 -rewardme.in,bfi0.com,inbound,001,World,0 -reyrey.net,reyrey.net,inbound,001,World,0.946402 -ricardoeletro.com.br,allin.com.br,inbound,001,World,0 -richersoundsvip.com,ibwmail.com,inbound,001,World,0 -richyrichmailer.com,maddog-productions.info,inbound,001,World,1 -rightmove.com,rightmove.com,inbound,001,World,0 -rigzonemail.com,rigzonemail.com,inbound,001,World,0 -rikunabi.com,rikunabi.com,inbound,001,World,1e-05 -ringcentral.com,ringcentral.com,inbound,001,World,1 -ripleyperu.com.pe,icommarketing.com,inbound,001,World,0 -riseup.net,riseup.net,inbound,001,World,1 -rivamail.com,mailurja.com,inbound,001,World,0 -rmtr.de,rapidmail.de,inbound,001,World,0 -rnmk.com,rnmk.com,inbound,001,World,0 -roadrunner.com,rr.com,inbound,001,World,0.005479 -roamans.com,neolane.net,inbound,001,World,0 -rocketmail.com,yahoo.{...},inbound,001,World,1 -rocketmail.com,yahoodns.net,outbound,001,World,1 -rockpath.info,rockpath.info,inbound,001,World,1 -rockwellcollins.com,rockwellcollins.com,inbound,001,World,1 -rogers.com,yahoo.{...},inbound,001,World,1 -rogers.com,yahoodns.net,outbound,001,World,1 -rookiestewsemails.com,rookiestewsemails.com,inbound,001,World,0 -roulartamail.be,roulartamail.be,inbound,001,World,0 -royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,001,World,0 -rpinow.org,app-info.net,inbound,001,World,0 -rpo9usa.email,rpo9usa.email,inbound,001,World,0 -rr.com,rr.com,inbound,001,World,0.008342 -rr.com,rr.com,outbound,001,World,0 -rsgsv.net,postini.com,inbound,001,World,0.093513 -rsgsv.net,rsgsv.net,inbound,001,World,0 -rsvpsv.net,rsvpsv.net,inbound,001,World,0 -rsvpsv.net,send.esp.br,inbound,001,World,0 -rsys2.com,amfam.com,inbound,001,World,0 -rsys2.com,cheaptickets.com,inbound,001,World,0 -rsys2.com,dishnetworkmail.com,inbound,001,World,0 -rsys2.com,e-comms.net,inbound,001,World,0 -rsys2.com,eharmony.com,inbound,001,World,0 -rsys2.com,fathead.com,inbound,001,World,0 -rsys2.com,intuit.com,inbound,001,World,0 -rsys2.com,kmart.com,inbound,001,World,0 -rsys2.com,kohlernews.com,inbound,001,World,0 -rsys2.com,lego.com,inbound,001,World,0 -rsys2.com,lenovo.com,inbound,001,World,0 -rsys2.com,modcloth.com,inbound,001,World,0 -rsys2.com,moxieinteractive.com,inbound,001,World,0 -rsys2.com,orbitz.com,inbound,001,World,0 -rsys2.com,payless.com,inbound,001,World,0 -rsys2.com,petsathome.com,inbound,001,World,0 -rsys2.com,quizzle.com,inbound,001,World,0 -rsys2.com,robeez.com,inbound,001,World,0 -rsys2.com,rsys1.com,inbound,001,World,0 -rsys2.com,rsys2.com,inbound,001,World,0 -rsys2.com,rsys3.com,inbound,001,World,0 -rsys2.com,rsys4.com,inbound,001,World,0 -rsys2.com,saucony.com,inbound,001,World,0 -rsys2.com,sears.com,inbound,001,World,0 -rsys2.com,shopbop.com,inbound,001,World,0 -rsys2.com,southwest.com,inbound,001,World,0 -rsys2.com,speeddatemail.com,inbound,001,World,0 -rsys2.com,thecompanystore.com,inbound,001,World,0 -rsys2.com,theknot.com,inbound,001,World,0 -rsys5.com,alibris.com,inbound,001,World,0 -rsys5.com,allstate-email.com,inbound,001,World,0 -rsys5.com,beachmint.com,inbound,001,World,0 -rsys5.com,belleandclive.com,inbound,001,World,0 -rsys5.com,br.dk,inbound,001,World,0 -rsys5.com,charlotterusse.com,inbound,001,World,0 -rsys5.com,comixology.com,inbound,001,World,0 -rsys5.com,cottonon.com,inbound,001,World,0 -rsys5.com,ediblearrangements.com,inbound,001,World,0 -rsys5.com,emailworldmarket.com,inbound,001,World,0 -rsys5.com,farfetch.com,inbound,001,World,0 -rsys5.com,frhiemailcommunications.com,inbound,001,World,0 -rsys5.com,harryanddavid.com,inbound,001,World,0 -rsys5.com,hollandandbarrett.com,inbound,001,World,0 -rsys5.com,icing.com,inbound,001,World,0 -rsys5.com,indigo.ca,inbound,001,World,0 -rsys5.com,jabong.com,inbound,001,World,0 -rsys5.com,jcrew.com,inbound,001,World,0 -rsys5.com,jjill.com,inbound,001,World,0 -rsys5.com,kanui.com.br,inbound,001,World,0 -rsys5.com,kirklands.com,inbound,001,World,0 -rsys5.com,lanebryant.com,inbound,001,World,0 -rsys5.com,lazada.com,inbound,001,World,0 -rsys5.com,leapfrog.com,inbound,001,World,0 -rsys5.com,llbean.com,inbound,001,World,0 -rsys5.com,lojascolombo.com.br,inbound,001,World,0 -rsys5.com,madewell.com,inbound,001,World,0 -rsys5.com,magazineluiza.com.br,inbound,001,World,0 -rsys5.com,missguided.co.uk,inbound,001,World,0 -rsys5.com,moma.org,inbound,001,World,0 -rsys5.com,nationalgeographic.com,inbound,001,World,0 -rsys5.com,neat.com,inbound,001,World,0 -rsys5.com,newbalance.com,inbound,001,World,0 -rsys5.com,news-voeazul.com.br,inbound,001,World,0 -rsys5.com,nordstrom.com,inbound,001,World,0 -rsys5.com,normthompson.com,inbound,001,World,0 -rsys5.com,novomundo.com.br,inbound,001,World,0 -rsys5.com,ourdeal.com.au,inbound,001,World,0 -rsys5.com,pier1.com,inbound,001,World,0 -rsys5.com,postini.com,inbound,001,World,0.0697 -rsys5.com,productmadness.com,inbound,001,World,0 -rsys5.com,rainbowshops.com,inbound,001,World,0 -rsys5.com,rei.com,inbound,001,World,0 -rsys5.com,roadrunnersports.com,inbound,001,World,0 -rsys5.com,rosettastone.com,inbound,001,World,0 -rsys5.com,seamless.com,inbound,001,World,0 -rsys5.com,serenaandlily.com,inbound,001,World,0 -rsys5.com,smiles.com.br,inbound,001,World,0 -rsys5.com,soubarato.com.br,inbound,001,World,0 -rsys5.com,strava.com,inbound,001,World,0 -rsys5.com,submarino.com.br,inbound,001,World,0 -rsys5.com,thewalkingcompany.com,inbound,001,World,0 -rsys5.com,tigerdirect.com,inbound,001,World,0 -rsys5.com,udemy.com,inbound,001,World,0 -rsys5.com,vitaminshoppe.com,inbound,001,World,0 -rsys5.com,vpusa.com,inbound,001,World,0 -rsys5.com,walmart.com.br,inbound,001,World,0 -rsys5.com,worldofwatches.com,inbound,001,World,0 -rsys5.com,xfinity.com,inbound,001,World,0 -rue21email.com,rue21email.com,inbound,001,World,0 -rueducommerce.com,groupe-rueducommerce.fr,inbound,001,World,0 -rummycirclemails.com,eccluster.com,inbound,001,World,0 -runkeeper.com,runkeeper.com,inbound,001,World,0 -runnet.jp,runnet.jp,inbound,001,World,0 -runtastic.com,runtastic.com,inbound,001,World,0 -rutenmail.com.tw,rutenmail.com.tw,inbound,001,World,0 -ruum.com,ruum.com,inbound,001,World,0 -ryanairmail.com,ryanairmail.com,inbound,001,World,0 -rzone.de,rzone.de,inbound,001,World,1 -s3s-br1.net,splio.com.br,inbound,001,World,0.908187 -s3s-main.net,splio.com,inbound,001,World,0.970428 -s4s-pl1.pl,splio.com,inbound,001,World,0.939032 -saavn.com,saavn.com,inbound,001,World,1 -safe-sender.net,safe-sender.net,inbound,001,World,0 -safelistextreme.com,quantumsafelist.com,inbound,001,World,0 -safelistpro.com,safelistpro.com,inbound,001,World,0.00014 -safeway.com,chtah.com,inbound,001,World,0 -safeway.com,safeway.com,inbound,001,World,0.000382 -sahibinden.com,sahibinden.com,inbound,001,World,0 -sailthru.com,sailthru.com,inbound,001,World,0 -saimails.in,infimail.com,inbound,001,World,0 -sainsburys.co.uk,emv5.com,inbound,001,World,0 -saisoncard.co.jp,saisoncard.co.jp,inbound,001,World,0 -saks.com,saks.com,inbound,001,World,0 -saksoff5th.com,saksoff5th.com,inbound,001,World,0 -sakura.ne.jp,sakura.ne.jp,inbound,001,World,0.640465 -salememail.net,salememail.net,inbound,001,World,0 -salesforce.com,postini.com,inbound,001,World,0.866057 -salesforce.com,salesforce.com,inbound,001,World,0.983322 -salesforce.com,salesforce.com,outbound,001,World,1 -salesmanago.pl,salesmanago.pl,inbound,001,World,0 -salliemae.com,salliemae.com,inbound,001,World,1 -salsalabs.net,salsalabs.net,inbound,001,World,0 -samashmusic.com,wc09.net,inbound,001,World,0 -samsclub.com,m0.net,inbound,001,World,0 -samsung.com,samsung.com,inbound,001,World,0.043941 -samsung.ru,samsung.ru,inbound,001,World,0 -samsungusa.com,samsungusa.com,inbound,001,World,0 -sanmina-sci.com,postini.com,inbound,001,World,0.999991 -sanmina.com,postini.com,inbound,001,World,0.9996 -sans.org,sans.org,inbound,001,World,0.497629 -santander.cl,santander.cl,inbound,001,World,0.999992 -santander.cl,santandersantiago.cl,inbound,001,World,1 -sapnetworkmail.com,sap-ag.de,inbound,001,World,1 -sapo.pt,sapo.pt,inbound,001,World,0.242839 -sapo.pt,sapo.pt,outbound,001,World,0 -saramin.co.kr,saramin.co.kr,inbound,001,World,0 -sassieshop.com,sassieshop.com,inbound,001,World,0.025887 -saturday.com,saturday.com,inbound,001,World,0 -savelivefresh.com,livesavemail.com,inbound,001,World,1 -savingdeals.in,infimail.com,inbound,001,World,0 -savingstar.com,savingstar.com,inbound,001,World,1 -sbcglobal.net,yahoo.{...},inbound,001,World,0.999991 -sbcglobal.net,yahoodns.net,outbound,001,World,1 -sbi.co.in,sbi.co.in,inbound,001,World,0 -sbr-inc.co.jp,hdemail.jp,inbound,001,World,0.000528 -sc.com,messagelabs.com,inbound,001,World,0.996156 -sc.com,sc.com,inbound,001,World,0.99683 -schwab.com,schwab.com,inbound,001,World,0.025055 -scmp.com,emarsys.net,inbound,001,World,0 -scoop.it,scoop.it,inbound,001,World,0 -scoopon.com.au,inxserver.de,inbound,001,World,1 -screwfix.info,fwdto.net,inbound,001,World,0 -sears.ca,sears.ca,inbound,001,World,0.005447 -searscard.com,searscard.com,inbound,001,World,1 -seaworld.com,seaworld.com,inbound,001,World,0 -secretescapes.com,secretescapes.com,inbound,001,World,0 -secure.ne.jp,secure.ne.jp,inbound,001,World,0.000356 -securence.com,securence.com,inbound,001,World,0.667957 -secureserver.net,secureserver.net,inbound,001,World,3.6e-05 -seek.com.au,seek.com.au,inbound,001,World,0 -seekingalpha.com,seekingalpha.com,inbound,001,World,1 -seekingalpha.com,sendgrid.net,inbound,001,World,1 -selectacast.net,selectacast.net,inbound,001,World,0.000561 -selection-priceminister.com,selection-priceminister.com,inbound,001,World,0 -semana.com,semana.com,inbound,001,World,0.005867 -senate.gov,senate.gov,inbound,001,World,0.992994 -sendearnings.com,sendearnings.com,inbound,001,World,0 -sender.lt,sritis.lt,inbound,001,World,0.000352 -sendgrid.info,sendgrid.net,inbound,001,World,0.999966 -sendgrid.me,sendgrid.net,inbound,001,World,1 -sendlane.com,sendlane.com,inbound,001,World,0 -sendpal.in,sendpal.in,inbound,001,World,0 -sendsmaily.info,sendsmaily.info,inbound,001,World,0 -seniorplanet.fr,seniorplanet.fr,inbound,001,World,0 -serpadres.es,chtah.net,inbound,001,World,0 -service-now.com,postini.com,inbound,001,World,0.988878 -service-now.com,service-now.com,inbound,001,World,0.998853 -serviciobancomer.com,serviciobancomer.com,inbound,001,World,0 -seznam.cz,seznam.cz,inbound,001,World,0.001781 -seznam.cz,seznam.cz,outbound,001,World,0.001741 -sfid01.com,sfid01.com,inbound,001,World,0 -sfimg.com,sfimarketing.com,inbound,001,World,0 -sfly.com,shutterfly.com,inbound,001,World,0 -sfr.fr,sfr.fr,inbound,001,World,0.236539 -sfr.fr,sfr.fr,outbound,001,World,0.004753 -shaadi.com,shaadi.com,inbound,001,World,0.000154 -shadowshopper.com,shadowshopper.com,inbound,001,World,5.6e-05 -shaw.ca,shaw.ca,inbound,001,World,0 -shaw.ca,shaw.ca,outbound,001,World,1 -sheplers.com,sheplers.com,inbound,001,World,0 -shiftplanning.com,shiftplanning.com,inbound,001,World,0 -shiksha.com,shiksha.com,inbound,001,World,0 -shinseibank.com,shinseibank.com,inbound,001,World,0 -shoedazzle.com,shoedazzle.com,inbound,001,World,0 -shoes.com,famousfootwear.com,inbound,001,World,0 -shop2gether.com.br,shop2gether.com.br,inbound,001,World,0 -shopbonton.com,shopbonton.com,inbound,001,World,0 -shopcluesemail.com,shopcluesemail.com,inbound,001,World,0 -shopcluesmail.com,shopcluesmail.com,inbound,001,World,0 -shophq.com,shophq.com,inbound,001,World,0 -shopjustice.com,shopjustice.com,inbound,001,World,0 -shopkick.com,shopkick.com,inbound,001,World,0 -shopko.com,shopko.com,inbound,001,World,0 -shopnineteenmails.in,iaires.com,inbound,001,World,0 -shoppersoptimum.ca,thindata.net,inbound,001,World,0 -shoppersstop.com,shoppersstop.com,inbound,001,World,0 -shoprite-email.com,email-mywebgrocer.com,inbound,001,World,0 -shoptime.com,shoptime.com,inbound,001,World,0 -shopto.net,shopto.net,inbound,001,World,1 -showingtime.com,showingtime.com,inbound,001,World,0 -showroomprive.com,showroomprive.be,inbound,001,World,0 -showroomprive.com,showroomprive.com,inbound,001,World,0.007102 -showroomprive.com,showroomprive.nl,inbound,001,World,0 -showroomprive.es,showroomprive.es,inbound,001,World,0 -showroomprive.it,showroomprive.pt,inbound,001,World,0 -showroomprive.pt,showroomprive.co.uk,inbound,001,World,0 -shtyle.fm,shtyle.fm,inbound,001,World,0 -shukatsu.jp,shukatsu.jp,inbound,001,World,0 -siella.jp,siella.jp,inbound,001,World,0.000199 -sierratradingpost.com,sierratradingpost.com,inbound,001,World,0.000885 -sigmabeauty.com,lstrk.net,inbound,001,World,1 -sii.cl,sii.cl,inbound,001,World,0 -simplesafelist.com,adminforfree.com,inbound,001,World,1 -simpletextadz.com,web-hosting.com,inbound,001,World,1 -simplyhired.com,simplyhired.com,inbound,001,World,0 -simplymarry.com,tbsl.in,inbound,001,World,0 -singsale.com.sg,singsale.com.sg,inbound,001,World,0 -siriusxm.com,xmradio.com,inbound,001,World,0 -sitecore-mailer.com,sendlabs.com,inbound,001,World,0 -sittercity.com,sittercity.com,inbound,001,World,0 -sixflags.com,sixflags.com,inbound,001,World,0 -skillpages-mailer.com,dynect.net,inbound,001,World,0 -skillpages-mailer.com,sendlabs.com,inbound,001,World,0 -skillpages-mailer.com,skillpagesmail.com,inbound,001,World,0 -sky.com,sky.com,inbound,001,World,8.8e-05 -skymall.com,skymall.com,inbound,001,World,0.001135 -skynet.be,belgacom.be,inbound,001,World,0.005903 -skype.com,delivery.net,inbound,001,World,0 -skype.com,skype.com,inbound,001,World,0 -skyscanner.net,skyscanner.net,inbound,001,World,1 -sld.cu,sld.cu,inbound,001,World,1 -slickdeals.net,slickdeals.net,inbound,001,World,4.9e-05 -slidesharemail.com,newslettergrid.com,inbound,001,World,1 -slidesharemail.com,slideshare.net,inbound,001,World,1 -slidesharemail.com,slidesharemail.com,inbound,001,World,1 -smartbrief.com,smartbrief.com,inbound,001,World,0 -smartdraw.com,smartdraw.com,inbound,001,World,0 -smartertravel.com,smartertravelmedia.com,inbound,001,World,0.021388 -smartphoneexperts.com,mailgun.net,inbound,001,World,1 -smartresponder.ru,smartresponder.ru,inbound,001,World,1 -smp.ne.jp,smp.ne.jp,inbound,001,World,0 -snagajob-email.com,snagajob-email.com,inbound,001,World,0 -snapdeal.com,snapdeal.com,inbound,001,World,0 -snapdealmail.in,snapdealmail.in,inbound,001,World,0 -snaphire.com,snaphire.com,inbound,001,World,0 -snapretail.com,snapretail.com,inbound,001,World,1 -socialappsmail.com,socialappsmail.com,inbound,001,World,1 -socialsex.biz,infinitypersonals.com,inbound,001,World,0 -sofmap.com,sofmap.com,inbound,001,World,0.0092 -softbank.jp,softbank.jp,inbound,001,World,0 -softbank.jp,softbank.jp,outbound,001,World,0 -softbank.ne.jp,softbank.ne.jp,inbound,001,World,0 -softbank.ne.jp,softbank.ne.jp,outbound,001,World,0 -solesociety.com,bronto.com,inbound,001,World,0 -solosenders.com,megasenders.com,inbound,001,World,8.2e-05 -solosenders.com,traxweb.org,inbound,001,World,0 -solveerrors.com,infimail.com,inbound,001,World,0 -soma.com,soma.com,inbound,001,World,0 -someecards.com,someecards.com,inbound,001,World,0 -songkick.com,songkick.com,inbound,001,World,1 -sony-latin.com,sony-latin.com,inbound,001,World,0.01735 -sony.com,sony.com,inbound,001,World,0.020225 -sony.jp,sony.jp,inbound,001,World,0.000654 -sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,001,World,0 -sonyrewards.com,sonyrewards.com,inbound,001,World,0 -soundcloudmail.com,soundcloudmail.com,inbound,001,World,0.999996 -sourceforge.net,sourceforge.net,inbound,001,World,1 -sourcenext.info,sourcenext.info,inbound,001,World,0 -southwest.com,southwest.com,inbound,001,World,0 -sp.gov.br,sp.gov.br,inbound,001,World,0.485842 -spanishdict.com,spanishdict.com,inbound,001,World,0.002698 -spareroom.co.uk,spareroom.co.uk,inbound,001,World,1 -sparklist.com,sparklist.com,inbound,001,World,0 -sparkpeople.com,sparkpeople.com,inbound,001,World,0 -spartoo.com,spartoo.com,inbound,001,World,0 -spectersoft.com,spectersoft.com,inbound,001,World,0 -speedyrewards-email.com,speedyrewards-email.com,inbound,001,World,0 -spencersonline.com,spencersonline.com,inbound,001,World,0 -spiritairlines.com,ctd004.net,inbound,001,World,0 -spiritairlines.com,ctd005.net,inbound,001,World,0 -splitwise.com,splitwise.com,inbound,001,World,1 -sportlobster.com,sportlobster.com,inbound,001,World,1 -sportsdirect.com,sportsdirect.com,inbound,001,World,0 -sportsline.com,cbsig.net,inbound,001,World,1 -sportsmansguide.com,sportsmansguide.com,inbound,001,World,0.736903 -spotifymail.com,spotifymail.com,inbound,001,World,1 -sprint.com,m0.net,inbound,001,World,0 -sqlservercentral.com,sqlservercentral.com,inbound,001,World,0 -square-enix.com,messagelabs.com,inbound,001,World,0.004493 -squareup.com,squareup.com,inbound,001,World,0.986452 -ssgadm.com,ssg.com,inbound,001,World,0 -staffeazymailers.com,iaires.com,inbound,001,World,0 -stakemail.com,iaires.com,inbound,001,World,0 -stampmail.in,iaires.com,inbound,001,World,0 -standaard.be,vummail.be,inbound,001,World,0 -standardbank.co.za,standardbank.co.za,inbound,001,World,0 -stanford.edu,highwire.org,inbound,001,World,1 -stanford.edu,stanford.edu,inbound,001,World,0.93025 -stansberryresearch.com,stansberry-re.net,inbound,001,World,0 -stansberryresearch.com,stansberryresearch.com,inbound,001,World,0.001008 -staples-pt.com,1-hostingservice.com,inbound,001,World,0 -staples.co.uk,ncrwebhost.de,inbound,001,World,0 -staples.com,staples.com,inbound,001,World,0.00638 -starbucks.com,iphmx.com,inbound,001,World,0.999476 -starbucks.com,starbucks.com,inbound,001,World,0 -stardockcorporation.com,stardockcorporation.com,inbound,001,World,0 -stardockentertainment.info,stardockentertainment.info,inbound,001,World,0 -starsports.com,eccluster.com,inbound,001,World,0 -startribune.com,startribune.com,inbound,001,World,0.001728 -startwire.com,jobsreport.com,inbound,001,World,1 -startwire.com,startwire.com,inbound,001,World,1 -starwoodhotels.com,outlook.com,inbound,001,World,1 -state-of-the-art-mailer.com,futurebanners.net,inbound,001,World,0 -state.gov,state.gov,inbound,001,World,0 -statefarm.com,statefarm.com,inbound,001,World,1 -stayfriends.de,stayfriends.de,inbound,001,World,0 -steampowered.com,steampowered.com,inbound,001,World,1 -steinmart.com,steinmart.com,inbound,001,World,0 -stelladot.com,stelladot.com,inbound,001,World,0 -stepstone.de,stepstone.com,inbound,001,World,0.001689 -stevemadden.com,stevemadden.com,inbound,001,World,0 -stjobs.sg,st701.com,inbound,001,World,1 -stjude.org,stjude.org,inbound,001,World,0.006716 -strava.com,strava.com,inbound,001,World,1 -streetauthoritydaily.com,streetauthoritydaily.com,inbound,001,World,0 -streeteasy.com,streeteasy.com,inbound,001,World,0 -striata.com,striata.com,inbound,001,World,0 -stubhub.com,stubhub.com,inbound,001,World,1 -studentbeans.com,emv8.com,inbound,001,World,0 -stumblemail.com,stumblemail.com,inbound,001,World,1 -stylecareers.com,stylecareers.com,inbound,001,World,0 -suafaturanet.com.br,suafaturanet.com.br,inbound,001,World,0.995846 -subito.it,subito.it,inbound,001,World,0 -subscribe.ru,subscribe.ru,inbound,001,World,0 -subscribermail.com,subscribermail.com,inbound,001,World,0 -subtend.info,subtend.info,inbound,001,World,1 -subway.com,subway.com,inbound,001,World,0 -sungard.com,postini.com,inbound,001,World,0.499946 -sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,001,World,1 -superbalist.com,sailthru.com,inbound,001,World,0 -superdeal.com.ua,mailersend.com,inbound,001,World,0 -superdrug.com,superdrug.com,inbound,001,World,0 -superjob.ru,superjob.ru,inbound,001,World,0 -supersafemailer.com,zoothost.com,inbound,001,World,0 -support-love.com,support-love.com,inbound,001,World,0 -supremelist.com,onlinehome-server.com,inbound,001,World,1 -surfmandelivery.com,surfmandelivery.com,inbound,001,World,0 -surlatable.com,surlatable.com,inbound,001,World,0 -surveyhelpcenter.com,jsmtp.net,inbound,001,World,0 -surveyjobopportunities.com,surveyjobopportunities.com,inbound,001,World,3.7e-05 -surveymonkey.com,surveymonkey.com,inbound,001,World,0 -surveysavvy.com,surveysavvy.com,inbound,001,World,0 -surveyspot.com,ssisurveys.com,inbound,001,World,0 -sut1.co.uk,sut1.co.uk,inbound,001,World,0.001789 -sut5.co.uk,sut5.co.uk,inbound,001,World,0 -swanson-vitamins.com,emv5.com,inbound,001,World,0 -sweepstakesalerts.com,sweepstakesalerts.com,inbound,001,World,0 -swimoutlet.com,isport.com,inbound,001,World,0 -sylectus.com,sylectus.com,inbound,001,World,0.361297 -sympatico.ca,hotmail.{...},inbound,001,World,1 -sympatico.ca,hotmail.{...},outbound,001,World,1 -synchronyfinancial.com,bigfootinteractive.com,inbound,001,World,0 -synergy360.jp,crmstyle.com,inbound,001,World,0 -t-online.de,t-online.de,inbound,001,World,1 -t-online.de,t-online.de,outbound,001,World,0.999939 -tadtopmails.com,tadtopmails.com,inbound,001,World,0 -taggedmail.com,taggedmail.com,inbound,001,World,0 -taipeifubon.com.tw,taipeifubon.com.tw,inbound,001,World,0 -taishinbank.com.tw,taishinbank.com.tw,inbound,001,World,0 -take2games.com,take2games.com,inbound,001,World,0 -talkmatch.com,talkmatch.com,inbound,001,World,0 -tamu.edu,tamu.edu,inbound,001,World,0.249656 -tanga.com,tanga.com,inbound,001,World,0 -tangeroutletsusa.com,bronto.com,inbound,001,World,0 -tanningmail.com,tanningmail.com,inbound,001,World,0 -tappingsolutionemail.com,tappingsolutionemail.com,inbound,001,World,0 -target-safelist.com,safelistpro.com,inbound,001,World,0.000215 -target.com,bigfootinteractive.com,inbound,001,World,0 -targetproblaster.com,targetproblaster.com,inbound,001,World,0 -targetx.com,targetx.com,inbound,001,World,0 -tarot.com,tarot.com,inbound,001,World,0 -tastefullysimpleparty.com,bigfootinteractive.com,inbound,001,World,0 -tasteofhome.com,tasteofhome.com,inbound,001,World,0 -tastingtable.com,tastingtable.com,inbound,001,World,0 -tatrabanka.sk,tatrabanka.sk,inbound,001,World,0 -taxi4sure.net,infimail.com,inbound,001,World,0 -tchibo.com.tr,euromsg.net,inbound,001,World,0 -tchibo.de,srv2.de,inbound,001,World,0.980445 -teach12.net,teach12.net,inbound,001,World,0 -teambuymail.com,teambuymail.com,inbound,001,World,0 -teamo.ru,teamo.ru,inbound,001,World,0 -teamsnap.com,teamsnap.com,inbound,001,World,1 -teamviewer.com,teamviewer.com,inbound,001,World,0 -teapartyinfo.org,teapartyinfo.org,inbound,001,World,0 -techgig.com,tbsl.in,inbound,001,World,0 -technolutions.net,technolutions.net,inbound,001,World,1 -techtarget.com,techtarget.com,inbound,001,World,0.001771 -telefonica.com,telefonica.com,inbound,001,World,0.998662 -telegraph.co.uk,telegraph.co.uk,inbound,001,World,0 -telenet.be,telenet-ops.be,inbound,001,World,0.000128 -teleportmyjob.com,clara.net,inbound,001,World,0 -telus.com,telus.com,inbound,001,World,0.000118 -telus.net,telus.net,inbound,001,World,0.006226 -templeandwebster.com.au,templeandwebster.com.au,inbound,001,World,5e-06 -ten24mail.com,ten24mail.com,inbound,001,World,0 -terra.com,terra.com,inbound,001,World,0.000463 -terra.com.br,terra.com,inbound,001,World,0 -terra.com.br,terra.com,outbound,001,World,0 -tesco.com,tesco.com,inbound,001,World,0 -testfunda.com,testfunda.com,inbound,001,World,0 -texasjobdepartment.com,texasjobdepartment.com,inbound,001,World,0 -textnow.me,textnow.me,inbound,001,World,0 -tgw.com,tgw.com,inbound,001,World,0 -theanimalrescuesite.com,theanimalrescuesite.com,inbound,001,World,0 -theatermania.com,wc09.net,inbound,001,World,0 -thebay.com,thebay.com,inbound,001,World,0 -thebodyshop-usa.com,email-bodyshop.com,inbound,001,World,0 -thebodyshop-usa.com,postdirect.com,inbound,001,World,0 -thecarousell.com,thecarousell.com,inbound,001,World,1 -thegrommet.com,lstrk.net,inbound,001,World,1 -theguardian.com,theguardian.com,inbound,001,World,0 -thehut.com,thehut.com,inbound,001,World,0 -theleadmagnet.com,your-server.de,inbound,001,World,1 -thelimited.com,thelimited.com,inbound,001,World,0 -themailbagsafelist.com,thomas-j-brown.com,inbound,001,World,0 -theoutnet.com,theoutnet.com,inbound,001,World,0 -thepamperedchef.com,thepamperedchef.com,inbound,001,World,0 -thephonehouse.es,splio.com,inbound,001,World,0.983578 -thephonehouse.es,splio.es,inbound,001,World,0.983266 -therealreal.com,email-realreal.com,inbound,001,World,0 -theskimm.com,theskimm.com,inbound,001,World,0 -thesource.ca,thesource.ca,inbound,001,World,0.00923 -thesovereigninvestor.com,sovereignsociety.com,inbound,001,World,0 -thewarehouse.co.nz,thewarehouse.co.nz,inbound,001,World,0 -thinkgeek.com,thinkgeek.com,inbound,001,World,1e-06 -thinkvidya.com,thinkvidya.com,inbound,001,World,0 -thirtyonegifts.com,thirtyonegifts.com,inbound,001,World,0 -thomascook.com,eccluster.com,inbound,001,World,0 -thoughtful-mind.com,thoughtful-mind.com,inbound,001,World,0 -thumbtack.com,thumbtack.com,inbound,001,World,1 -ticketmaster.com,ticketmaster.com,inbound,001,World,0.251337 -ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,001,World,0 -ticketmonster.co.kr,ticketmonster.co.kr,inbound,001,World,0 -timehop.com,timehop.com,inbound,001,World,1 -timeout.com,ec-cluster.com,inbound,001,World,0 -timesjobs.com,tbsl.in,inbound,001,World,0 -timesjobsmail.com,tbsl.in,inbound,001,World,0 -timesofindia.com,indiatimes.com,inbound,001,World,0 -timewarnercable.com,bigfootinteractive.com,inbound,001,World,0 -timeweb.ru,timeweb.ru,inbound,001,World,1 -tinyletterapp.com,tinyletterapp.com,inbound,001,World,0 -tiscali.it,tiscali.it,outbound,001,World,0 -tmart.com,chtah.net,inbound,001,World,0 -tmp.com,tmpw.com,inbound,001,World,0 -tobi.com,messagebus.com,inbound,001,World,0 -tobizaru.jp,tobizaru.jp,inbound,001,World,0 -tocoo.jp,aics.ne.jp,inbound,001,World,0 -toluna.com,toluna.com,inbound,001,World,0 -tomtommailer.com,tomtommailer.com,inbound,001,World,0 -topface.com,topface.com,inbound,001,World,1e-06 -topica.com,topica-silver-y.com,inbound,001,World,0 -topqpon.si,topqpon.si,inbound,001,World,0 -topspin.net,topspin.net,inbound,001,World,1 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,001,World,0 -touchbase2.com,infimail.com,inbound,001,World,0 -touchbase2.com,mailurja.com,inbound,001,World,0 -touchbasepro.com,touchbasepro.com,inbound,001,World,0 -tower.jp,tower.jp,inbound,001,World,0 -townhallmail.com,townhallmail.com,inbound,001,World,0 -townnews-mail.com,townnews-mail.com,inbound,001,World,0 -townsquaremedia.info,sailthru.com,inbound,001,World,0 -toysrus.com,epsl1.com,inbound,001,World,0 -trabajar.com,trabajo.org,inbound,001,World,0.999997 -trabalhar.com,trabajo.org,inbound,001,World,0.999994 -tradeloop.com,tradeloop.com,inbound,001,World,1 -trademe.co.nz,trademe.co.nz,inbound,001,World,0.991916 -trafficboostermailer.com,trafficboostermailer.com,inbound,001,World,1 -trafficleads2incomevm.com,zoothost.com,inbound,001,World,0 -trafficprolist.com,thomas-j-brown.com,inbound,001,World,0 -trafficwave.net,trafficwave.net,inbound,001,World,0 -transittraveljobinsider.com,transittraveljobinsider.com,inbound,001,World,0 -transportexchangegroup.com,transportexchangegroup.com,inbound,001,World,0.998703 -transversal.net,transversal.net,inbound,001,World,1 -travelchannel.com,travelchannel.com,inbound,001,World,0 -travelocity.com,travelocity.com,inbound,001,World,0.000194 -travelzoo.com,travelzoo.com,inbound,001,World,0 -travisperkins.co.uk,travisperkins.co.uk,inbound,001,World,0 -trclient.com,trclient.com,inbound,001,World,0 -treemall.com.tw,symphox.com,inbound,001,World,0 -trello.com,mandrillapp.com,inbound,001,World,1 -trialsmith.com,membercentral.com,inbound,001,World,0 -tricaemidia.com.br,tricaemidia.com.br,inbound,001,World,0 -triongames.com,triongames.com,inbound,001,World,0.085718 -tripadvisor.com,tripadvisor.com,inbound,001,World,0.000588 -tripit.com,tripit.com,inbound,001,World,0 -tripolis.com,tripolis.com,inbound,001,World,0 -trovit.com,trovit.com,inbound,001,World,0 -trulia.com,trulia.com,inbound,001,World,0 -tsite.jp,tsite.jp,inbound,001,World,0 -tsmmail.com,tsmmail.com,inbound,001,World,0 -tsutaya.co.jp,tsutaya.co.jp,inbound,001,World,0 -tucasa.com,grupodtm.com,inbound,001,World,0 -tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,001,World,0 -tumblr.com,tumblr.com,inbound,001,World,1 -turbine.com,turbine.com,inbound,001,World,0.013592 -turkcell.com.tr,turkcell.com.tr,inbound,001,World,0.043492 -turner.com,cnn.com,inbound,001,World,0 -twe-safelist.com,adminforfree.com,inbound,001,World,1 -twitch.tv,justin.tv,inbound,001,World,0 -twitter.com,postini.com,inbound,001,World,0.765163 -twitter.com,twitter.com,inbound,001,World,0.999969 -twoomail.com,netlogmail.com,inbound,001,World,0 -twoomail.com,twoomail.com,inbound,001,World,0 -type.jp,type.jp,inbound,001,World,0 -u-shopping.com.tw,u-shopping.com.tw,inbound,001,World,0 -uber.com,uber.com,inbound,001,World,1 -ubi.com,ubi.com,inbound,001,World,0 -ubivox.com,ubivox.com,inbound,001,World,0.956275 -ubuntu.com,canonical.com,inbound,001,World,0.000129 -ucla.edu,ucla.edu,inbound,001,World,0.806204 -udnpaper.com,udnpaper.com,inbound,001,World,0.998858 -udnshopping.com,udnshopping.com,inbound,001,World,0 -uga.edu,outlook.com,inbound,001,World,1 -uhaul.com,uhaul.com,inbound,001,World,0.997947 -uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,001,World,0 -uiuc.edu,illinois.edu,inbound,001,World,0.999815 -ukr.net,fwdcdn.com,inbound,001,World,1 -ukr.net,ukr.net,outbound,001,World,0.999992 -ulta.com,exacttarget.com,inbound,001,World,0 -ulta.com,ulta.com,inbound,001,World,0.034861 -ulteem.com,ulteem.com,inbound,001,World,0 -ultimateadsites.net,ultimateadsites.net,inbound,001,World,1 -umd.edu,umd.edu,inbound,001,World,0.021709 -umn.edu,umn.edu,inbound,001,World,0.966992 -umpiredigital.com,inboxmarketer.com,inbound,001,World,0 -unilever.com,mxlogic.net,inbound,001,World,1 -uniqlo-usa.com,uniqlo-usa.com,inbound,001,World,0 -united.com,coair.com,inbound,001,World,1 -united.com,united.com,inbound,001,World,0 -unitedrepublic.org,unitedrepublic.org,inbound,001,World,1 -universalorlando.com,universalorlando.com,inbound,001,World,0 -unosinsidersclub.com,unosinsidersclub.com,inbound,001,World,0 -unrollmail.com,unrollmail.com,inbound,001,World,1 -uol.com.br,uol.com.br,inbound,001,World,0 -uol.com.br,uol.com.br,outbound,001,World,0 -upenn.edu,upenn.edu,inbound,001,World,0.16521 -upromise.com,delivery.net,inbound,001,World,0 -ups.com,ups.com,inbound,001,World,0.997955 -urbanoutfitters.com,freepeople.com,inbound,001,World,0 -urx.com.br,urx.com.br,inbound,001,World,0 -usaa.com,usaa.com,inbound,001,World,0.639434 -usafisnews.org,usafisnews.org,inbound,001,World,0 -usahockey-email.com,usahockey-email.com,inbound,001,World,0 -usajobs.gov,opm.gov,inbound,001,World,0.931948 -usc.edu,usc.edu,inbound,001,World,0.300314 -uscourts.gov,uscourts.gov,inbound,001,World,0 -uslargestsafelist.com,zoothost.com,inbound,001,World,0 -usndr.com,unisender.com,inbound,001,World,1 -usndr.com,usndr.com,inbound,001,World,1 -usp.br,usp.br,inbound,001,World,0.044176 -usps.com,usps.gov,inbound,001,World,0.909588 -usps.gov,usps.gov,inbound,001,World,0.940315 -ustream.tv,mailgun.net,inbound,001,World,1 -ustream.tv,mailgun.us,inbound,001,World,1 -usx.com.br,uqx.com.br,inbound,001,World,0 -usx.com.br,usx.com.br,inbound,001,World,0 -usx.com.br,utx.com.br,inbound,001,World,0 -utfsm.cl,utfsm.cl,inbound,001,World,0.002318 -utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,001,World,0 -utx.com.br,utx.com.br,inbound,001,World,0 -uvarosa.com.br,uvarosa.com.br,inbound,001,World,0 -vacationstogo.com,vacationstogo.com,inbound,001,World,0 -vagas.com.br,vagas.com.br,inbound,001,World,0 -vakifbank.com.tr,vakifbank.com.tr,inbound,001,World,1 -valuedopinions.co.uk,researchnow-usa.com,inbound,001,World,1 -vanheusenrewards.com,vanheusenrewards.com,inbound,001,World,0 -vd.nl,emsecure.net,inbound,001,World,0 -velocityfrequentflyer.com,virginaustralia.com,inbound,001,World,0.003442 -venca.es,eccluster.com,inbound,001,World,0 -venere.com,kiwari.com,inbound,001,World,0 -vente-exclusive.com,vente-exclusive.com,inbound,001,World,0.000463 -venteprivee.com,venteprivee.com,inbound,001,World,0 -venusswimwear.net,venusswimwear.net,inbound,001,World,0 -verabradleymail.com,verabradleymail.com,inbound,001,World,0 -verizon.com,verizon.com,inbound,001,World,0.999793 -verizon.net,verizon.net,inbound,001,World,0.00713 -verizon.net,verizon.net,outbound,001,World,0 -verizon.net,yahoo.{...},inbound,001,World,0.999435 -verizonwireless.com,verizonwireless.com,inbound,001,World,0.972245 -vfoutletvip.com,vfoutletvip.com,inbound,001,World,0 -vhmnetworkemail.com,jobdiagnosis.com,inbound,001,World,0 -viagogo.com,chtah.net,inbound,001,World,0 -viajanet.com.br,viajanet.com.br,inbound,001,World,0.001345 -vicinity.nl,picsrv.net,inbound,001,World,0.022107 -victoriassecret.com,victoriassecret.com,inbound,001,World,0 -videotron.ca,videotron.ca,inbound,001,World,0.005886 -vietcombank.com.vn,vietcombank.com.vn,inbound,001,World,1 -vikingrivercruises.com,bfi0.com,inbound,001,World,0 -vimeo.com,vimeo.com,inbound,001,World,0 -vindale.com,vindale.com,inbound,001,World,0 -vipvoice.com,npdor.com,inbound,001,World,0 -viraladmagnet.com,viraladmagnet.com,inbound,001,World,1 -viralsender.com,viralsender.com,inbound,001,World,0 -virgilio.it,virgilio.net,inbound,001,World,0 -virginia.edu,virginia.edu,inbound,001,World,0.055153 -virtualtarget.com.br,virtualtarget.com.br,inbound,001,World,0 -vistaprint.com,vistaprint.com,inbound,001,World,0 -vistaprint.com.au,vistaprint.com.au,inbound,001,World,0 -visualsoft.co.uk,visualsoft.co.uk,inbound,001,World,0 -vitacost.com,vitacost.com,inbound,001,World,0 -vitaladviews.com,zoothost.com,inbound,001,World,0 -vitaminworld.com,email-nbtyinc.com,inbound,001,World,0 -vivastreet.com,viwii.net,inbound,001,World,0 -vk.com,vkontakte.ru,inbound,001,World,0 -vocus.com,vocus.com,inbound,001,World,0 -vodafone.com,vodafone.in,inbound,001,World,0.617518 -vonage.com,vonagenetworks.net,inbound,001,World,0.006531 -votervoice.net,votervoice.net,inbound,001,World,0 -vouchercloud.com,vouchercloud.com,inbound,001,World,0 -vovici.com,vovici.com,inbound,001,World,0 -voyageprive.com,cccampaigns.net,inbound,001,World,0 -voyageprive.es,ccmdcampaigns.net,inbound,001,World,0 -voyageprive.it,ccmdcampaigns.net,inbound,001,World,0 -voyages-sncf.com,neolane.net,inbound,001,World,0 -vpass.ne.jp,clickmailer.jp,inbound,001,World,0 -vpcontact.com,vpcontact.com,inbound,001,World,0 -vresp.com,verticalresponse.com,inbound,001,World,0 -vt.edu,vt.edu,inbound,001,World,0.978548 -vtext.com,vtext.com,inbound,001,World,0 -vtext.com,vtext.com,outbound,001,World,1 -vudu.com,vudu.com,inbound,001,World,0 -vueling.com,vueling.com,inbound,001,World,0 -vuezone.com,vuezone.com,inbound,001,World,0 -vzwpix.com,vtext.com,inbound,001,World,0 -wagjag.com,wagjag.com,inbound,001,World,0 -walgreens.com,walgreens.com,inbound,001,World,0.006128 -wallst.com,wallst.com,inbound,001,World,0 -wallstreetdaily.com,wallstreetdaily.com,inbound,001,World,0 -walmart.ca,walmart.ca,inbound,001,World,0 -walmart.com,walmart.com,inbound,001,World,0.315059 -wanadoo.fr,orange.fr,inbound,001,World,0 -wanadoo.fr,orange.fr,outbound,001,World,0 -warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,001,World,0 -waterstones.com,waterstones.com,inbound,001,World,0 -wattpad.com,wattpad.com,inbound,001,World,1 -waves-audio.com,emv8.com,inbound,001,World,0 -way2movies.net,way2movies.net,inbound,001,World,0 -way2sms.biz,way2sms.biz,inbound,001,World,0 -way2sms.in,way2sms.in,inbound,001,World,0 -way2smsemail.com,way2smsemail.com,inbound,001,World,0 -way2smsemails.com,way2smsemails.com,inbound,001,World,0 -way2smsmail.in,way2smsmail.in,inbound,001,World,0 -way2smsmails.com,way2smsmails.com,inbound,001,World,0 -wayfair.com,csnstores.com,inbound,001,World,0 -wealthyaffiliate.com,wealthyaffiliate.com,inbound,001,World,1 -web.de,web.de,inbound,001,World,0.999986 -web.de,web.de,outbound,001,World,1 -webcas.net,webcas.net,inbound,001,World,0 -webmd.com,webmd.com,inbound,001,World,2e-06 -webs.com,epsl1.com,inbound,001,World,0 -websaver.ca,websaver.ca,inbound,001,World,0 -websitesettings.com,stabletransit.com,inbound,001,World,0 -websitewelcome.com,websitewelcome.com,inbound,001,World,1 -webstars2k.com,webstars2k.com,inbound,001,World,1 -wechat.com,qq.com,inbound,001,World,1 -weebly.com,weeblymail.com,inbound,001,World,0 -wegottickets.com,wegottickets.com,inbound,001,World,0 -weheartit.com,weheartit.com,inbound,001,World,1 -wehkamp.nl,wehkamp.nl,inbound,001,World,0 -wellsfargo.com,wellsfargo.com,inbound,001,World,1.0 -wellsfargoadvisors.com,wellsfargo.com,inbound,001,World,1 -wemakeprice.com,wemakeprice.com,inbound,001,World,0 -westelm.com,westelm.com,inbound,001,World,0 -westmarine.com,westmarine.com,inbound,001,World,0 -westwing.com.br,cust-cluster.com,inbound,001,World,0 -westwing.es,ecm-cluster.com,inbound,001,World,0 -westwing.ru,ecm-cluster.com,inbound,001,World,0 -wetransfer.com,wetransfer.com,inbound,001,World,0.999751 -wetsealnewsletter.com,wetsealnewsletter.com,inbound,001,World,0 -wgbh.org,wgbh.org,inbound,001,World,0 -whaakky.com,whaakky.com,inbound,001,World,0 -whatcounts.com,wc09.net,inbound,001,World,0 -whentowork.com,whentowork.com,inbound,001,World,0 -whereareyounow.com,wayn.net,inbound,001,World,0 -whitehouse.gov,whitehouse.gov,inbound,001,World,0 -whitelabelpros.com,whitelabelpros.com,inbound,001,World,0 -wiggle.com,wiggle.com,inbound,001,World,0 -wikia.com,wikia.com,inbound,001,World,0.550228 -wikimedia.org,wikimedia.org,inbound,001,World,0 -william-reed.com,neolane.net,inbound,001,World,0 -williamhill.com,williamhill.com,inbound,001,World,0 -williams-sonoma.com,williams-sonoma.com,inbound,001,World,0 -windstream.net,windstream.net,inbound,001,World,0.002172 -wine.com,wine.com,inbound,001,World,0 -winkalmail.com,fnbox.com,inbound,001,World,0 -wisdomitservices.com,infimail.com,inbound,001,World,0 -wldemail-mailer.com,wldemail.com,inbound,001,World,0 -wldemail.com,emarsys.net,inbound,001,World,0 -wmtransfer.com,wmtransfer.com,inbound,001,World,0 -wnd.com,emv4.net,inbound,001,World,0 -wnd.com,worldnetdaily.com,inbound,001,World,0 -wolfmedia.us,wolfmedia.us,inbound,001,World,0 -womanwithin.com,womanwithin.com,inbound,001,World,0 -woodforest.com,woodforest.com,inbound,001,World,1 -wordfly.com,wordfly.com,inbound,001,World,0 -work.ua,work.ua,inbound,001,World,1 -workcircle.com,workcircle.net,inbound,001,World,0 -workhunter.net,workhunter.net,inbound,001,World,1 -workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,001,World,0 -worldsingles.co,worldsingles.com,inbound,001,World,0 -worldwinner.com,worldwinner.com,inbound,001,World,0.112191 -wotif.com,whatcounts.com,inbound,001,World,0 -wowcher.co.uk,wowcher.co.uk,inbound,001,World,0.000373 -wp.com,wordpress.com,inbound,001,World,0 -wp.pl,wp.pl,inbound,001,World,0.998027 -wp.pl,wp.pl,outbound,001,World,1 -wpengine.com,wpengine.com,inbound,001,World,1 -writers-community.com,writers-community.com,inbound,001,World,0 -writersstore.com,writersstore.com,inbound,001,World,0 -wsjemail.com,wsjemail.com,inbound,001,World,0 -wuaki.tv,chtah.net,inbound,001,World,0 -wustl.edu,app-info.net,inbound,001,World,0 -wwe.com,wwe.com,inbound,001,World,8e-06 -www.gov.tw,hinet.net,inbound,001,World,8e-05 -wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,001,World,0 -xbox.com,xbox.com,inbound,001,World,0 -xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,001,World,0 -xen.org,xen.org,inbound,001,World,1 -xing.com,xing.com,inbound,001,World,0 -xmailix.com,xmailix.com,inbound,001,World,0 -xmeeting.com,xmeeting.com,inbound,001,World,1 -xmr3.com,messagereach.com,inbound,001,World,0.999933 -xoom.com,xoom.com,inbound,001,World,0 -xpnews.com.br,xpnews.com.br,inbound,001,World,1 -xxxconnect.com,infinitypersonals.com,inbound,001,World,0 -yahoo-inc.com,yahoo.{...},inbound,001,World,0.999974 -yahoo.co.jp,yahoo.co.jp,inbound,001,World,1.5e-05 -yahoo.co.jp,yahoo.co.jp,outbound,001,World,0 -yahoo.{...},postini.com,inbound,001,World,0.767368 -yahoo.{...},yahoo.co.jp,inbound,001,World,0 -yahoo.{...},yahoo.{...},inbound,001,World,0.999416 -yahoo.{...},yahoodns.net,outbound,001,World,1 -yahoogroups.com,yahoodns.net,outbound,001,World,1 -yakala.co,euromsg.net,inbound,001,World,0 -yammer.com,yammer.com,inbound,001,World,1 -yandex.ru,yandex.net,inbound,001,World,0.999698 -yandex.ru,yandex.ru,outbound,001,World,1 -yapikredi.com.tr,yapikredi.com.tr,inbound,001,World,1 -yapstone.com,yapstone.com,inbound,001,World,0 -yelp.com,yelpcorp.com,inbound,001,World,0 -yesbank.in,yesbank.in,inbound,001,World,0.108676 -yipit.com,yipit.com,inbound,001,World,1 -ymail.com,yahoo.{...},inbound,001,World,1 -ymail.com,yahoodns.net,outbound,001,World,1 -ymlpserver.net,ymlpserver.net,inbound,001,World,0 -ymlpsrv.net,ymlpsrv.net,inbound,001,World,0 -yodobashi.com,yodobashi.com,inbound,001,World,0 -yoox.com,yoox.com,inbound,001,World,0 -youmail.com,youmail.com,inbound,001,World,0 -youravon.com,email-avonglobal.com,inbound,001,World,0 -youreletters3.com,equitymaster.com,inbound,001,World,0 -yourezads.com,yourezads.com,inbound,001,World,0.002974 -yourezlist.com,simplicityads.com,inbound,001,World,9.5e-05 -yourhostingaccount.com,yourhostingaccount.com,inbound,001,World,0 -yournewsletters.net,everydayhealth.com,inbound,001,World,0 -youversion.com,youversion.com,inbound,001,World,1 -zacks.com,zacks.com,inbound,001,World,0.999188 -zalando.be,fagms.de,inbound,001,World,0 -zalando.dk,fagms.de,inbound,001,World,0 -zalando.fi,fagms.de,inbound,001,World,0 -zalando.it,fagms.de,inbound,001,World,0 -zalando.nl,fagms.de,inbound,001,World,0 -zalando.pl,fagms.de,inbound,001,World,0 -zappos.com,zappos.com,inbound,001,World,0.626758 -zara.com,cheetahmail.com,inbound,001,World,0 -zattoo.com,sendnode.com,inbound,001,World,0 -zelonews.com.br,zelonews.com.br,inbound,001,World,0 -zendesk.com,zdsys.com,inbound,001,World,1 -zgalleriestyle.com,zgalleriestyle.com,inbound,001,World,0 -zibmail.info,zibmail.info,inbound,001,World,2e-06 -zillow.com,zillow.com,inbound,001,World,2.82542783334609e-07 -zinio.com,zinio.com,inbound,001,World,0.006193 -zinio.net,zinio.com,inbound,001,World,1 -zipalerts.com,sendgrid.net,inbound,001,World,1 -zipalerts.com,zipalerts.com,inbound,001,World,1 -ziprealty.com,ziprealty.com,inbound,001,World,0.994545 -ziprecruiter.com,ziprecruiter.com,inbound,001,World,1 -zivamewear.com,infimail.com,inbound,001,World,0 -zizigo.com,euromsg.net,inbound,001,World,0 -zlavadna.sk,zlavadna.sk,inbound,001,World,0 -zoom.com.br,zoom.com.br,inbound,001,World,0 -zoominternet.net,synacor.com,inbound,001,World,0 -zoosk.com,zoosk.com,inbound,001,World,0 -zoothost.com,zoothost.com,inbound,001,World,0.107168 -zorpia.com,zorpia.com,inbound,001,World,0.56104 -zovifashion.com,eccluster.com,inbound,001,World,0 -zulily.com,zulily.com,inbound,001,World,0 -zumzi.com,neogen.ro,inbound,001,World,0 -zumzi.com,zumzi.com,inbound,001,World,0 -zyngamail.com,zyngamail.com,inbound,001,World,0 -zzounds.com,zzounds.com,inbound,001,World,0 -careers24.com,careers24.com,inbound,002,Africa,0 -cpm.co.ma,cpm.co.ma,inbound,002,Africa,0 -fnb.co.za,fnb.co.za,inbound,002,Africa,0 -gmail.com,telkomadsl.co.za,inbound,002,Africa,0.999989 -gmail.com,vodacom.co.za,inbound,002,Africa,1 -gtbank.com,gtbank.com,inbound,002,Africa,0.056121 -pnetweb.co.za,hosting.co.za,inbound,002,Africa,0 -pnetweb.co.za,salesnet.co.za,inbound,002,Africa,0 -bigpond.com,bigpond.com,inbound,009,Oceania,0 -bigpond.com,bigpond.com,outbound,009,Oceania,1 -empoweredcomms.com.au,empoweredcomms.com.au,inbound,009,Oceania,0 -gmail.com,bigpond.net.au,inbound,009,Oceania,0.999907 -gmail.com,iinet.net.au,inbound,009,Oceania,0.966945 -gmail.com,optusnet.com.au,inbound,009,Oceania,0.992845 -gmail.com,tpgi.com.au,inbound,009,Oceania,0.999648 -mbounces.com,emdbms.com,inbound,009,Oceania,0 -realestate.com.au,realestate.com.au,inbound,009,Oceania,0 -snaphire.com,snaphire.com,inbound,009,Oceania,0 -trademe.co.nz,trademe.co.nz,inbound,009,Oceania,0.991916 -1-day.co.nz,1-day.co.nz,inbound,019,Americas,0 -12manage.com,netarrest.com,inbound,019,Americas,0.99998 -1800petmeds.com,1800petmeds.com,inbound,019,Americas,0.00599 -2touchbase.com,infimail.com,inbound,019,Americas,0 -4wheelparts.com,4wheelparts.com,inbound,019,Americas,0 -99acres.com,99acres.com,inbound,019,Americas,0 -aafes.com,aafes.com,inbound,019,Americas,0 -abercrombie-email.com,abercrombie-email.com,inbound,019,Americas,0 -abercrombiekids-email.com,abercrombie-email.com,inbound,019,Americas,0 -about.com,about.com,inbound,019,Americas,0 -about.com,sailthru.com,inbound,019,Americas,0 -acemserv.com,acemserv.com,inbound,019,Americas,0 -activesafelist.com,zoothost.com,inbound,019,Americas,0 -adidasusnews.com,adidasusnews.com,inbound,019,Americas,0 -adjockeys.com,thomas-j-brown.com,inbound,019,Americas,0 -admastersafelist.com,zoothost.com,inbound,019,Americas,0 -adorama.com,adorama.com,inbound,019,Americas,0 -adpirate.net,thomas-j-brown.com,inbound,019,Americas,0 -adtpulse.com,adtpulse.com,inbound,019,Americas,0 -ae.com,ae.com,inbound,019,Americas,0 -aexp.com,aexp.com,inbound,019,Americas,1 -affairalert.com,iverificationsystems.com,inbound,019,Americas,0 -agorafinancial.com,agorafinancial.com,inbound,019,Americas,0 -airbnb.com,airbnb.com,inbound,019,Americas,0.867975 -airbrake.io,mailgun.net,inbound,019,Americas,1 -airfarewatchdog.com,smartertravelmedia.com,inbound,019,Americas,0.012002 -airmiles.ca,bigfootinteractive.com,inbound,019,Americas,0 -akcijatau.lt,akcijatau.lt,inbound,019,Americas,0 -alarm.com,alarm.com,inbound,019,Americas,0 -alarmnet.com,alarmnet.com,inbound,019,Americas,0 -alaskaair.com,alaskaair.com,inbound,019,Americas,0 -alertid.com,alertid.com,inbound,019,Americas,0 -allheart.com,allheart.com,inbound,019,Americas,0 -allmodern.com,allmodern.com,inbound,019,Americas,0 -allout.org,allout.org,inbound,019,Americas,1 -allrecipes.com,allrecipes.com,inbound,019,Americas,0 -allstate.com,rsys1.com,inbound,019,Americas,0 -alm.com,sailthru.com,inbound,019,Americas,0 -alumniclass.com,alumniclass.com,inbound,019,Americas,0 -alumniconnections.com,alumniconnections.com,inbound,019,Americas,0 -amazon.{...},postini.com,inbound,019,Americas,0.658136 -amazonses.com,postini.com,inbound,019,Americas,0.733998 -americanas.com,americanas.com,inbound,019,Americas,0 -americanbar.org,abanet.org,inbound,019,Americas,0.00574 -amubm.com,amubm.com,inbound,019,Americas,1 -ancestry.com,ancestry.com,inbound,019,Americas,0 -angelbroking.in,infimail.com,inbound,019,Americas,0 -anghami.com,mailgun.net,inbound,019,Americas,1 -anthropologie.com,freepeople.com,inbound,019,Americas,0 -aol.com,aol.com,inbound,019,Americas,0.999528 -aol.com,aol.com,outbound,019,Americas,0.999984 -aol.com,sailthru.com,inbound,019,Americas,0 -aol.net,aol.com,inbound,019,Americas,1 -apache.org,apache.org,inbound,019,Americas,0 -apnacomplex.com,apnacomplex.com,inbound,019,Americas,0.999981 -apply-4-jobs.com,apply-4-jobs.com,inbound,019,Americas,0 -aptmail.in,mailurja.com,inbound,019,Americas,0 -arcamax.com,arcamax.com,inbound,019,Americas,1.4e-05 -aritzia.com,aritzia.com,inbound,019,Americas,0 -armaniexchange.com,bronto.com,inbound,019,Americas,0 -asadventure.com,asadventure.com,inbound,019,Americas,0 -asana.com,asana.com,inbound,019,Americas,1 -ashleymadison.com,ashleymadison.com,inbound,019,Americas,1 -asos.com,asos.com,inbound,019,Americas,0 -assembla.com,assembla.com,inbound,019,Americas,1 -astrology.com,astrology.com,inbound,019,Americas,0 -astrology.com,hsnlmailsvc.com,inbound,019,Americas,0 -astrology.com,webstakes.com,inbound,019,Americas,0 -astrology.com,wsafmailsvc.com,inbound,019,Americas,0 -atlassian.net,uc-inf.net,inbound,019,Americas,1 -atrapalo.cl,atrapalo.com,inbound,019,Americas,0 -atrapalo.com,atrapalo.com,inbound,019,Americas,0 -att-mail.com,att-mail.com,inbound,019,Americas,1.9e-05 -att-mail.com,att.com,inbound,019,Americas,0.999966 -att.net,att.net,outbound,019,Americas,0.204629 -att.net,mycingular.net,inbound,019,Americas,0.00021 -att.net,yahoo.{...},inbound,019,Americas,1 -australiagsm.net,australiagsm.net,inbound,019,Americas,0 -autoloop.us,loop28.com,inbound,019,Americas,0 -avaaz.org,avaaz.org,inbound,019,Americas,0 -avalanchesafelist.com,zoothost.com,inbound,019,Americas,0 -aveda.com,esteelauder.com,inbound,019,Americas,0 -avenue.com,avenue.com,inbound,019,Americas,0 -avg.com,avg.com,inbound,019,Americas,0 -avomail.com,avomail.com,inbound,019,Americas,0 -b2b-mail.net,b2b-mail.net,inbound,019,Americas,0 -b2b-mail.net,contact-list.net,inbound,019,Americas,0 -babycenter.com,rsys3.com,inbound,019,Americas,0 -babyoye.com,babyoye.com,inbound,019,Americas,0.011564 -badoo.com,monopost.com,inbound,019,Americas,1 -baligam.co.il,baligam.co.il,inbound,019,Americas,1 -banamex.com,ibrands.es,inbound,019,Americas,0 -bancoahorrofamsa.com,avantel.net.mx,inbound,019,Americas,1 -bancomercorreo.com,bancomercorreo.com,inbound,019,Americas,0 -bandsintown.com,bandsintown.com,inbound,019,Americas,1 -barclaycard.co.uk,barclays.co.uk,inbound,019,Americas,0 -barenecessities.com,barenecessities.com,inbound,019,Americas,0 -barleyment.ca,barleyment.ca,inbound,019,Americas,0 -barneys.com,barneys.com,inbound,019,Americas,0 -baseballsavings.com,baseballsavings.com,inbound,019,Americas,0 -basspronews.com,basspronews.com,inbound,019,Americas,0 -bathandbodyworks.com,bathandbodyworks.com,inbound,019,Americas,0 -baublebar.com,baublebar.com,inbound,019,Americas,0.011219 -bayt.com,bayt.com,inbound,019,Americas,2e-06 -bcp.com.pe,bcp.com.pe,inbound,019,Americas,1 -bellsouth.net,att.net,outbound,019,Americas,0 -bellsouth.net,yahoo.{...},inbound,019,Americas,1 -bergdorfgoodmanemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -bespokeoffers.co.uk,chtah.net,inbound,019,Americas,0 -bestbuy.ca,bestbuy.ca,inbound,019,Americas,0 -bestdealsforyou.in,elabs5.com,inbound,019,Americas,0 -bevmo.com,bevmo.com,inbound,019,Americas,0.000967 -bhcosmetics.com,bronto.com,inbound,019,Americas,0 -bhg.com,meredith.com,inbound,019,Americas,0 -bigfishgames.com,bigfishgames.com,inbound,019,Americas,0 -biglist.com,biglist.com,inbound,019,Americas,0 -bigtent.com,carezen.net,inbound,019,Americas,0 -bionexo.com,bionexo.com.br,inbound,019,Americas,0.999594 -birthdayalarm.com,monkeyinferno.net,inbound,019,Americas,0 -bitbucket.org,bitbucket.org,inbound,019,Americas,0 -bitlysupport.com,mailgun.info,inbound,019,Americas,1 -bitlysupport.com,mailgun.us,inbound,019,Americas,1 -bitslane.email,bitslane.email,inbound,019,Americas,0 -bitstatement.org,bitstatement.org,inbound,019,Americas,1 -bizjournals.com,bizjournals.com,inbound,019,Americas,0 -bizmailtoday.com,bizmailtoday.com,inbound,019,Americas,0 -bjs.com,bjs.com,inbound,019,Americas,0 -blablacar.com,blablacar.com,inbound,019,Americas,1 -blackberry.com,blackberry.com,inbound,019,Americas,0 -blackboard.com,blackboard.com,inbound,019,Americas,1 -blissworld.com,lstrk.net,inbound,019,Americas,1 -bloomingdales.com,bloomingdales.com,inbound,019,Americas,0 -bloomingdalesoutlets.com,bloomingdalesoutlets.com,inbound,019,Americas,0 -bluehost.com,bluehost.com,inbound,019,Americas,0.000971 -bluehost.com,hostmonster.com,inbound,019,Americas,0 -bluehost.com,unifiedlayer.com,inbound,019,Americas,0 -blueshellgames.com,blueshellgames.com,inbound,019,Americas,0 -bluestatedigital.com,bluestatedigital.com,inbound,019,Americas,0 -bluestonemx.com,bluestonemx.com,inbound,019,Americas,1 -bm23.com,bronto.com,inbound,019,Americas,0 -bm324.com,bronto.com,inbound,019,Americas,0 -bmdeda99.com,bmdeda99.com,inbound,019,Americas,0 -bn.com,bn.com,inbound,019,Americas,0 -bnetmail.com,bnetmail.com,inbound,019,Americas,0 -bol.com.br,bol.com.br,inbound,019,Americas,0 -bol.com.br,bol.com.br,outbound,019,Americas,0 -boletinrenuevo.com,boletinrenuevo.com,inbound,019,Americas,0 -bomnegocio.com,bomnegocio.com,inbound,019,Americas,0.588708 -bonobos.com,bronto.com,inbound,019,Americas,0 -bookbub.com,bookbub.com,inbound,019,Americas,1 -booking.com,booking.com,inbound,019,Americas,1 -bookingbuddy.com,smartertravelmedia.com,inbound,019,Americas,0.000487 -boomtownroi.com,boomtownroi.com,inbound,019,Americas,0 -boscovs.com,boscovs.com,inbound,019,Americas,0 -bostonproper.com,bostonproper.com,inbound,019,Americas,0 -boutiquesecret.com,chtah.net,inbound,019,Americas,0 -bradfordexchange.com,bradfordexchange.com,inbound,019,Americas,0 -bradsdeals.com,bradsdeals.com,inbound,019,Americas,0 -brassring.com,brassring.com,inbound,019,Americas,1 -briantracyintl.com,briantracyintl.com,inbound,019,Americas,0 -brijj.com,brijj.com,inbound,019,Americas,0 -brincltd.com,brincltd.com,inbound,019,Americas,0 -bronto.com,bronto.com,inbound,019,Americas,0 -bsf01.com,bsftransmit33.com,inbound,019,Americas,0 -buffalo.edu,buffalo.edu,inbound,019,Americas,0.001059 -bumeran.com,bumeran.com,inbound,019,Americas,0 -burlingtoncoatfactory.com,burlingtoncoatfactory.com,inbound,019,Americas,0 -burton.co.uk,burton.co.uk,inbound,019,Americas,0 -buscojobs.com,amazonaws.com,inbound,019,Americas,0 -buy123.com.tw,buy123.com.tw,inbound,019,Americas,1 -bzm.mobi,nmsrv.com,inbound,019,Americas,1 -c21stores.com,c21stores.com,inbound,019,Americas,0 -ca.gov,ca.gov,inbound,019,Americas,0.702758 -cafepress.com,cafepress.com,inbound,019,Americas,0.000778 -caixa.gov.br,caixa.gov.br,inbound,019,Americas,0 -californiapsychicsemail.com,californiapsychicsemail.com,inbound,019,Americas,0 -calottery.com,calottery.com,inbound,019,Americas,0.999426 -camel.com,rjrsignup.com,inbound,019,Americas,0 -canadianvisaexpert.net,canadianvisaexpert.net,inbound,019,Americas,0 -cancer.org,delivery.net,inbound,019,Americas,0 -capillary.co.in,capillary.co.in,inbound,019,Americas,1 -capitalone.com,bigfootinteractive.com,inbound,019,Americas,0 -capitalone360.com,ingdirect.com,inbound,019,Americas,0 -care.com,care.com,inbound,019,Americas,0 -care2.com,care2.com,inbound,019,Americas,0 -careerage.com,careerage.com,inbound,019,Americas,0 -careerflash.net,careerflash.net,inbound,019,Americas,1 -carmamail.com,carmamail.com,inbound,019,Americas,0 -carnivalfunmail.com,carnivalfunmail.com,inbound,019,Americas,0 -carolsdaughter.com,carolsdaughter.com,inbound,019,Americas,0 -carwale.com,carwale.com,inbound,019,Americas,0 -case.edu,cwru.edu,inbound,019,Americas,0.999942 -castingnetworks.com,castingnetworks.com,inbound,019,Americas,0.000102 -causes.com,causes.com,inbound,019,Americas,1 -cb2.com,cb2.com,inbound,019,Americas,0 -cbsig.net,cbsig.net,inbound,019,Americas,1 -ccbchurch.com,ccbchurch.com,inbound,019,Americas,1 -ccialerts.com,ccialerts.com,inbound,019,Americas,0 -ccs.com,footlocker.com,inbound,019,Americas,2.5e-05 -cenlat.com,cenlat.com,inbound,019,Americas,0.064459 -cerberusapp.com,cerberusapp.com,inbound,019,Americas,1 -cfmvmail.com,cfmvmail.com,inbound,019,Americas,0 -chabad.org,chabad.org,inbound,019,Americas,0 -champssports.com,footlocker.com,inbound,019,Americas,0.000131 -change.org,change.org,inbound,019,Americas,1 -charlestyrwhitt.com,charlestyrwhitt.com,inbound,019,Americas,0 -charter.net,charter.net,inbound,019,Americas,0 -charter.net,charter.net,outbound,019,Americas,0 -chase.com,jpmchase.com,inbound,019,Americas,0.999999 -chatcitynotifications.com,chatcitynotifications.com,inbound,019,Americas,0 -chaturbate.com,chaturbate.com,inbound,019,Americas,1 -cheapairmailer.com,cheapairmailer.com,inbound,019,Americas,0 -check.me,check.me,inbound,019,Americas,0 -cheekylovers.com,ropot.net,inbound,019,Americas,0 -chess.com,chess.com,inbound,019,Americas,1 -chicagotribune.com,latimes.com,inbound,019,Americas,0 -chicos.com,chicos.com,inbound,019,Americas,0.000156 -childrensplace.com,childrensplace.com,inbound,019,Americas,0 -christianbook.com,christianbook.com,inbound,019,Americas,1 -christianmingle.com,christianmingle.com,inbound,019,Americas,0 -chtah.com,chtah.net,inbound,019,Americas,0 -chtah.net,chtah.net,inbound,019,Americas,0 -cincghq.com,searchhomesingta.com,inbound,019,Americas,1 -circleofmomsmail.com,circleofmomsmail.com,inbound,019,Americas,0 -citibank.com,bigfootinteractive.com,inbound,019,Americas,0 -ck.com,ck.com,inbound,019,Americas,0 -clarks.com,clarks.com,inbound,019,Americas,0 -clickdimensions.com,clickdimensions.com,inbound,019,Americas,0 -clickexperts.net,clickexperts.net,inbound,019,Americas,0 -clicktoviewthisurl.org,clicktoviewthisurl.org,inbound,019,Americas,0 -climber.com,climber.com,inbound,019,Americas,0 -clinique.com,esteelauder.com,inbound,019,Americas,0 -clubcupon.com.ar,clubcupon.com.ar,inbound,019,Americas,0 -cmm01.com,coremotivesmarketing.com,inbound,019,Americas,0 -coach.com,delivery.net,inbound,019,Americas,0 -codeproject.com,codeproject.com,inbound,019,Americas,0 -coldwatercreek.com,coldwatercreek.com,inbound,019,Americas,0 -collectionsetc.com,collectionsetc.com,inbound,019,Americas,0 -columbia.edu,columbia.edu,inbound,019,Americas,0.762355 -comcast.net,comcast.net,inbound,019,Americas,0.919244 -comcast.net,comcast.net,outbound,019,Americas,0.999998 -comenity.net,alldata.net,inbound,019,Americas,1 -comenity.net,bigfootinteractive.com,inbound,019,Americas,0 -commonfloor.com,commonfloor.com,inbound,019,Americas,1 -compute.internal,amazonaws.com,inbound,019,Americas,0.875148 -computerworld.com,computerworld.com,inbound,019,Americas,0 -comunicacaodemkt.com,locaweb.com.br,inbound,019,Americas,0 -constantcontact.com,confirmedcc.com,inbound,019,Americas,0 -constantcontact.com,constantcontact.com,inbound,019,Americas,5.3e-05 -constantcontact.com,postini.com,inbound,019,Americas,0.042128 -containerstore.com,containerstore.com,inbound,019,Americas,0 -convio.net,convio.net,inbound,019,Americas,0 -coremotivesmarketing.com,coremotivesmarketing.com,inbound,019,Americas,0 -cornell.edu,cornell.edu,inbound,019,Americas,0.192285 -correosocc.com,correosocc.com,inbound,019,Americas,1 -costco.co.uk,costco.com,inbound,019,Americas,0 -costco.com,costco.com,inbound,019,Americas,7e-06 -costcophotocenter.com,wc09.net,inbound,019,Americas,0 -costcoservices.com,costco.com,inbound,019,Americas,0 -cotswoldoutdoor.com,cotswoldoutdoor.com,inbound,019,Americas,0 -couchsurfing.org,couchsurfing.com,inbound,019,Americas,0 -couponamama.com,couponamama.com,inbound,019,Americas,1 -cox.com,cox.com,inbound,019,Americas,0.001665 -cox.net,cox.net,inbound,019,Americas,0.009187 -cox.net,cox.net,outbound,019,Americas,0 -coyotelogistics.com,postini.com,inbound,019,Americas,0 -cp20.com,cp20.com,inbound,019,Americas,0 -cpbnc.com,cpbnc.com,inbound,019,Americas,0 -cpbnc.com,fye.com,inbound,019,Americas,0 -crackle.com,crackle.com,inbound,019,Americas,0 -craigslist.org,craigslist.org,inbound,019,Americas,0 -craigslist.org,craigslist.org,outbound,019,Americas,1 -crashlytics.com,crashlytics.com,inbound,019,Americas,1 -crashlytics.com,sendgrid.net,inbound,019,Americas,1 -crateandbarrel.com,crateandbarrel.com,inbound,019,Americas,0 -creationsrewards.net,creationsrewards.net,inbound,019,Americas,0 -creditkarma.com,creditkarma.com,inbound,019,Americas,1 -crosswalkmail.com,crosswalkmail.com,inbound,019,Americas,0 -crowdcut.com,crowdcut.com,inbound,019,Americas,1 -crunchyroll.com,crunchyroll.com,inbound,019,Americas,0 -cumulusdist.net,cumulusdist.net,inbound,019,Americas,0 -cuponatic.com.pe,cuponatic.com.pe,inbound,019,Americas,1 -cuponicamail.com,fnbox.com,inbound,019,Americas,0 -curbednetwork.com,curbednetwork.com,inbound,019,Americas,1 -cuspemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -custombriefings.com,custombriefings.com,inbound,019,Americas,0 -customercenter.net,customercenter.net,inbound,019,Americas,0.996453 -customeriomail.com,customeriomail.com,inbound,019,Americas,1 -cxomedia.com,cxomedia.com,inbound,019,Americas,0 -cybercoders.com,cybercoders.com,inbound,019,Americas,0 -dafiti.cl,dafiti.cl,inbound,019,Americas,0 -dailyhoroscope.com,tarot.com,inbound,019,Americas,0 -datehookup.com,datehookup.com,inbound,019,Americas,0 -datingvipnotifications.com,datingvipnotifications.com,inbound,019,Americas,0 -daviacalendar.com,daviacalendar.com,inbound,019,Americas,1 -davidsbridal.com,davidsbridal.com,inbound,019,Americas,0 -davidstea.com,bronto.com,inbound,019,Americas,0 -daz3d.com,bronto.com,inbound,019,Americas,0 -dealersocket.com,dealersocket.com,inbound,019,Americas,0 -dealsaver.com,secondstreetmedia.com,inbound,019,Americas,0.999823 -debshops.com,lstrk.net,inbound,019,Americas,1 -deliasshopemail.com,deliasshopemail.com,inbound,019,Americas,0 -delivery.net,delivery.net,inbound,019,Americas,0 -delivery.net,m0.net,inbound,019,Americas,0 -dell.com,bfi0.com,inbound,019,Americas,0 -dentalsenders.com,dentalsenders.com,inbound,019,Americas,0 -descontos.pt,descontos.pt,inbound,019,Americas,0 -designerapparel.com,myperfectsale.com,inbound,019,Americas,1 -despegar.com,despegar.com,inbound,019,Americas,0 -dhgate.com,chtah.net,inbound,019,Americas,0 -dice.com,dice.com,inbound,019,Americas,0 -digitalmailer.com,digitalmailer.com,inbound,019,Americas,0 -digitalmedia-comunicacion.es,chtah.net,inbound,019,Americas,0 -digitalromanceinc.com,digitalromanceinc.com,inbound,019,Americas,1 -directv.com,directv.com,inbound,019,Americas,0.026649 -discover.com,discoverfinancial.com,inbound,019,Americas,1 -disparadordeemails.com,locaweb.com.br,inbound,019,Americas,0 -disqus.net,disqus.net,inbound,019,Americas,0 -dn.net,naukri.com,inbound,019,Americas,0 -docusign.net,docusign.net,inbound,019,Americas,0.985435 -donationnet.net,donationnet.net,inbound,019,Americas,0 -dorothyperkins.com,dorothyperkins.com,inbound,019,Americas,0 -dowjones.info,dowjones.info,inbound,019,Americas,0 -dptagent.biz,dptagent.biz,inbound,019,Americas,0 -dptagent.net,dptagent.net,inbound,019,Americas,0 -dreamhost.com,dreamhost.com,inbound,019,Americas,0 -dreamwidth.org,dreamwidth.org,inbound,019,Americas,0 -drhinternet.net,drhinternet.net,inbound,019,Americas,0 -driftem.com,emce2.in,inbound,019,Americas,0 -driftem.com,mailurja.com,inbound,019,Americas,0 -dropbox.com,dropbox.com,inbound,019,Americas,1 -dropboxmail.com,dropbox.com,inbound,019,Americas,1 -dsw.com,dsw.com,inbound,019,Americas,0 -ducks.org,uptilt.com,inbound,019,Americas,0 -duke.edu,duke.edu,inbound,019,Americas,0.308101 -dukecareers.com,dukecareers.com,inbound,019,Americas,1 -dvor.com,dvor.com,inbound,019,Americas,0 -dynamite-safelist.com,thomas-j-brown.com,inbound,019,Americas,0 -dynect-mailer.net,sendlabs.com,inbound,019,Americas,0 -e-activist.com,e-activist.com,inbound,019,Americas,0 -e-beallsonline.com,e-stagestores.com,inbound,019,Americas,0 -e-costco.mx,costco.com,inbound,019,Americas,0 -e-goodysonline.com,e-stagestores.com,inbound,019,Americas,0 -e-peebles.com,e-stagestores.com,inbound,019,Americas,0 -e-rewards.net,e-rewards.net,inbound,019,Americas,1 -e-stagestores.com,e-stagestores.com,inbound,019,Americas,0 -e-travelclub.es,e-travelclub.es,inbound,019,Americas,0 -e2ma.net,e2ma.net,inbound,019,Americas,1 -ea.com,ea.com,inbound,019,Americas,0.001232 -eaccess.net,postini.com,inbound,019,Americas,0 -earn-e-miles.com,earn-e-miles.com,inbound,019,Americas,0 -earnerslist.com,traxweb.net,inbound,019,Americas,9e-06 -earthfare-email.com,edclient2.com,inbound,019,Americas,0 -earthlink.net,earthlink.net,inbound,019,Americas,0.031648 -earthlink.net,earthlink.net,outbound,019,Americas,0 -eastbay.com,footlocker.com,inbound,019,Americas,0 -easyhealthoptions.com,easyhealthoptions.com,inbound,019,Americas,1 -easyroommate.com,easyroommate.com,inbound,019,Americas,0 -ebates.com,bfi0.com,inbound,019,Americas,0 -ebay.{...},ebay.{...},inbound,019,Americas,0.99941 -ebizac2.com,ebizac2.com,inbound,019,Americas,0 -eblastengine.com,secondstreetmedia.com,inbound,019,Americas,0.999827 -ec2.internal,amazonaws.com,inbound,019,Americas,0.769271 -ecasend.com,ecasend.com,inbound,019,Americas,0 -ed.gov,leepfrog.com,inbound,019,Americas,0.106381 -ed10.net,ed10.com,inbound,019,Americas,0 -edirect1.com,ivytech.edu,inbound,019,Americas,0 -edmodo.com,edmodo.com,inbound,019,Americas,1.1e-05 -educationzone.co.in,iaires.com,inbound,019,Americas,0 -effectivesafelist.com,zoothost.com,inbound,019,Americas,0 -eharmony.com,eharmony.com,inbound,019,Americas,1e-06 -eigbox.net,eigbox.net,inbound,019,Americas,0 -elabs3.com,elabs3.com,inbound,019,Americas,0 -elabs3.com,meritline.com,inbound,019,Americas,0 -elabs5.com,elabs5.com,inbound,019,Americas,0 -elabs6.com,elabs6.com,inbound,019,Americas,0 -elanceonline.com,elanceonline.com,inbound,019,Americas,0 -eleadtrack.net,eleadtrack.net,inbound,019,Americas,0 -elitesafelist.com,elitesafelist.com,inbound,019,Americas,0 -email-cooking.com,email-cooking.com,inbound,019,Americas,0 -email-galls.com,email-galls.com,inbound,019,Americas,1 -email-od.com,smtprelayserver.com,inbound,019,Americas,0.999779 -email-ticketdada.com,email-ticketdada.com,inbound,019,Americas,1 -email4-beyond.com,email4-beyond.com,inbound,019,Americas,0 -emailcounts.com,secureserver.net,inbound,019,Americas,0 -emaildir2.com,emaildirect.net,inbound,019,Americas,0 -emaildir2.com,espsnd.com,inbound,019,Americas,0 -emailnotify.net,emailnotify.net,inbound,019,Americas,0.961424 -emailsbancoestado.cl,emailsbancoestado.cl,inbound,019,Americas,0 -emailsripley.cl,etarget.cl,inbound,019,Americas,0 -embluejet.com,embluejet.com,inbound,019,Americas,0 -embluejet.com,emblueuser.com,inbound,019,Americas,0 -emcsend.com,emcsend.com,inbound,019,Americas,0 -emergencyemail.org,emergencyemail.org,inbound,019,Americas,0 -emktsender.net,locaweb.com.br,inbound,019,Americas,0 -emma.cl,emma.cl,inbound,019,Americas,0.996895 -emobile.ad.jp,postini.com,inbound,019,Americas,0 -employboard.com,employboard.com,inbound,019,Americas,1 -entregadeemails.com,locaweb.com.br,inbound,019,Americas,0 -entregadordecampanhas.net,locaweb.com.br,inbound,019,Americas,0 -enviodecampanhas.net,locaweb.com.br,inbound,019,Americas,0 -enviodemkt.com.br,locaweb.com.br,inbound,019,Americas,0 -epriority.com,epriority.com,inbound,019,Americas,0 -equifax.com,equifax.com,inbound,019,Americas,1 -equussafelist.com,equussafelist.com,inbound,019,Americas,0.000265 -esri.com,esri.com,inbound,019,Americas,0.983104 -esteelauder.com,esteelauder.com,inbound,019,Americas,0 -evanguard.com,evanguard.com,inbound,019,Americas,0 -eventbrite.com,eventbrite.com,inbound,019,Americas,0 -eversavelocal.com,eversavelocal.com,inbound,019,Americas,0 -everydayhealthinc.com,waterfrontmedia.net,inbound,019,Americas,0 -everyjobforme.com,everyjobforme.com,inbound,019,Americas,0 -exchangesolutions.com,exchangesolutions.com,inbound,019,Americas,0.000143 -exec-u-net-mail.com,exec-u-net-mail.com,inbound,019,Americas,0 -exprpt.com,exprpt.com,inbound,019,Americas,0 -expvtinboxhub.net,expvtinboxhub.net,inbound,019,Americas,0 -fabletics.com,bronto.com,inbound,019,Americas,0 -facebook.com,facebook.com,inbound,019,Americas,0.719445 -facebook.com,facebook.com,outbound,019,Americas,1 -facebookappmail.com,facebook.com,inbound,019,Americas,1 -facebookmail.com,facebook.com,inbound,019,Americas,1 -facebookmail.com,postini.com,inbound,019,Americas,0.592681 -facebookmail.com,yahoo.{...},inbound,019,Americas,1 -familychristianmail.com,familychristianmail.com,inbound,019,Americas,0 -famousfootwear.com,famousfootwear.com,inbound,019,Americas,0 -fanfiction.com,fictionpress.com,inbound,019,Americas,1 -farmersonly.com,mailgun.us,inbound,019,Americas,1 -fashion2hub.in,mgenie.in,inbound,019,Americas,0 -fastgb.com,fastgb.com,inbound,019,Americas,0 -fastlistmailer.com,zoothost.com,inbound,019,Americas,0 -fastweb.com,fastweb.com,inbound,019,Americas,0 -fbi.gov,fbi.gov,inbound,019,Americas,0 -fbmta.com,fbmta.com,inbound,019,Americas,0 -fc2.com,fc2.com,inbound,019,Americas,0.000601 -fedoraproject.org,fedoraproject.org,inbound,019,Americas,0 -feedblitz.com,feedblitz.com,inbound,019,Americas,0 -fetlifemail.com,fetlifemail.com,inbound,019,Americas,0 -fibertel.com.ar,fibertel.com.ar,inbound,019,Americas,0.003898 -fidelizador.org,fidelizador.org,inbound,019,Americas,0 -findexpvtinbox.com,findexpvtinbox.com,inbound,019,Americas,0 -finishline.com,finishline.com,inbound,019,Americas,0 -firemountaingems.com,firemountaingems.com,inbound,019,Americas,0 -fisher-price.com,fisher-price.com,inbound,019,Americas,0 -fitbit.com,fitbit.com,inbound,019,Americas,1 -fitnessmagazine.com,meredith.com,inbound,019,Americas,0 -fiverr.com,fiverr.com,inbound,019,Americas,0 -flexmls.com,flexmls.com,inbound,019,Americas,0.999992 -flightaware.com,flightaware.com,inbound,019,Americas,0.020698 -flipkart.com,flipkart.com,inbound,019,Americas,1 -flirt.com,ropot.net,inbound,019,Americas,0 -flirthookup.com,flirthookup.com,inbound,019,Americas,1 -flyceb.com,flyceb.com,inbound,019,Americas,0 -flyfrontier.com,flyfrontier.com,inbound,019,Americas,0 -foolsubs.com,foolcs.com,inbound,019,Americas,0 -foolsubs.com,foolsubs.com,inbound,019,Americas,0 -footaction.com,footlocker.com,inbound,019,Americas,0 -footlocker.com,footlocker.com,inbound,019,Americas,0.000605 -forever21.com,forever21.com,inbound,019,Americas,0 -fortisbusinessmedia.com,fortisbusinessmedia.com,inbound,019,Americas,0 -foursquare.com,foursquare.com,inbound,019,Americas,1 -foxnews.com,foxnews.com,inbound,019,Americas,0.0139 -fragrancenet.com,fragrancenet.com,inbound,019,Americas,0.000427 -francescas.com,bronto.com,inbound,019,Americas,0 -freeadsmailer.com,zoothost.com,inbound,019,Americas,0 -freebeesafelist.com,zoothost.com,inbound,019,Americas,0 -freebizmag.com,delivery.net,inbound,019,Americas,0 -freebsd.org,freebsd.org,inbound,019,Americas,0.999845 -freecycle.org,freecycle.org,inbound,019,Americas,1 -freedesktop.org,freedesktop.org,inbound,019,Americas,0 -freeflys.com,freeflys.com,inbound,019,Americas,0 -freelancer.com,freelancer.com,inbound,019,Americas,0 -freelancer.com,freelancernotify.com,inbound,019,Americas,0 -freelancer.com,getafreelancer.com,inbound,019,Americas,0 -freelists.org,iquest.net,inbound,019,Americas,0 -freelotto.com,plasmanetinc.com,inbound,019,Americas,0 -freepeople.com,freepeople.com,inbound,019,Americas,0 -freesafelistking.com,zoothost.com,inbound,019,Americas,0 -freshdesk.com,freshdesk.com,inbound,019,Americas,1 -freshers2015.com,secureserver.net,inbound,019,Americas,0 -freshlatesave.com,freshlatesave.com,inbound,019,Americas,1 -friskone.com,mailurja.com,inbound,019,Americas,0 -frontsight.com,frontsight.com,inbound,019,Americas,0 -frys.com,frys.com,inbound,019,Americas,0.00388 -frysmail.com,frysmail.com,inbound,019,Americas,0 -fspeletters.com,agorapub.co.uk,inbound,019,Americas,0 -fuelrewards.com,britecast.com,inbound,019,Americas,0 -futureshop.com,futureshop.com,inbound,019,Americas,0 -gaiaonline.com,gaiaonline.com,inbound,019,Americas,0 -gamehouse.com,gamehouse.com,inbound,019,Americas,0 -gbyguess.com,guess.com,inbound,019,Americas,0.001787 -gemoney.com,rsys1.com,inbound,019,Americas,0 -generalmills.com,boxtops4education.com,inbound,019,Americas,0 -generalmills.com,pillsbury.com,inbound,019,Americas,0 -gentoo.org,gentoo.org,inbound,019,Americas,1 -get-me-jobs.com,get-me-jobs.com,inbound,019,Americas,0 -gethired.com,gethired.com,inbound,019,Americas,1 -getitfree.us,getitfree.us,inbound,019,Americas,0 -getpaidsolutions.com,getpaidsolutions.com,inbound,019,Americas,1 -getpocket.com,bronto.com,inbound,019,Americas,0 -ghin.com,ghinconnect.com,inbound,019,Americas,0 -ghup.in,mgenie.in,inbound,019,Americas,0 -gillyhicks-email.com,abercrombie-email.com,inbound,019,Americas,0 -github.com,github.com,inbound,019,Americas,1 -github.com,postini.com,inbound,019,Americas,0.837872 -glassdoor.com,glassdoor.com,inbound,019,Americas,1 -glasses.com,glasses.com,inbound,019,Americas,0 -gliq.com,gliq.com,inbound,019,Americas,0.99852 -globalsafelist.com,globalsafelist.com,inbound,019,Americas,0 -globaltestmarket.com,globaltestmarket.com,inbound,019,Americas,0 -gmail.com,amazonaws.com,inbound,019,Americas,0.994381 -gmail.com,anteldata.net.uy,inbound,019,Americas,0.998653 -gmail.com,bell.ca,inbound,019,Americas,0.992481 -gmail.com,bellsouth.net,inbound,019,Americas,0.9996 -gmail.com,blackberry.com,inbound,019,Americas,0.992207 -gmail.com,brasiltelecom.net.br,inbound,019,Americas,0.99995 -gmail.com,centurytel.net,inbound,019,Americas,0.999415 -gmail.com,cgocable.net,inbound,019,Americas,0.99829 -gmail.com,charter.com,inbound,019,Americas,0.999109 -gmail.com,claro.net.br,inbound,019,Americas,1 -gmail.com,comcast.net,inbound,019,Americas,0.999621 -gmail.com,comcastbusiness.net,inbound,019,Americas,0.985462 -gmail.com,cox.net,inbound,019,Americas,0.962619 -gmail.com,embarqhsd.net,inbound,019,Americas,0.999554 -gmail.com,franchiseindia.com,inbound,019,Americas,1 -gmail.com,frontiernet.net,inbound,019,Americas,0.995233 -gmail.com,gvt.net.br,inbound,019,Americas,0.999513 -gmail.com,lorexddns.net,inbound,019,Americas,0 -gmail.com,majesticmoneymailer.com,inbound,019,Americas,1 -gmail.com,mchsi.com,inbound,019,Americas,0.999951 -gmail.com,movistar.cl,inbound,019,Americas,0.999361 -gmail.com,mycingular.net,inbound,019,Americas,0.999918 -gmail.com,myvzw.com,inbound,019,Americas,0.99992 -gmail.com,naukri.com,inbound,019,Americas,0.000998 -gmail.com,optonline.net,inbound,019,Americas,0.999776 -gmail.com,postini.com,inbound,019,Americas,0.650888 -gmail.com,qwest.net,inbound,019,Americas,0.997574 -gmail.com,rcn.com,inbound,019,Americas,0.999275 -gmail.com,rogers.com,inbound,019,Americas,0.999916 -gmail.com,rr.com,inbound,019,Americas,0.986894 -gmail.com,sbcglobal.net,inbound,019,Americas,0.998817 -gmail.com,shawcable.net,inbound,019,Americas,0.999998 -gmail.com,spcsdns.net,inbound,019,Americas,0.999998 -gmail.com,suddenlink.net,inbound,019,Americas,0.961593 -gmail.com,telecom.net.ar,inbound,019,Americas,0.999664 -gmail.com,telesp.net.br,inbound,019,Americas,0.999743 -gmail.com,telus.com,inbound,019,Americas,1 -gmail.com,telus.net,inbound,019,Americas,0.974663 -gmail.com,tmodns.net,inbound,019,Americas,1 -gmail.com,veloxzone.com.br,inbound,019,Americas,0.999969 -gmail.com,verizon.net,inbound,019,Americas,0.990214 -gmail.com,videotron.ca,inbound,019,Americas,0.967196 -gmail.com,vtr.net,inbound,019,Americas,0.999072 -gmail.com,websitewelcome.com,inbound,019,Americas,1 -gmail.com,wideopenwest.com,inbound,019,Americas,0.999729 -gmail.com,windstream.net,inbound,019,Americas,0.951847 -gmail.com,yahoo.{...},inbound,019,Americas,0.999992 -gmail.com,zoothost.com,inbound,019,Americas,0.033858 -gob.ar,gob.ar,inbound,019,Americas,0.160006 -gob.ec,gob.ec,inbound,019,Americas,0.623867 -godaddy.com,secureserver.net,inbound,019,Americas,0 -gogecapital.com,rsys1.com,inbound,019,Americas,0 -goldenopsafelist.com,zoothost.com,inbound,019,Americas,0 -goldstar.com,goldstar.com,inbound,019,Americas,1 -golfmnb.com,golfmnb.com,inbound,019,Americas,0 -google.com,postini.com,inbound,019,Americas,0.584887 -gopusamedia.com,gopusamedia.com,inbound,019,Americas,0 -govdelivery.com,govdelivery.com,inbound,019,Americas,0 -governmentjobs.com,governmentjobs.com,inbound,019,Americas,0 -gpmailer.com.br,parperfeito.com,inbound,019,Americas,0 -grassrootsaction.com,grassfire.net,inbound,019,Americas,0 -greatergood.com,greatergood.com,inbound,019,Americas,0 -groupon.{...},chtah.net,inbound,019,Americas,0 -groupon.{...},groupon.{...},inbound,019,Americas,1.0 -groupon.{...},postini.com,inbound,019,Americas,0.886672 -grouponmail.{...},grouponmail.{...},inbound,019,Americas,0 -grupos.com.br,grupos.com.br,inbound,019,Americas,0 -guess.ca,guess.com,inbound,019,Americas,0.000807 -guess.com,guess.com,inbound,019,Americas,0.003805 -guessfactory.com,guess.com,inbound,019,Americas,0.00164 -gustazos.com,cityoferta.com,inbound,019,Americas,1 -hallmark.com,hallmark.com,inbound,019,Americas,0 -hannaandersson.com,hannaandersson.com,inbound,019,Americas,0.013533 -harristeetermail.com,harristeetermail.com,inbound,019,Americas,0.99673 -harvard.edu,harvard.edu,inbound,019,Americas,0.274906 -hayneedle.com,hayneedle.com,inbound,019,Americas,0 -helpareporter.net,helpareporter.com,inbound,019,Americas,0 -herculist.com,herculist.com,inbound,019,Americas,0 -hilton.com,hiltonemail.com,inbound,019,Americas,0 -hipchat.com,hipchat.com,inbound,019,Americas,1 -hispavista.com,hispavista.com,inbound,019,Americas,0 -hollister-email.com,abercrombie-email.com,inbound,019,Americas,0 -homeaway.com,haspf.com,inbound,019,Americas,0 -homedecorators.com,homedecorators.com,inbound,019,Americas,0 -homedepot.com,homedepot.com,inbound,019,Americas,1 -hootsuite.com,hootsuite.com,inbound,019,Americas,1 -horchowemail.com,horchowemail.com,inbound,019,Americas,0 -hostelworld.com,bronto.com,inbound,019,Americas,0 -hostgator.com,hostgator.com,inbound,019,Americas,0.98061 -hostgator.com,websitewelcome.com,inbound,019,Americas,1 -hotmail.{...},hotmail.{...},inbound,019,Americas,1 -hotmail.{...},hotmail.{...},outbound,019,Americas,1 -hotmail.{...},postini.com,inbound,019,Americas,0.723143 -hotornot.com,monopost.com,inbound,019,Americas,1 -hotschedules.com,hotschedules.com,inbound,019,Americas,0 -house.gov,house.gov,inbound,019,Americas,0.999966 -houseoffraser.co.uk,houseoffraser.co.uk,inbound,019,Americas,0 -houzz.com,houzz.com,inbound,019,Americas,1 -hubspot.com,hubspot.com,inbound,019,Americas,1 -hungry-girl.com,hungry-girl.com,inbound,019,Americas,0 -iamlgnd2.com,iamlgnd2.com,inbound,019,Americas,1 -ibsys.com,ibsys.com,inbound,019,Americas,9e-05 -icbc.com.ar,clickexperts.net,inbound,019,Americas,0 -icbc.com.ar,standardbank.com.ar,inbound,019,Americas,0 -icims.com,icims.com,inbound,019,Americas,0.999939 -icors.org,lsoft.us,inbound,019,Americas,0 -icpbounce.com,icpbounce.com,inbound,019,Americas,0 -idc.email,nmsrv.com,inbound,019,Americas,1 -idgconnect-resources.com,idgconnect-resources.com,inbound,019,Americas,0 -ieee.org,ieee.org,inbound,019,Americas,0.999912 -ig.com.br,ig.com.br,inbound,019,Americas,0 -ig.com.br,ig.com.br,outbound,019,Americas,0 -igot-mails.com,zoothost.com,inbound,019,Americas,0 -iimjobs.com,iimjobs.com,inbound,019,Americas,1 -ikmultimedianews.com,ikmultimedianews.com,inbound,019,Americas,0.993319 -illinois.edu,illinois.edu,inbound,019,Americas,0.866481 -imageshost.ca,imageshost.ca,inbound,019,Americas,0 -imakenews.net,imakenews.com,inbound,019,Americas,0 -imo.im,imo.im,inbound,019,Americas,1 -imodules.com,imodules.com,inbound,019,Americas,0 -imvu.com,imvu.com,inbound,019,Americas,7e-06 -inboxdollars.com,inboxdollars.com,inbound,019,Americas,0 -inboxfirst.com,inboxfirst.com,inbound,019,Americas,0 -inboxmarketer-mail.com,inboxmarketer-mail.com,inbound,019,Americas,0.999877 -inboxpays.com,inboxpays.com,inbound,019,Americas,0 -inboxpounds.co.uk,inboxpounds.co.uk,inbound,019,Americas,0 -indeed.com,indeed.com,inbound,019,Americas,0.000121 -indeedemail.com,indeedemail.com,inbound,019,Americas,0 -independentlivingbullion.com,independentlivingbullion.com,inbound,019,Americas,0 -infobradesco.com.br,infobradesco.com.br,inbound,019,Americas,0 -infojobs.com.br,anuntis.com,inbound,019,Americas,0 -infomoney.com.br,infomoney.com.br,inbound,019,Americas,1 -informz.net,informz.net,inbound,019,Americas,0 -infradead.org,infradead.org,inbound,019,Americas,1 -inman.com,inman.com,inbound,019,Americas,0 -innovyx.net,innovyx.net,inbound,019,Americas,0 -insidehook.com,sailthru.com,inbound,019,Americas,0 -instagram.com,facebook.com,inbound,019,Americas,1 -instantprofitlist.com,screenshotads.com,inbound,019,Americas,0 -interac.ca,certapay.com,inbound,019,Americas,0 -interactivebrokers.com,interactivebrokers.com,inbound,019,Americas,1 -interactiverealtyservices.com,interactiverealtyservices.com,inbound,019,Americas,0 -intercom.io,mailgun.info,inbound,019,Americas,1 -interealty.net,interealty.net,inbound,019,Americas,1 -interweave.com,interweave.com,inbound,019,Americas,0 -intliv2.net,internationalliving.com,inbound,019,Americas,0 -invalidemail.com,taleo.net,inbound,019,Americas,1 -investopedia.com,vclk.net,inbound,019,Americas,0.999999 -investorplace.com,investorplace.com,inbound,019,Americas,0.995093 -irctcshopping.com,chtah.net,inbound,019,Americas,0 -iridium.com,iridium.com,inbound,019,Americas,0.997882 -isendservice.com.br,isendservice.com.br,inbound,019,Americas,0 -itau-unibanco.com.br,itau.com.br,inbound,019,Americas,0 -ittoolbox.com,ittoolbox.com,inbound,019,Americas,0 -ittoolbox.com,toolbox.com,inbound,019,Americas,0 -itwhitepapers.com,itwhitepapers.com,inbound,019,Americas,0 -iwantoneofthose.com,thehut.com,inbound,019,Americas,0 -ixs1.net,ixs1.net,inbound,019,Americas,0.005131 -jcpenney.com,jcpenney.com,inbound,019,Americas,9e-06 -jeevansathi.com,jeevansathi.com,inbound,019,Americas,0 -jetsetter.com,smartertravelmedia.com,inbound,019,Americas,0.041888 -jibjab.com,storybots.com,inbound,019,Americas,0 -jira.com,uc-inf.net,inbound,019,Americas,1 -jobscentral.com.sg,mailgun.net,inbound,019,Americas,1 -jobson.com,jobsonmail.com,inbound,019,Americas,0 -jobsradar.com,jobsradar.com,inbound,019,Americas,0 -jockeycomfort.com,jockeycomfort.com,inbound,019,Americas,0 -johnstonandmurphy-email.com,johnstonandmurphy-email.com,inbound,019,Americas,0 -jomashop.com,lstrk.net,inbound,019,Americas,1 -josbank.com,josbank.com,inbound,019,Americas,0 -jossandmain.com,jossandmain.com,inbound,019,Americas,0 -jtv.com,jtv.com,inbound,019,Americas,0 -juno.com,untd.com,inbound,019,Americas,0 -juno.com,untd.com,outbound,019,Americas,0 -justdial.com,mailurja.com,inbound,019,Americas,0 -justfab.com,bronto.com,inbound,019,Americas,0 -justfab.fr,bronto.com,inbound,019,Americas,0 -keek.com,keek.com,inbound,019,Americas,1 -kernel.org,kernel.org,inbound,019,Americas,0 -kgstores.com,kgstores.com,inbound,019,Americas,0 -kickstarter.com,kickstarter.com,inbound,019,Americas,1 -kidsfootlocker.com,footlocker.com,inbound,019,Americas,0 -kik.com,kik.com,inbound,019,Americas,1 -kimblegroup.com,kimblegroup.com,inbound,019,Americas,1 -kintera.com,kintera.com,inbound,019,Americas,0 -klaviyomail.com,klaviyomail.com,inbound,019,Americas,1 -klove.com,emfbroadcasting.com,inbound,019,Americas,0 -komando.com,komando.com,inbound,019,Americas,0 -kp.org,kp.org,inbound,019,Americas,0.999975 -krogermail.com,bigfootinteractive.com,inbound,019,Americas,0 -landmarketingmailer.com,zoothost.com,inbound,019,Americas,0 -landofnod.com,landofnod.com,inbound,019,Americas,0.002525 -languagepod101.com,eclient10.com,inbound,019,Americas,0 -languagepod101.com,eddlvr.com,inbound,019,Americas,0 -languagepod101.com,ednwsltr3.com,inbound,019,Americas,0 -languagepod101.com,ednwsltr8.com,inbound,019,Americas,0 -languagepod101.com,emaildirect.net,inbound,019,Americas,0 -lasenza.com,lasenza.com,inbound,019,Americas,0 -lastcallemail.com,lastcallemail.com,inbound,019,Americas,0 -latimes.com,latimes.com,inbound,019,Americas,0 -lauraashley.com,lauraashley.com,inbound,019,Americas,0 -leftlanesports.com,auspient.com,inbound,019,Americas,0.00705 -leftlanesports.com,leftlanesports.com,inbound,019,Americas,0 -legalshieldassociate.com,legalshield.com,inbound,019,Americas,0 -lexico.com,lexico.com,inbound,019,Americas,0 -life360.com,life360.com,inbound,019,Americas,1 -lindenlab.com,lindenlab.com,inbound,019,Americas,0.999444 -linkedin.com,linkedin.com,inbound,019,Americas,0.999873 -linkedin.com,postini.com,inbound,019,Americas,0.713391 -listeneremail.net,listeneremail.net,inbound,019,Americas,0 -listia.com,listia.com,inbound,019,Americas,1 -listnerds.com,listnerds.com,inbound,019,Americas,0 -listreturn.com,zoothost.com,inbound,019,Americas,0 -listserve.com,listserve.com,inbound,019,Americas,0 -listvolta.com,listvolta.com,inbound,019,Americas,0 -listwire.com,listwire.com,inbound,019,Americas,0 -live.{...},hotmail.{...},inbound,019,Americas,1 -live.{...},hotmail.{...},outbound,019,Americas,1 -livefyre.com,andbit.net,inbound,019,Americas,1 -livescribe.com,bronto.com,inbound,019,Americas,0 -livingsocial.com,livingsocial.com,inbound,019,Americas,0 -livrariasaraiva.com.br,livrariasaraiva.com.br,inbound,019,Americas,0 -localhires.com,localhires.com,inbound,019,Americas,1 -logitech.com,dvsops.com,inbound,019,Americas,0 -lojasmarisa.com.br,lojasmarisa.com.br,inbound,019,Americas,0 -lolsolos.com,ultimateadsites.net,inbound,019,Americas,0.999996 -lonelywifehookup.com,iverificationsystems.com,inbound,019,Americas,0 -lordandtaylor.com,lordandtaylor.com,inbound,019,Americas,0 -loveaholics.com,ropot.net,inbound,019,Americas,0 -lovelywholesale.com,lovelywholesale.com,inbound,019,Americas,1 -lrsmail.com,lrsmail.com,inbound,019,Americas,0 -lt02.net,listrak.com,inbound,019,Americas,1 -lt02.net,lstrk.net,inbound,019,Americas,1 -ltdcommodities.com,ltdcomm.net,inbound,019,Americas,0 -luckymag.com,mkt4500.com,inbound,019,Americas,0 -lulu.com,bronto.com,inbound,019,Americas,0 -lulus.com,lstrk.net,inbound,019,Americas,1 -lumosity.com,lumosity.com,inbound,019,Americas,1 -lyst.com,lyst.com,inbound,019,Americas,1 -maccosmetics.com,esteelauder.com,inbound,019,Americas,0 -macupdate.com,mailgun.info,inbound,019,Americas,1 -madmels.info,ultimateadsites.net,inbound,019,Americas,1 -madmimi.com,madmimi.com,inbound,019,Americas,0 -magicjack.com,magicjack.com,inbound,019,Americas,1 -magnetdev.com,magnetmail.net,inbound,019,Americas,0 -mail-route.com,mail-route.com,inbound,019,Americas,0 -mail-thestreet.com,mail-thestreet.com,inbound,019,Americas,0 -mail.mil,mail.mil,inbound,019,Americas,0 -mailaccurate.com,mgenie.in,inbound,019,Americas,0 -mailfacil.com.br,md02.com,inbound,019,Americas,0 -mailfeast.com,mgenie.in,inbound,019,Americas,0 -mailgun.org,mailgun.info,inbound,019,Americas,1 -mailgun.org,mailgun.net,inbound,019,Americas,1 -mailgun.org,mailgun.us,inbound,019,Americas,1 -mailingathome.net,mailingathome.net,inbound,019,Americas,0.999995 -mailjayde.com,mailjayde.com,inbound,019,Americas,0 -mailmachine1050.com,mailmachine1050.com,inbound,019,Americas,0 -mailsend1.com,mailsend6.com,inbound,019,Americas,0 -mailsender.com.br,mailsender.com.br,inbound,019,Americas,0 -manager.com.br,manager.com.br,inbound,019,Americas,0 -mandrillapp.com,mandrillapp.com,inbound,019,Americas,1 -mandrillapp.com,myjobhelperalerts.com,inbound,019,Americas,1 -marcustheatres.com,movio.co,inbound,019,Americas,0 -markandgraham.com,markandgraham.com,inbound,019,Americas,0 -marketer-safelist.com,jsalfianmarketing.com,inbound,019,Americas,1 -marketinghq.net,elabs8.com,inbound,019,Americas,0 -marketingprofs.com,marketingprofs.com,inbound,019,Americas,0.004412 -marlboro.com,marlboro.com,inbound,019,Americas,0 -maropost.com,biotrustnews.com,inbound,019,Americas,0 -maropost.com,mailing-truthaboutabs.com,inbound,019,Americas,0 -maropost.com,maropost.com,inbound,019,Americas,0 -maropost.com,mp2201.com,inbound,019,Americas,0 -masivapp.com,masivapp.com,inbound,019,Americas,1 -massageenvyclinics.com,massageenvyclinics.com,inbound,019,Americas,0 -masterbase.com,masterbase.com,inbound,019,Americas,0 -mastercard-email.com,mastercard-email.com,inbound,019,Americas,0 -mate1.net,mate1.net,inbound,019,Americas,0 -matrixemailer.com,matrixemailer.com,inbound,019,Americas,0 -mbstrm.com,mobilestorm.com,inbound,019,Americas,0 -mcafee.com,mcafee.com,inbound,019,Americas,0.979126 -mcarthurglen.com,mcarthurglen.com,inbound,019,Americas,0 -mcdlv.net,mcdlv.net,inbound,019,Americas,0 -mckinsey.com,bigfootinteractive.com,inbound,019,Americas,0 -mcsv.net,mcsv.net,inbound,019,Americas,0 -mdlinx.com,mdlinx.com,inbound,019,Americas,0 -mec.gov.br,mec.gov.br,inbound,019,Americas,0 -mediabistro.com,iworld.com,inbound,019,Americas,0.006428 -medpagetoday.com,wc09.net,inbound,019,Americas,0 -meetmemail.com,meetmemail.com,inbound,019,Americas,0 -megasenders.com,megasenders.com,inbound,019,Americas,0.07662 -memberdealsusa.com,memberdealsusa.com,inbound,019,Americas,0 -menswearhouse.com,menswearhouse.com,inbound,019,Americas,0 -mercadojobs.com,sendgrid.net,inbound,019,Americas,1 -mercola.com,mercola.com,inbound,019,Americas,0.000369 -messagegears.net,messagegears.net,inbound,019,Americas,0 -met-art.com,hydentra.com,inbound,019,Americas,1 -mgo.com,bronto.com,inbound,019,Americas,0 -michaels.com,chtah.net,inbound,019,Americas,0 -michaels.com,michaels.com,inbound,019,Americas,0 -microcentermedia.com,bfi0.com,inbound,019,Americas,0 -microsoft.com,hotmail.{...},inbound,019,Americas,1 -microsoft.com,msn.com,inbound,019,Americas,1 -midnightsunsafelist.com,zoothost.com,inbound,019,Americas,0 -milfaholic.com,iverificationsystems.com,inbound,019,Americas,0 -miltnews.com,miltnews.com,inbound,019,Americas,0 -mindbodyonline.com,mindbodyonline.com,inbound,019,Americas,1 -mindfieldonline.com,mindfieldonline.com,inbound,019,Americas,0 -mindmoviesmail.com,mindmoviesmail.com,inbound,019,Americas,0.004239 -mindvalleymail3.com,mindvalleymail3.com,inbound,019,Americas,0 -mint.com,mint.com,inbound,019,Americas,0 -minted.com,messagelabs.com,inbound,019,Americas,0.999669 -missselfridge.com,wallis-fashion.com,inbound,019,Americas,0 -mistersafelist.com,zoothost.com,inbound,019,Americas,0 -mit.edu,mit.edu,inbound,019,Americas,0.869568 -mjinn.com,mailurja.com,inbound,019,Americas,0 -mkt015.com,mkt015.com,inbound,019,Americas,0 -mkt022.com,mkt022.com,inbound,019,Americas,0 -mkt063.com,mkt063.com,inbound,019,Americas,0 -mkt1136.com,mkt1136.com,inbound,019,Americas,0 -mkt1985.com,fmlinks.net,inbound,019,Americas,0 -mkt2010.com,mkt2010.com,inbound,019,Americas,0 -mkt2106.com,mkt2106.com,inbound,019,Americas,0 -mkt2170.com,mkt2170.com,inbound,019,Americas,0 -mkt2181.com,mkt2181.com,inbound,019,Americas,0 -mkt2615.com,mkt2615.com,inbound,019,Americas,0 -mkt2813.com,mkt2813.com,inbound,019,Americas,0 -mkt2944.com,mkt2944.com,inbound,019,Americas,0 -mkt3134.com,mkt3134.com,inbound,019,Americas,0 -mkt3142.com,mkt3142.com,inbound,019,Americas,0 -mkt3156.com,mkt3156.com,inbound,019,Americas,0 -mkt3203.com,mkt3203.com,inbound,019,Americas,0 -mkt346.com,mkt346.com,inbound,019,Americas,0 -mkt3544.com,mkt3544.com,inbound,019,Americas,0 -mkt3622.com,mkt3622.com,inbound,019,Americas,0 -mkt3682.com,mkt3682.com,inbound,019,Americas,0 -mkt3690.com,mkt3690.com,inbound,019,Americas,0 -mkt3695.com,mkt3695.com,inbound,019,Americas,0 -mkt3804.com,mkt3804.com,inbound,019,Americas,0 -mkt3815.com,mkt3815.com,inbound,019,Americas,0 -mkt3952.com,xoom.com,inbound,019,Americas,0 -mkt4355.com,mkt4355.com,inbound,019,Americas,0 -mkt4364.com,mkt4364.com,inbound,019,Americas,0 -mkt459.com,mkt459.com,inbound,019,Americas,0 -mkt4701.com,mkt4701.com,inbound,019,Americas,0 -mkt4728.com,mkt4728.com,inbound,019,Americas,0 -mkt4731.com,mkt4731.com,inbound,019,Americas,0 -mkt4738.com,mkt4738.com,inbound,019,Americas,0 -mkt5071.com,mkt5071.com,inbound,019,Americas,0 -mkt5098.com,mkt5098.com,inbound,019,Americas,0 -mkt5131.com,mkt5131.com,inbound,019,Americas,0 -mkt5144.com,mkt5144.com,inbound,019,Americas,0 -mkt5144.com,mkt5980.com,inbound,019,Americas,0 -mkt5144.com,mkt5981.com,inbound,019,Americas,0 -mkt5181.com,mkt5181.com,inbound,019,Americas,0 -mkt5269.com,mkt5269.com,inbound,019,Americas,0 -mkt529.com,mkt529.com,inbound,019,Americas,0 -mkt5297.com,mkt5297.com,inbound,019,Americas,0 -mkt5297.com,mkt5309.com,inbound,019,Americas,0 -mkt5371.com,mkt5371.com,inbound,019,Americas,0 -mkt5806.com,mkt5806.com,inbound,019,Americas,0 -mkt5934.com,mkt5934.com,inbound,019,Americas,0 -mkt5937.com,mkt5937.com,inbound,019,Americas,0 -mkt5970.com,mkt5970.com,inbound,019,Americas,0 -mkt6100.com,mkt6098.com,inbound,019,Americas,0 -mkt6276.com,mkt6276.com,inbound,019,Americas,0 -mkt6323.com,mkt6323.com,inbound,019,Americas,0 -mkt746.com,mkt746.com,inbound,019,Americas,0 -mkt824.com,mkt869.com,inbound,019,Americas,0 -mktdillards.com,mktdillards.com,inbound,019,Americas,0 -mo1send.com,mo1send.com,inbound,019,Americas,0 -mobly.com.br,mobly.com.br,inbound,019,Americas,0 -modellsemail.com,n-email.net,inbound,019,Americas,0 -moneymorning.com,moneymappress.com,inbound,019,Americas,0 -monster.com,monster.com,inbound,019,Americas,0 -monster.com,tmpw.net,inbound,019,Americas,0.000503 -moon-ray.com,moon-ray.com,inbound,019,Americas,0 -mooresclothing.com,mooresclothing.com,inbound,019,Americas,0 -moveon.org,moveon.org,inbound,019,Americas,0.996147 -mrmlsmatrix.com,mrmlsmatrix.com,inbound,019,Americas,0 -ms.com,ms.com,inbound,019,Americas,1 -msn.com,hotmail.{...},inbound,019,Americas,1 -msn.com,hotmail.{...},outbound,019,Americas,1 -mta.info,ealert.com,inbound,019,Americas,0 -mtasv.net,mtasv.net,inbound,019,Americas,0.999999 -musicnotes-alerts.com,mybuys.com,inbound,019,Americas,0 -mustanglist.com,mustanglist.com,inbound,019,Americas,0 -mycheapoair.com,mycheapoair.com,inbound,019,Americas,0.957931 -mydailymoment.biz,mydailymoment.biz,inbound,019,Americas,0 -mydailymoment.info,mydailymoment.info,inbound,019,Americas,0 -mydailymoment.net,mydailymoment.net,inbound,019,Americas,0 -mydailymoment.us,mydailymoment.us,inbound,019,Americas,0 -myfedloan.org,aessuccess.org,inbound,019,Americas,0.328917 -myfxbook.com,myfxbook.com,inbound,019,Americas,0 -mygreatlakes.org,glhec.org,inbound,019,Americas,0.000247 -myhealthwealthandhappiness.com,myhealthwealthandhappiness.com,inbound,019,Americas,0 -mymeijer.com,mymeijer.com,inbound,019,Americas,0 -myngp.com,ngpweb.com,inbound,019,Americas,0 -myperfectsale.com,myperfectsale.com,inbound,019,Americas,1 -myprotein.com,thehut.com,inbound,019,Americas,0 -mysafelistmailer.com,mysafelistmailer.com,inbound,019,Americas,0.00033 -mysmartprice.com,itzwow.com,inbound,019,Americas,1 -myzamanamail.com,myzamanamail.com,inbound,019,Americas,0 -n-email.net,n-email.net,inbound,019,Americas,0 -n-email1.net,n-email1.net,inbound,019,Americas,0 -n-email4.net,n-email4.net,inbound,019,Americas,0 -nanomail.com.br,araie.com.br,inbound,019,Americas,1 -napitipp.hu,napitipp.hu,inbound,019,Americas,0 -nasa.gov,nasa.gov,inbound,019,Americas,0.101289 -nastygal.com,bronto.com,inbound,019,Americas,0 -nationbuilder.com,nationbuilder.com,inbound,019,Americas,1 -nature.com,nature.com,inbound,019,Americas,0 -naukri.com,naukri.com,inbound,019,Americas,0.000533 -nauta.cu,etecsa.net,inbound,019,Americas,0 -nba.com,nba.com,inbound,019,Americas,0.005829 -nbaa.org,nbaa.org,inbound,019,Americas,0 -ncl.com,ncl.com,inbound,019,Americas,0 -neimanmarcusemail.com,neimanmarcusemail.com,inbound,019,Americas,0 -net-a-porter.com,net-a-porter.com,inbound,019,Americas,0 -net-empregos.com,net-empregos.com,inbound,019,Americas,0 -netcommunity1.com,blackbaud.com,inbound,019,Americas,0 -netflix.com,netflix.com,inbound,019,Americas,1 -netopia.pt,netopia.pt,inbound,019,Americas,0 -netprosoftmail.com,netprosoftmail.com,inbound,019,Americas,0 -netsuite.com,netsuite.com,inbound,019,Americas,0.60682 -networkworld.com,networkworld.com,inbound,019,Americas,0 -newgrounds.com,newgrounds.com,inbound,019,Americas,0 -newrelic.com,sendlabs.com,inbound,019,Americas,0 -newspaperdirect.com,newspaperdirect.com,inbound,019,Americas,0.000177 -newyorktimesinfo.com,newyorktimesinfo.com,inbound,019,Americas,0 -nexcess.net,nexcess.net,inbound,019,Americas,0.039513 -nextdoor.com,mailgun.info,inbound,019,Americas,1 -nfl.com,bfi0.com,inbound,019,Americas,0 -nih.gov,nih.gov,inbound,019,Americas,8.8e-05 -ninewestmail.com,ninewestmail.com,inbound,019,Americas,0 -ning.com,ning.com,inbound,019,Americas,0 -nixle.com,nixle.com,inbound,019,Americas,0 -nl00.net,netline.com,inbound,019,Americas,5e-06 -nl00.net,nl00.net,inbound,019,Americas,0 -nongnu.org,gnu.org,inbound,019,Americas,1 -nordstrom.com,taleo.net,inbound,019,Americas,1 -nortonfromsymantec.com,rsys1.com,inbound,019,Americas,0 -novidadeslojasrenner.com.br,novidadeslojasrenner.com.br,inbound,019,Americas,0 -numbersusa.com,numbersusa.com,inbound,019,Americas,0 -nyandcompany.com,nyandcompany.com,inbound,019,Americas,0 -ocadomail.com,ocadomail.com,inbound,019,Americas,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,019,Americas,0 -ofertasefacil.com.br,ofertasefacil.com.br,inbound,019,Americas,0 -ofertop.pe,icommarketing.com,inbound,019,Americas,0 -officedepot.com,officedepot.com,inbound,019,Americas,2.6e-05 -olympiaedge.net,olympiaedge.net,inbound,019,Americas,0 -oneindia.in,infimail.com,inbound,019,Americas,0 -oneindia.in,mailurja.com,inbound,019,Americas,0 -onepatriotplace.com,britecast.com,inbound,019,Americas,0 -onestopplus.com,neolane.net,inbound,019,Americas,0 -onetravelspecials.com,onetravelspecials.com,inbound,019,Americas,0.365386 -online.com,cnet.com,inbound,019,Americas,0 -onthecitymail.org,onthecitymail.org,inbound,019,Americas,1 -oo155.com,bsftransmit7.com,inbound,019,Americas,0 -oo155.com,oo155.com,inbound,019,Americas,0 -openstack.org,openstack.org,inbound,019,Americas,0.997933 -openstackmail.com,infimail.com,inbound,019,Americas,0 -opentable.com,opentable.com,inbound,019,Americas,2e-06 -opinionoutpost.com,opinionoutpost.com,inbound,019,Americas,0 -opinionsquare.com,opinionsquare.com,inbound,019,Americas,0 -oprah.com,oprah.com,inbound,019,Americas,0 -opticsplanet.com,opticsplanet.com,inbound,019,Americas,0 -optonline.net,cv.net,inbound,019,Americas,0 -optonline.net,optonline.net,outbound,019,Americas,0 -oriental-trading.com,oriental-trading.com,inbound,019,Americas,0 -outlook.com,hotmail.{...},inbound,019,Americas,1 -outlook.com,hotmail.{...},outbound,019,Americas,1 -overnightprints.com,chtah.net,inbound,019,Americas,0 -ovuline.com,ovuline.com,inbound,019,Americas,1 -pagoda.com,zales.com,inbound,019,Americas,0 -pagseguro.com.br,uol.com.br,inbound,019,Americas,0 -pair.com,pair.com,inbound,019,Americas,0.893803 -palmscasinoresort.com,palmscasinoresort.com,inbound,019,Americas,0 -pampers.com,bfi0.com,inbound,019,Americas,0 -pandaresearch.com,pandaresearch.com,inbound,019,Americas,0 -pandora.com,pandora.com,inbound,019,Americas,1 -pandora.net,pandora.net,inbound,019,Americas,1 -parents.com,meredith.com,inbound,019,Americas,0 -parkmobileglobal.com,parkmobile.us,inbound,019,Americas,0 -path.com,path.com,inbound,019,Americas,1 -patriotupdate.com,inboxfirst.com,inbound,019,Americas,0 -payback.in,chtah.net,inbound,019,Americas,0 -paytm.com,paytm.com,inbound,019,Americas,1 -pbteen.com,pbteen.com,inbound,019,Americas,0 -pch.com,ed10.com,inbound,019,Americas,0 -pchfrontpage.com,ed10.com,inbound,019,Americas,0 -pchlotto.com,ed10.com,inbound,019,Americas,0 -pchplayandwin.com,ed10.com,inbound,019,Americas,0 -pchsearch.com,ed10.com,inbound,019,Americas,0 -pcmag.com,ittoolbox.com,inbound,019,Americas,0 -pcworld.com,pcworld.com,inbound,019,Americas,0 -pd25.com,pd25.com,inbound,019,Americas,1 -pearlsofwealth.com,pearlsofwealth.com,inbound,019,Americas,1 -peartreegreetings.com,rexcraft.com,inbound,019,Americas,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,019,Americas,0 -pepboys.com,pepboys.com,inbound,019,Americas,0 -pepperfry.com,epidm.net,inbound,019,Americas,0 -perfectpriceindia.com,infimail.com,inbound,019,Americas,0 -perfora.net,perfora.net,inbound,019,Americas,1 -permissionresearch.com,permissionresearch.com,inbound,019,Americas,0 -personalliberty.com,personalliberty.com,inbound,019,Americas,1 -pga.com,pga.com,inbound,019,Americas,0 -pgeveryday.com,bfi0.com,inbound,019,Americas,0 -phoenix.edu,phoenix.edu,inbound,019,Americas,0 -phsmtpbox.com,phsmtpbox.com,inbound,019,Americas,0 -pinterest.com,pinterest.com,inbound,019,Americas,1 -pivotaltracker.com,pivotaltracker.com,inbound,019,Americas,1 -pixable.com,pixable.com,inbound,019,Americas,1 -pizzahut.com,quikorder.com,inbound,019,Americas,0 -plexapp.com,plex.tv,inbound,019,Americas,1 -pmailus.com,patrontechnology.com,inbound,019,Americas,0 -pnc.com,messagelabs.com,inbound,019,Americas,0.997194 -pobox.com,pobox.com,inbound,019,Americas,0.975372 -pof.com,plentyoffish.co.uk,inbound,019,Americas,0 -pogo.com,pogo.com,inbound,019,Americas,0 -polyvore.com,polyvore.com,inbound,019,Americas,1 -postcardfromhell.com,cyberthugs.com,inbound,019,Americas,1 -potterybarn.com,potterybarn.com,inbound,019,Americas,0 -potterybarnkids.com,potterybarnkids.com,inbound,019,Americas,0 -preferredpetclub.com,preferredpetclub.com,inbound,019,Americas,0 -presslaff.net,dat-e-baseonline.com,inbound,019,Americas,0 -pressmartmail.com,pressmartmail.com,inbound,019,Americas,0 -priorityoneemail.com,priorityoneemail.com,inbound,019,Americas,0 -private-elist.com,private-elist.com,inbound,019,Americas,0 -progressive.com,progressive.com,inbound,019,Americas,0.999893 -promedmail.org,childrenshospital.org,inbound,019,Americas,1 -propertysolutions.com,propertysolutions.com,inbound,019,Americas,1 -prospectgeysercoop.com,prospectgeysercoop.com,inbound,019,Americas,1 -providesupport.com,providesupport.com,inbound,019,Americas,0.000103 -proxyvote.com,adp-ics.com,inbound,019,Americas,0.908635 -psu.edu,psu.edu,inbound,019,Americas,0.073843 -puffinmailer.com,zoothost.com,inbound,019,Americas,0 -purewow.com,purewow.com,inbound,019,Americas,0 -purlsmail.com,purlsmail.com,inbound,019,Americas,0 -pxsmail.com,pxsmail.com,inbound,019,Americas,0 -q.com,synacor.com,inbound,019,Americas,0.999734 -qemailserver.com,qemailserver.com,inbound,019,Americas,0 -qtropnews.com,qtropnews.com,inbound,019,Americas,1.7e-05 -qualicorp.com.br,qualicorp.com.br,inbound,019,Americas,1 -qualitysafelist.com,zoothost.com,inbound,019,Americas,0 -queopinas.com,confirmit.com,inbound,019,Americas,1 -quickrewards.net,quickrewards.net,inbound,019,Americas,0 -quikr.com,quikr.com,inbound,019,Americas,0 -quora.com,quora.com,inbound,019,Americas,1 -qvcemail.com,qvcemail.com,inbound,019,Americas,0 -radarsystems.net,radarsystems.net,inbound,019,Americas,1 -radiantretailapps.com,radiantretailapps.com,inbound,019,Americas,0 -radioshack.com,radioshack.com,inbound,019,Americas,0 -rapattoni.com,rapmls.com,inbound,019,Americas,0 -razerzone.com,chtah.net,inbound,019,Americas,0 -rbc.com,rbc.com,inbound,019,Americas,0.003061 -rdio.com,rdio.com,inbound,019,Americas,1 -realtor.org,realtor.org,inbound,019,Americas,0 -realtytrac.com,realtytrac.com,inbound,019,Americas,0.001681 -recipe.com,meredith.com,inbound,019,Americas,0 -recruit.net,recruit.net,inbound,019,Americas,0 -redbox.com,redbox.com,inbound,019,Americas,0 -redfin.com,redfin.com,inbound,019,Americas,0 -redtri.com,redtri.com,inbound,019,Americas,0 -reebokusnews.com,reebokusnews.com,inbound,019,Americas,0 -reebonz.com,ed10.com,inbound,019,Americas,0 -reebonz.com,reebonz.com,inbound,019,Americas,1 -registro.br,registro.br,inbound,019,Americas,0 -rent.com,rent.com,inbound,019,Americas,0.000397 -rentalcars.com,rentalcars.com,inbound,019,Americas,1 -renweb.com,renweb.com,inbound,019,Americas,0 -researchgate.net,researchgate.net,inbound,019,Americas,0 -restorationhardware.com,restorationhardware.com,inbound,019,Americas,0 -retailmenot.com,retailmenot.com,inbound,019,Americas,0 -reverbnation.com,reverbnation.com,inbound,019,Americas,0 -rewardme.in,bfi0.com,inbound,019,Americas,0 -ricardoeletro.com.br,allin.com.br,inbound,019,Americas,0 -rigzonemail.com,rigzonemail.com,inbound,019,Americas,0 -ripleyperu.com.pe,icommarketing.com,inbound,019,Americas,0 -riseup.net,riseup.net,inbound,019,Americas,1 -rivamail.com,mailurja.com,inbound,019,Americas,0 -rnmk.com,rnmk.com,inbound,019,Americas,0 -roamans.com,neolane.net,inbound,019,Americas,0 -rockwellcollins.com,rockwellcollins.com,inbound,019,Americas,1 -rpinow.org,app-info.net,inbound,019,Americas,0 -rr.com,rr.com,inbound,019,Americas,0.03696 -rr.com,rr.com,outbound,019,Americas,0 -rsgsv.net,rsgsv.net,inbound,019,Americas,0 -rsvpsv.net,rsvpsv.net,inbound,019,Americas,0 -rsvpsv.net,send.esp.br,inbound,019,Americas,0 -rsys2.com,amfam.com,inbound,019,Americas,0 -rsys2.com,cheaptickets.com,inbound,019,Americas,0 -rsys2.com,dishnetworkmail.com,inbound,019,Americas,0 -rsys2.com,e-comms.net,inbound,019,Americas,0 -rsys2.com,eharmony.com,inbound,019,Americas,0 -rsys2.com,fathead.com,inbound,019,Americas,0 -rsys2.com,intuit.com,inbound,019,Americas,0 -rsys2.com,kmart.com,inbound,019,Americas,0 -rsys2.com,kohlernews.com,inbound,019,Americas,0 -rsys2.com,lego.com,inbound,019,Americas,0 -rsys2.com,lenovo.com,inbound,019,Americas,0 -rsys2.com,modcloth.com,inbound,019,Americas,0 -rsys2.com,moxieinteractive.com,inbound,019,Americas,0 -rsys2.com,orbitz.com,inbound,019,Americas,0 -rsys2.com,payless.com,inbound,019,Americas,0 -rsys2.com,petsathome.com,inbound,019,Americas,0 -rsys2.com,quizzle.com,inbound,019,Americas,0 -rsys2.com,robeez.com,inbound,019,Americas,0 -rsys2.com,rsys1.com,inbound,019,Americas,0 -rsys2.com,rsys2.com,inbound,019,Americas,0 -rsys2.com,rsys3.com,inbound,019,Americas,0 -rsys2.com,rsys4.com,inbound,019,Americas,0 -rsys2.com,saucony.com,inbound,019,Americas,0 -rsys2.com,sears.com,inbound,019,Americas,0 -rsys2.com,shopbop.com,inbound,019,Americas,0 -rsys2.com,southwest.com,inbound,019,Americas,0 -rsys2.com,speeddatemail.com,inbound,019,Americas,0 -rsys2.com,thecompanystore.com,inbound,019,Americas,0 -rsys2.com,theknot.com,inbound,019,Americas,0 -rsys5.com,alibris.com,inbound,019,Americas,0 -rsys5.com,allstate-email.com,inbound,019,Americas,0 -rsys5.com,beachmint.com,inbound,019,Americas,0 -rsys5.com,belleandclive.com,inbound,019,Americas,0 -rsys5.com,br.dk,inbound,019,Americas,0 -rsys5.com,charlotterusse.com,inbound,019,Americas,0 -rsys5.com,comixology.com,inbound,019,Americas,0 -rsys5.com,cottonon.com,inbound,019,Americas,0 -rsys5.com,ediblearrangements.com,inbound,019,Americas,0 -rsys5.com,emailworldmarket.com,inbound,019,Americas,0 -rsys5.com,farfetch.com,inbound,019,Americas,0 -rsys5.com,frhiemailcommunications.com,inbound,019,Americas,0 -rsys5.com,harryanddavid.com,inbound,019,Americas,0 -rsys5.com,hollandandbarrett.com,inbound,019,Americas,0 -rsys5.com,icing.com,inbound,019,Americas,0 -rsys5.com,indigo.ca,inbound,019,Americas,0 -rsys5.com,jabong.com,inbound,019,Americas,0 -rsys5.com,jcrew.com,inbound,019,Americas,0 -rsys5.com,jjill.com,inbound,019,Americas,0 -rsys5.com,kanui.com.br,inbound,019,Americas,0 -rsys5.com,kirklands.com,inbound,019,Americas,0 -rsys5.com,lanebryant.com,inbound,019,Americas,0 -rsys5.com,lazada.com,inbound,019,Americas,0 -rsys5.com,leapfrog.com,inbound,019,Americas,0 -rsys5.com,llbean.com,inbound,019,Americas,0 -rsys5.com,lojascolombo.com.br,inbound,019,Americas,0 -rsys5.com,madewell.com,inbound,019,Americas,0 -rsys5.com,magazineluiza.com.br,inbound,019,Americas,0 -rsys5.com,missguided.co.uk,inbound,019,Americas,0 -rsys5.com,moma.org,inbound,019,Americas,0 -rsys5.com,nationalgeographic.com,inbound,019,Americas,0 -rsys5.com,neat.com,inbound,019,Americas,0 -rsys5.com,newbalance.com,inbound,019,Americas,0 -rsys5.com,news-voeazul.com.br,inbound,019,Americas,0 -rsys5.com,nordstrom.com,inbound,019,Americas,0 -rsys5.com,normthompson.com,inbound,019,Americas,0 -rsys5.com,novomundo.com.br,inbound,019,Americas,0 -rsys5.com,ourdeal.com.au,inbound,019,Americas,0 -rsys5.com,pier1.com,inbound,019,Americas,0 -rsys5.com,productmadness.com,inbound,019,Americas,0 -rsys5.com,rainbowshops.com,inbound,019,Americas,0 -rsys5.com,rei.com,inbound,019,Americas,0 -rsys5.com,roadrunnersports.com,inbound,019,Americas,0 -rsys5.com,rosettastone.com,inbound,019,Americas,0 -rsys5.com,seamless.com,inbound,019,Americas,0 -rsys5.com,serenaandlily.com,inbound,019,Americas,0 -rsys5.com,smiles.com.br,inbound,019,Americas,0 -rsys5.com,soubarato.com.br,inbound,019,Americas,0 -rsys5.com,strava.com,inbound,019,Americas,0 -rsys5.com,submarino.com.br,inbound,019,Americas,0 -rsys5.com,thewalkingcompany.com,inbound,019,Americas,0 -rsys5.com,tigerdirect.com,inbound,019,Americas,0 -rsys5.com,udemy.com,inbound,019,Americas,0 -rsys5.com,vitaminshoppe.com,inbound,019,Americas,0 -rsys5.com,vpusa.com,inbound,019,Americas,0 -rsys5.com,walmart.com.br,inbound,019,Americas,0 -rsys5.com,worldofwatches.com,inbound,019,Americas,0 -rsys5.com,xfinity.com,inbound,019,Americas,0 -rue21email.com,rue21email.com,inbound,019,Americas,0 -ruum.com,ruum.com,inbound,019,Americas,0 -saavn.com,saavn.com,inbound,019,Americas,1 -safelistextreme.com,quantumsafelist.com,inbound,019,Americas,0 -safelistpro.com,safelistpro.com,inbound,019,Americas,0.00014 -safeway.com,chtah.com,inbound,019,Americas,0 -sailthru.com,sailthru.com,inbound,019,Americas,0 -saks.com,saks.com,inbound,019,Americas,0 -saksoff5th.com,saksoff5th.com,inbound,019,Americas,0 -salememail.net,salememail.net,inbound,019,Americas,0 -salesforce.com,postini.com,inbound,019,Americas,0.816952 -salesforce.com,salesforce.com,inbound,019,Americas,1 -salesforce.com,salesforce.com,outbound,019,Americas,1 -salliemae.com,salliemae.com,inbound,019,Americas,1 -salsalabs.net,salsalabs.net,inbound,019,Americas,0 -samashmusic.com,wc09.net,inbound,019,Americas,0 -samsclub.com,m0.net,inbound,019,Americas,0 -sans.org,sans.org,inbound,019,Americas,0.497629 -santander.cl,santander.cl,inbound,019,Americas,0.999992 -santander.cl,santandersantiago.cl,inbound,019,Americas,1 -sassieshop.com,sassieshop.com,inbound,019,Americas,0.025887 -savelivefresh.com,livesavemail.com,inbound,019,Americas,1 -savingstar.com,savingstar.com,inbound,019,Americas,1 -sbcglobal.net,yahoo.{...},inbound,019,Americas,1 -sbcglobal.net,yahoodns.net,outbound,019,Americas,1 -sc.com,messagelabs.com,inbound,019,Americas,0.996156 -schwab.com,schwab.com,inbound,019,Americas,0.025055 -seaworld.com,seaworld.com,inbound,019,Americas,0 -securence.com,securence.com,inbound,019,Americas,0.670246 -secureserver.net,secureserver.net,inbound,019,Americas,4.4e-05 -seekingalpha.com,seekingalpha.com,inbound,019,Americas,1 -seekingalpha.com,sendgrid.net,inbound,019,Americas,1 -senate.gov,senate.gov,inbound,019,Americas,0.992994 -sendearnings.com,sendearnings.com,inbound,019,Americas,0 -sendgrid.info,sendgrid.net,inbound,019,Americas,1 -sendgrid.me,sendgrid.net,inbound,019,Americas,1 -sendlane.com,sendlane.com,inbound,019,Americas,0 -serpadres.es,chtah.net,inbound,019,Americas,0 -service-now.com,postini.com,inbound,019,Americas,0.9858 -service-now.com,service-now.com,inbound,019,Americas,0.998562 -sfimg.com,sfimarketing.com,inbound,019,Americas,0 -sfly.com,shutterfly.com,inbound,019,Americas,0 -shaadi.com,shaadi.com,inbound,019,Americas,0.00037 -shadowshopper.com,shadowshopper.com,inbound,019,Americas,5.6e-05 -shaw.ca,shaw.ca,inbound,019,Americas,0 -shaw.ca,shaw.ca,outbound,019,Americas,1 -sheplers.com,sheplers.com,inbound,019,Americas,0 -shiftplanning.com,shiftplanning.com,inbound,019,Americas,0 -shiksha.com,shiksha.com,inbound,019,Americas,0 -shoedazzle.com,shoedazzle.com,inbound,019,Americas,0 -shoes.com,famousfootwear.com,inbound,019,Americas,0 -shopbonton.com,shopbonton.com,inbound,019,Americas,0 -shopcluesemail.com,shopcluesemail.com,inbound,019,Americas,0 -shopcluesmail.com,shopcluesmail.com,inbound,019,Americas,0 -shophq.com,shophq.com,inbound,019,Americas,0 -shopkick.com,shopkick.com,inbound,019,Americas,0 -shopko.com,shopko.com,inbound,019,Americas,0 -shoppersoptimum.ca,thindata.net,inbound,019,Americas,0 -shoptime.com,shoptime.com,inbound,019,Americas,0 -shtyle.fm,shtyle.fm,inbound,019,Americas,0 -sierratradingpost.com,sierratradingpost.com,inbound,019,Americas,0.000885 -sigmabeauty.com,lstrk.net,inbound,019,Americas,1 -sii.cl,sii.cl,inbound,019,Americas,0 -simplyhired.com,simplyhired.com,inbound,019,Americas,0 -siriusxm.com,xmradio.com,inbound,019,Americas,0 -sitecore-mailer.com,sendlabs.com,inbound,019,Americas,0 -sittercity.com,sittercity.com,inbound,019,Americas,0 -sixflags.com,sixflags.com,inbound,019,Americas,0 -skillpages-mailer.com,sendlabs.com,inbound,019,Americas,0 -skillpages-mailer.com,skillpagesmail.com,inbound,019,Americas,0 -sky.com,sky.com,inbound,019,Americas,0 -skype.com,delivery.net,inbound,019,Americas,0 -sld.cu,sld.cu,inbound,019,Americas,1 -slidesharemail.com,newslettergrid.com,inbound,019,Americas,1 -slidesharemail.com,slidesharemail.com,inbound,019,Americas,1 -smartbrief.com,smartbrief.com,inbound,019,Americas,0 -smartdraw.com,smartdraw.com,inbound,019,Americas,0 -smartertravel.com,smartertravelmedia.com,inbound,019,Americas,0.021434 -smartphoneexperts.com,mailgun.net,inbound,019,Americas,1 -snapretail.com,snapretail.com,inbound,019,Americas,1 -socialappsmail.com,socialappsmail.com,inbound,019,Americas,1 -solesociety.com,bronto.com,inbound,019,Americas,0 -solosenders.com,megasenders.com,inbound,019,Americas,8.2e-05 -solosenders.com,traxweb.org,inbound,019,Americas,0 -soma.com,soma.com,inbound,019,Americas,0 -someecards.com,someecards.com,inbound,019,Americas,0 -songkick.com,songkick.com,inbound,019,Americas,1 -sony-latin.com,sony-latin.com,inbound,019,Americas,0.01735 -sonyentertainmentnetwork.com,sonyentertainmentnetwork.com,inbound,019,Americas,0 -sourceforge.net,sourceforge.net,inbound,019,Americas,1 -southwest.com,southwest.com,inbound,019,Americas,0 -sp.gov.br,sp.gov.br,inbound,019,Americas,0.496357 -sparkpeople.com,sparkpeople.com,inbound,019,Americas,0 -spectersoft.com,spectersoft.com,inbound,019,Americas,0 -spencersonline.com,spencersonline.com,inbound,019,Americas,0 -spiritairlines.com,ctd004.net,inbound,019,Americas,0 -spiritairlines.com,ctd005.net,inbound,019,Americas,0 -splitwise.com,splitwise.com,inbound,019,Americas,1 -sportlobster.com,sportlobster.com,inbound,019,Americas,1 -sportsdirect.com,sportsdirect.com,inbound,019,Americas,0 -sportsline.com,cbsig.net,inbound,019,Americas,1 -sportsmansguide.com,sportsmansguide.com,inbound,019,Americas,0.736903 -sprint.com,m0.net,inbound,019,Americas,0 -squareup.com,squareup.com,inbound,019,Americas,0.986452 -stanford.edu,highwire.org,inbound,019,Americas,1 -stanford.edu,stanford.edu,inbound,019,Americas,0.93025 -stansberryresearch.com,stansberryresearch.com,inbound,019,Americas,0.001541 -staples.com,staples.com,inbound,019,Americas,0.006379 -starbucks.com,starbucks.com,inbound,019,Americas,0 -stardockcorporation.com,stardockcorporation.com,inbound,019,Americas,0 -stardockentertainment.info,stardockentertainment.info,inbound,019,Americas,0 -startribune.com,startribune.com,inbound,019,Americas,0.001728 -startwire.com,jobsreport.com,inbound,019,Americas,1 -startwire.com,startwire.com,inbound,019,Americas,1 -state.gov,state.gov,inbound,019,Americas,0 -statefarm.com,statefarm.com,inbound,019,Americas,1 -steinmart.com,steinmart.com,inbound,019,Americas,0 -stevemadden.com,stevemadden.com,inbound,019,Americas,0 -streetauthoritydaily.com,streetauthoritydaily.com,inbound,019,Americas,0 -stumblemail.com,stumblemail.com,inbound,019,Americas,1 -suafaturanet.com.br,suafaturanet.com.br,inbound,019,Americas,0.995846 -subscribermail.com,subscribermail.com,inbound,019,Americas,0 -sungard.com,postini.com,inbound,019,Americas,0.498265 -sunwingvacationinfo.ca,sunwingvacationinfo.ca,inbound,019,Americas,1 -superbalist.com,sailthru.com,inbound,019,Americas,0 -supersafemailer.com,zoothost.com,inbound,019,Americas,0 -supremelist.com,onlinehome-server.com,inbound,019,Americas,1 -surlatable.com,surlatable.com,inbound,019,Americas,0 -surveyjobopportunities.com,surveyjobopportunities.com,inbound,019,Americas,3.7e-05 -surveymonkey.com,surveymonkey.com,inbound,019,Americas,0 -surveysavvy.com,surveysavvy.com,inbound,019,Americas,0 -sylectus.com,sylectus.com,inbound,019,Americas,0.361297 -sympatico.ca,hotmail.{...},outbound,019,Americas,1 -taggedmail.com,taggedmail.com,inbound,019,Americas,0 -tamu.edu,tamu.edu,inbound,019,Americas,0.249876 -tangeroutletsusa.com,bronto.com,inbound,019,Americas,0 -target-safelist.com,safelistpro.com,inbound,019,Americas,0.000215 -targetproblaster.com,targetproblaster.com,inbound,019,Americas,0 -targetx.com,targetx.com,inbound,019,Americas,0 -tarot.com,tarot.com,inbound,019,Americas,0 -tastefullysimpleparty.com,bigfootinteractive.com,inbound,019,Americas,0 -tastingtable.com,tastingtable.com,inbound,019,Americas,0 -taxi4sure.net,infimail.com,inbound,019,Americas,0 -teach12.net,teach12.net,inbound,019,Americas,0 -techtarget.com,techtarget.com,inbound,019,Americas,0.001771 -telus.net,telus.net,inbound,019,Americas,0.006226 -ten24mail.com,ten24mail.com,inbound,019,Americas,0 -terra.com,terra.com,inbound,019,Americas,0.000463 -terra.com.br,terra.com,inbound,019,Americas,0 -terra.com.br,terra.com,outbound,019,Americas,0 -tesco.com,tesco.com,inbound,019,Americas,0 -testfunda.com,testfunda.com,inbound,019,Americas,0 -textnow.me,textnow.me,inbound,019,Americas,0 -tgw.com,tgw.com,inbound,019,Americas,0 -theanimalrescuesite.com,theanimalrescuesite.com,inbound,019,Americas,0 -theatermania.com,wc09.net,inbound,019,Americas,0 -thebay.com,thebay.com,inbound,019,Americas,0 -thegrommet.com,lstrk.net,inbound,019,Americas,1 -thehut.com,thehut.com,inbound,019,Americas,0 -thelimited.com,thelimited.com,inbound,019,Americas,0 -themailbagsafelist.com,thomas-j-brown.com,inbound,019,Americas,0 -theoutnet.com,theoutnet.com,inbound,019,Americas,0 -theskimm.com,theskimm.com,inbound,019,Americas,0 -thesovereigninvestor.com,sovereignsociety.com,inbound,019,Americas,0 -thirtyonegifts.com,thirtyonegifts.com,inbound,019,Americas,0 -thoughtful-mind.com,thoughtful-mind.com,inbound,019,Americas,0 -thumbtack.com,thumbtack.com,inbound,019,Americas,1 -ticketmaster.com,ticketmaster.com,inbound,019,Americas,0.769059 -tmart.com,chtah.net,inbound,019,Americas,0 -tmp.com,tmpw.com,inbound,019,Americas,0 -toluna.com,toluna.com,inbound,019,Americas,0 -tomtommailer.com,tomtommailer.com,inbound,019,Americas,0 -topspin.net,topspin.net,inbound,019,Americas,1 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,019,Americas,0 -touchbase2.com,infimail.com,inbound,019,Americas,0 -touchbase2.com,mailurja.com,inbound,019,Americas,0 -touchbasepro.com,touchbasepro.com,inbound,019,Americas,0 -townhallmail.com,townhallmail.com,inbound,019,Americas,0 -townsquaremedia.info,sailthru.com,inbound,019,Americas,0 -toysrus.com,epsl1.com,inbound,019,Americas,0 -trafficboostermailer.com,trafficboostermailer.com,inbound,019,Americas,1 -trafficleads2incomevm.com,zoothost.com,inbound,019,Americas,0 -trafficprolist.com,thomas-j-brown.com,inbound,019,Americas,0 -trialsmith.com,membercentral.com,inbound,019,Americas,0 -tricaemidia.com.br,tricaemidia.com.br,inbound,019,Americas,0 -tripit.com,tripit.com,inbound,019,Americas,0 -tuesdaymorningmail.com,tuesdaymorningmail.com,inbound,019,Americas,0 -tumblr.com,tumblr.com,inbound,019,Americas,1 -twitch.tv,justin.tv,inbound,019,Americas,0 -uber.com,uber.com,inbound,019,Americas,1 -ubi.com,ubi.com,inbound,019,Americas,0 -ubivox.com,ubivox.com,inbound,019,Americas,0.935106 -ucla.edu,ucla.edu,inbound,019,Americas,0.806204 -uhaul.com,uhaul.com,inbound,019,Americas,0.997947 -uiuc.edu,illinois.edu,inbound,019,Americas,0.999815 -ultimateadsites.net,ultimateadsites.net,inbound,019,Americas,1 -umd.edu,umd.edu,inbound,019,Americas,0.021709 -umn.edu,umn.edu,inbound,019,Americas,0.966964 -umpiredigital.com,inboxmarketer.com,inbound,019,Americas,0 -united.com,coair.com,inbound,019,Americas,1 -united.com,united.com,inbound,019,Americas,0 -universalorlando.com,universalorlando.com,inbound,019,Americas,0 -unrollmail.com,unrollmail.com,inbound,019,Americas,1 -uol.com.br,uol.com.br,inbound,019,Americas,0 -uol.com.br,uol.com.br,outbound,019,Americas,0 -upenn.edu,upenn.edu,inbound,019,Americas,0.16521 -upromise.com,delivery.net,inbound,019,Americas,0 -ups.com,ups.com,inbound,019,Americas,0.997955 -urbanoutfitters.com,freepeople.com,inbound,019,Americas,0 -usaa.com,usaa.com,inbound,019,Americas,0.072606 -usafisnews.org,usafisnews.org,inbound,019,Americas,0 -usajobs.gov,opm.gov,inbound,019,Americas,0.931948 -usc.edu,usc.edu,inbound,019,Americas,0.300314 -uscourts.gov,uscourts.gov,inbound,019,Americas,0 -uslargestsafelist.com,zoothost.com,inbound,019,Americas,0 -usndr.com,unisender.com,inbound,019,Americas,1 -usp.br,usp.br,inbound,019,Americas,0.044595 -usps.com,usps.gov,inbound,019,Americas,0.909591 -usps.gov,usps.gov,inbound,019,Americas,0.940315 -ustream.tv,mailgun.net,inbound,019,Americas,1 -ustream.tv,mailgun.us,inbound,019,Americas,1 -utfsm.cl,utfsm.cl,inbound,019,Americas,0.000352 -vacationstogo.com,vacationstogo.com,inbound,019,Americas,0 -vagas.com.br,vagas.com.br,inbound,019,Americas,0 -valuedopinions.co.uk,researchnow-usa.com,inbound,019,Americas,1 -vanheusenrewards.com,vanheusenrewards.com,inbound,019,Americas,0 -velocityfrequentflyer.com,virginaustralia.com,inbound,019,Americas,0 -vente-exclusive.com,vente-exclusive.com,inbound,019,Americas,0 -venusswimwear.net,venusswimwear.net,inbound,019,Americas,0 -verabradleymail.com,verabradleymail.com,inbound,019,Americas,0 -verizon.net,verizon.net,inbound,019,Americas,0.00713 -verizon.net,verizon.net,outbound,019,Americas,0 -verizonwireless.com,verizonwireless.com,inbound,019,Americas,0.972245 -vhmnetworkemail.com,jobdiagnosis.com,inbound,019,Americas,0 -viagogo.com,chtah.net,inbound,019,Americas,0 -viajanet.com.br,viajanet.com.br,inbound,019,Americas,0 -victoriassecret.com,victoriassecret.com,inbound,019,Americas,0 -videotron.ca,videotron.ca,inbound,019,Americas,0.005886 -vikingrivercruises.com,bfi0.com,inbound,019,Americas,0 -vimeo.com,vimeo.com,inbound,019,Americas,0 -vindale.com,vindale.com,inbound,019,Americas,0 -vipvoice.com,npdor.com,inbound,019,Americas,0 -viraladmagnet.com,viraladmagnet.com,inbound,019,Americas,1 -virginia.edu,virginia.edu,inbound,019,Americas,0.055153 -virtualtarget.com.br,virtualtarget.com.br,inbound,019,Americas,0 -vitacost.com,vitacost.com,inbound,019,Americas,0 -vitaladviews.com,zoothost.com,inbound,019,Americas,0 -vivastreet.com,viwii.net,inbound,019,Americas,0 -votervoice.net,votervoice.net,inbound,019,Americas,0 -vresp.com,verticalresponse.com,inbound,019,Americas,0 -vt.edu,vt.edu,inbound,019,Americas,0.978548 -vtext.com,vtext.com,inbound,019,Americas,0 -vtext.com,vtext.com,outbound,019,Americas,1 -vuezone.com,vuezone.com,inbound,019,Americas,0 -vzwpix.com,vtext.com,inbound,019,Americas,0 -wagjag.com,wagjag.com,inbound,019,Americas,0 -walgreens.com,walgreens.com,inbound,019,Americas,0.006128 -wallst.com,wallst.com,inbound,019,Americas,0 -wallstreetdaily.com,wallstreetdaily.com,inbound,019,Americas,0 -waterstones.com,waterstones.com,inbound,019,Americas,0 -wattpad.com,wattpad.com,inbound,019,Americas,1 -way2movies.net,way2movies.net,inbound,019,Americas,0 -wayfair.com,csnstores.com,inbound,019,Americas,0 -webs.com,epsl1.com,inbound,019,Americas,0 -websaver.ca,websaver.ca,inbound,019,Americas,0 -websitesettings.com,stabletransit.com,inbound,019,Americas,0 -websitewelcome.com,websitewelcome.com,inbound,019,Americas,1 -webstars2k.com,webstars2k.com,inbound,019,Americas,1 -weebly.com,weeblymail.com,inbound,019,Americas,0 -wellsfargo.com,wellsfargo.com,inbound,019,Americas,1.0 -wellsfargoadvisors.com,wellsfargo.com,inbound,019,Americas,1 -westelm.com,westelm.com,inbound,019,Americas,0 -westmarine.com,westmarine.com,inbound,019,Americas,0 -wetransfer.com,wetransfer.com,inbound,019,Americas,1 -wetsealnewsletter.com,wetsealnewsletter.com,inbound,019,Americas,0 -whatcounts.com,wc09.net,inbound,019,Americas,0 -whentowork.com,whentowork.com,inbound,019,Americas,0 -wikia.com,wikia.com,inbound,019,Americas,1 -williams-sonoma.com,williams-sonoma.com,inbound,019,Americas,0 -windstream.net,windstream.net,inbound,019,Americas,0.002172 -wine.com,wine.com,inbound,019,Americas,0 -winkalmail.com,fnbox.com,inbound,019,Americas,0 -wisdomitservices.com,infimail.com,inbound,019,Americas,0 -wldemail-mailer.com,wldemail.com,inbound,019,Americas,0 -womanwithin.com,womanwithin.com,inbound,019,Americas,0 -woodforest.com,woodforest.com,inbound,019,Americas,1 -workingincanada.gc.ca,sdc-dsc.gc.ca,inbound,019,Americas,0 -worldsingles.co,worldsingles.com,inbound,019,Americas,0 -wotif.com,whatcounts.com,inbound,019,Americas,0 -wp.com,wordpress.com,inbound,019,Americas,0 -wuaki.tv,chtah.net,inbound,019,Americas,0 -wustl.edu,app-info.net,inbound,019,Americas,0 -wwe.com,wwe.com,inbound,019,Americas,8e-06 -xen.org,xen.org,inbound,019,Americas,1 -xmeeting.com,xmeeting.com,inbound,019,Americas,1 -xmr3.com,messagereach.com,inbound,019,Americas,0.999933 -xoom.com,xoom.com,inbound,019,Americas,0 -xpnews.com.br,xpnews.com.br,inbound,019,Americas,1 -yahoo.{...},postini.com,inbound,019,Americas,0.664066 -yahoo.{...},yahoo.{...},inbound,019,Americas,0.999 -yahoo.{...},yahoodns.net,outbound,019,Americas,1 -yelp.com,yelpcorp.com,inbound,019,Americas,0 -yipit.com,yipit.com,inbound,019,Americas,1 -ymail.com,yahoo.{...},inbound,019,Americas,1 -ymail.com,yahoodns.net,outbound,019,Americas,1 -yoox.com,yoox.com,inbound,019,Americas,0 -youmail.com,youmail.com,inbound,019,Americas,0 -youreletters3.com,equitymaster.com,inbound,019,Americas,0 -yourezads.com,yourezads.com,inbound,019,Americas,0.002974 -yourezlist.com,simplicityads.com,inbound,019,Americas,9.5e-05 -yourhostingaccount.com,yourhostingaccount.com,inbound,019,Americas,0 -youversion.com,youversion.com,inbound,019,Americas,1 -zacks.com,zacks.com,inbound,019,Americas,0.999687 -zara.com,cheetahmail.com,inbound,019,Americas,0 -zendesk.com,zdsys.com,inbound,019,Americas,1 -zgalleriestyle.com,zgalleriestyle.com,inbound,019,Americas,0 -zibmail.info,zibmail.info,inbound,019,Americas,9e-06 -zinio.com,zinio.com,inbound,019,Americas,0 -zipalerts.com,zipalerts.com,inbound,019,Americas,1 -ziprealty.com,ziprealty.com,inbound,019,Americas,0.994545 -ziprecruiter.com,ziprecruiter.com,inbound,019,Americas,1 -zivamewear.com,infimail.com,inbound,019,Americas,0 -zoothost.com,zoothost.com,inbound,019,Americas,0.107168 -zorpia.com,zorpia.com,inbound,019,Americas,1 -zulily.com,zulily.com,inbound,019,Americas,0 -zzounds.com,zzounds.com,inbound,019,Americas,0 -0101.co.jp,0101.co.jp,inbound,142,Asia,0 -04auto.biz,01auto.biz,inbound,142,Asia,0 -123.com.tw,123.com.tw,inbound,142,Asia,0 -160by2.us,160by2.us,inbound,142,Asia,0 -163.com,163.com,inbound,142,Asia,0.534088 -163.com,netease.com,outbound,142,Asia,1 -17life.com.tw,17life.com.tw,inbound,142,Asia,0 -1lejend.com,asumeru.com,inbound,142,Asia,0 -1lejend.com,asumeru001.com,inbound,142,Asia,0 -33go.com.tw,33go.com.tw,inbound,142,Asia,0 -518.com.tw,518.com.tw,inbound,142,Asia,0 -7net.com.tw,7net.com.tw,inbound,142,Asia,0 -ab0.jp,altovision.co.jp,inbound,142,Asia,0 -activetrail.com,atmailsvr.net,inbound,142,Asia,0 -activetrail.com,mymarketing.co.il,inbound,142,Asia,0 -adchiever.com,kinder-rash-marketing.com,inbound,142,Asia,0 -adityabirla.com,adityabirla.com,inbound,142,Asia,0.00615 -agora.co.il,1host.co.il,inbound,142,Asia,0 -airtel.com,airtel.in,inbound,142,Asia,0.064377 -alertsindia.in,alertsindia.in,inbound,142,Asia,1 -alibaba.com,alibaba.com,inbound,142,Asia,0 -aliexpress.com,alibaba.com,inbound,142,Asia,0 -alipay.com,alipay.com,inbound,142,Asia,1 -alljob.co.il,alljob.co.il,inbound,142,Asia,0 -amazon.{...},amazon.{...},inbound,142,Asia,0 -ana.co.jp,ana.co.jp,inbound,142,Asia,0.534416 -anghami.com,mailgun.net,inbound,142,Asia,1 -apple.com,apple.com,inbound,142,Asia,0.999817 -artscow.com,dyxnet.com,inbound,142,Asia,0.000355 -asus.com,asus.com,inbound,142,Asia,0 -baycrews.co.jp,webcas.net,inbound,142,Asia,0 -beamtele.com,beamtele.com,inbound,142,Asia,0 -beanfun.com,beanfun.com,inbound,142,Asia,1 -belluna.net,belluna.net,inbound,142,Asia,0 -betrend.com,betrend.com,inbound,142,Asia,0 -bharatmatrimony.com,bharatmatrimony.com,inbound,142,Asia,1 -blayn.jp,bserver.jp,inbound,142,Asia,0 -bme.jp,bserver.jp,inbound,142,Asia,0 -bookoffonline.co.jp,bookoffonline.co.jp,inbound,142,Asia,0 -brands4friends.jp,webcas.net,inbound,142,Asia,0 -brandsfever.com,mailgun.net,inbound,142,Asia,1 -buyma.com,buyma.com,inbound,142,Asia,0 -cam2life.com,hinet.net,inbound,142,Asia,0 -camsonline.com,camsonline.com,inbound,142,Asia,0.136261 -careesma.in,careesma.in,inbound,142,Asia,0 -ccavenue.com,avenues.info,inbound,142,Asia,1 -chance.com,data-hotel.net,inbound,142,Asia,0 -chiangcn.com,chiangcn.com,inbound,142,Asia,0 -chinatrust.com.tw,chinatrust.com.tw,inbound,142,Asia,0.001272 -cityheaven.net,cityheaven.net,inbound,142,Asia,0 -clickmailer.jp,clickmailer.jp,inbound,142,Asia,9e-05 -cocacola.co.jp,cocacola.co.jp,inbound,142,Asia,0 -combzmail.jp,combzmail.jp,inbound,142,Asia,0 -communitymatrimony.com,communitymatrimony.com,inbound,142,Asia,1 -conrepmail.com,conrepmail.com,inbound,142,Asia,0 -cookpad.com,cookpad.com,inbound,142,Asia,0 -cpc.gov.in,cpc.gov.in,inbound,142,Asia,0 -crmstyle.com,crmstyle.com,inbound,142,Asia,0 -crocos.jp,crocos.jp,inbound,142,Asia,0 -ctrip.com,ctrip.com,inbound,142,Asia,0.019448 -cuenote.jp,cuenote.jp,inbound,142,Asia,0 -cw.com.tw,cw.com.tw,inbound,142,Asia,0.002535 -cyberlinkmember.com,cyberlinkmember.com,inbound,142,Asia,0 -dena.ne.jp,dena.ne.jp,inbound,142,Asia,0.000249 -dietnavi.com,data-hotel.net,inbound,142,Asia,0 -dip-net.co.jp,dip-net.co.jp,inbound,142,Asia,0 -directresponsemanager.com,wide.ne.jp,inbound,142,Asia,0 -disc.co.jp,disc.co.jp,inbound,142,Asia,0.001051 -dishtv.co.in,dishtv.co.in,inbound,142,Asia,0.047664 -disqus.net,disqus.net,inbound,142,Asia,0 -dks.com.tw,dks.com.tw,inbound,142,Asia,0.018134 -dmm.com,dmm.com,inbound,142,Asia,0 -docomo.ne.jp,docomo.ne.jp,inbound,142,Asia,0 -docomo.ne.jp,docomo.ne.jp,outbound,142,Asia,0 -dreammail.ne.jp,dreammail.jp,inbound,142,Asia,0 -drushim.co.il,drushim.co.il,inbound,142,Asia,0 -dstyleweb.com,dstyleweb.com,inbound,142,Asia,0.001099 -ec21.com,ec21.com,inbound,142,Asia,0.002263 -ecnavi.jp,ecnavi.jp,inbound,142,Asia,0 -en-japan.com,en-japan.com,inbound,142,Asia,0.000204 -eonet.ne.jp,eonet.ne.jp,inbound,142,Asia,0.99955 -epaper.com.tw,epaper.com.tw,inbound,142,Asia,0 -eplus.jp,eplus.jp,inbound,142,Asia,0 -eslitebooks.com,eslitebooks.com,inbound,142,Asia,0 -euromsg.net,euromsg.net,inbound,142,Asia,0 -evaair.com,evaair.com,inbound,142,Asia,0.007363 -ezweb.ne.jp,ezweb.ne.jp,inbound,142,Asia,0.282076 -ezweb.ne.jp,ezweb.ne.jp,outbound,142,Asia,0 -farmersonly.com,mailgun.net,inbound,142,Asia,1 -felissimo.jp,felissimo.jp,inbound,142,Asia,0 -finansbank.com.tr,finansbank.com.tr,inbound,142,Asia,0.927 -fmworld.net,fmworld.net,inbound,142,Asia,0 -fofa.jp,mpme.jp,inbound,142,Asia,0 -freeml.com,gmo-media.jp,inbound,142,Asia,0 -ftchinese.com,ftchinese.com,inbound,142,Asia,0 -fubonshop.com,fubonshop.com,inbound,142,Asia,0 -gamecity.ne.jp,gamecity.ne.jp,inbound,142,Asia,0 -garanti.com.tr,garanti.com.tr,inbound,142,Asia,0.421936 -gilt.jp,gilt.jp,inbound,142,Asia,0 -globalsources.com,globalsources.com,inbound,142,Asia,0.003024 -globetel.com.ph,globetel.com.ph,inbound,142,Asia,1 -gmail.com,asianet.co.th,inbound,142,Asia,0.998511 -gmail.com,au-net.ne.jp,inbound,142,Asia,1 -gmail.com,bbtec.net,inbound,142,Asia,1 -gmail.com,data-hotel.net,inbound,142,Asia,0.000607 -gmail.com,hinet.net,inbound,142,Asia,0.973114 -gmail.com,mtnl.net.in,inbound,142,Asia,0.999527 -gmail.com,netvigator.com,inbound,142,Asia,0.818263 -gmail.com,ocn.ne.jp,inbound,142,Asia,0.99466 -gmail.com,panda-world.ne.jp,inbound,142,Asia,1 -gmail.com,seed.net.tw,inbound,142,Asia,0.992252 -gmail.com,singnet.com.sg,inbound,142,Asia,0.968389 -gmail.com,totbb.net,inbound,142,Asia,0.999876 -gmo.jp,gmo-media.jp,inbound,142,Asia,0 -gmoes.jp,gmoes.jp,inbound,142,Asia,0 -gmt.ne.jp,gmt.ne.jp,inbound,142,Asia,0 -gnavi.co.jp,gnavi.co.jp,inbound,142,Asia,0.002441 -gomaji.com,gomaji.com,inbound,142,Asia,0 -gree.jp,gree.jp,inbound,142,Asia,0 -groupon.{...},groupon.{...},inbound,142,Asia,0 -gunosy.com,gunosy.com,inbound,142,Asia,0.998682 -gurunavi.jp,gurunavi.jp,inbound,142,Asia,0 -hdfcbank.com,powerelay.com,inbound,142,Asia,1 -hdfcbank.net,powerelay.com,inbound,142,Asia,1 -hdfcbank.net,quickvmail.com,inbound,142,Asia,0 -heteml.jp,heteml.jp,inbound,142,Asia,0.950296 -hinet.net,hinet.net,inbound,142,Asia,0.007064 -hinet.net,hinet.net,outbound,142,Asia,0.00565 -home.ne.jp,zaq.ne.jp,inbound,142,Asia,9e-06 -i-part.com.tw,i-part.com.tw,inbound,142,Asia,0.001865 -ibps.in,sify.net,inbound,142,Asia,0 -ibpsorg.org,sify.net,inbound,142,Asia,0 -icicibank.com,icicibank.com,inbound,142,Asia,0.009835 -icicisecurities.com,icicibank.com,inbound,142,Asia,0.000803 -icloud.com,apple.com,inbound,142,Asia,1 -imi.ne.jp,lifemedia.jp,inbound,142,Asia,0 -indiaproperty.com,indiaproperty.com,inbound,142,Asia,0 -intage.co.jp,intage.co.jp,inbound,142,Asia,0.00557 -intuit.com,intuit.com,inbound,142,Asia,0.983698 -isbank.com.tr,isbank.com.tr,inbound,142,Asia,0.009655 -itsmyascent.com,itsmyascent.com,inbound,142,Asia,0 -itunes.com,apple.com,inbound,142,Asia,1 -jcity.com,jcity.com,inbound,142,Asia,0 -jobinthailand.com,jobinthailand.com,inbound,142,Asia,1 -jobmaster.co.il,jobmaster1.co.il,inbound,142,Asia,0 -jobsdbalert.co.id,jobsdbalert.co.id,inbound,142,Asia,0 -jobsdbalert.com,jobsdbalert.com,inbound,142,Asia,0 -jobsdbalert.com.hk,jobsdbalert.com.hk,inbound,142,Asia,0 -jobsdbalert.com.sg,jobsdbalert.com.sg,inbound,142,Asia,0 -jobstreet.com,jobstreet.com,inbound,142,Asia,0 -joshin.co.jp,joshin.co.jp,inbound,142,Asia,8e-06 -kagoya.net,kagoya.net,inbound,142,Asia,0.013052 -karvy.com,karvy.com,inbound,142,Asia,0.045186 -kasikornbank.com,kasikornbank.com,inbound,142,Asia,0 -kotak.com,kotak.com,inbound,142,Asia,0.005566 -krs.bz,tricorn.net,inbound,142,Asia,0 -kvbmail.com,kvbmail.com,inbound,142,Asia,0 -lancers.jp,lancers.jp,inbound,142,Asia,0 -lelong.my,lelong.com.my,inbound,142,Asia,1 -lelong.my,lelong.net.my,inbound,142,Asia,1 -line.me,naver.com,inbound,142,Asia,1 -livedoor.com,livedoor.com,inbound,142,Asia,0 -luxa.jp,luxa.jp,inbound,142,Asia,0 -m3.com,m3.com,inbound,142,Asia,0 -mag2.com,tandem-m.com,inbound,142,Asia,0 -magicbricks.com,tbsl.in,inbound,142,Asia,0 -mail-boss.com,mail-boss.com,inbound,142,Asia,0 -mailgun.org,mailgun.net,inbound,142,Asia,1 -mbga.jp,mbga.jp,inbound,142,Asia,0 -mixi.jp,mixi.jp,inbound,142,Asia,0 -mmagic.jp,mmagic.jp,inbound,142,Asia,0.000531 -mobile01.com,mobile01.com,inbound,142,Asia,0 -monex.co.jp,monex.co.jp,inbound,142,Asia,0.019424 -moneycontrol.com,active18.com,inbound,142,Asia,0 -moneyforward.com,moneyforward.com,inbound,142,Asia,0 -monipla.jp,aainc.co.jp,inbound,142,Asia,0 -monster.co.in,monster.co.in,inbound,142,Asia,0 -morhipo.com,euromsg.net,inbound,142,Asia,0 -mpme.jp,mpme.jp,inbound,142,Asia,0 -mpse.jp,emsaqua.jp,inbound,142,Asia,0 -mpse.jp,emsbeige.jp,inbound,142,Asia,0 -mpse.jp,emsbrown.jp,inbound,142,Asia,0 -mpse.jp,emscyan.jp,inbound,142,Asia,0 -mpse.jp,emsgold.jp,inbound,142,Asia,0 -mpse.jp,emslime.jp,inbound,142,Asia,0 -mpse.jp,emsnavy.jp,inbound,142,Asia,0 -mpse.jp,emspink.jp,inbound,142,Asia,0 -mpse.jp,emssnow.jp,inbound,142,Asia,0 -mpse.jp,mpme.jp,inbound,142,Asia,0 -mpse.jp,yahoo.co.jp,inbound,142,Asia,0 -mynavi.jp,mynavi.jp,inbound,142,Asia,3.5e-05 -naver.com,naver.com,inbound,142,Asia,1 -naver.com,naver.com,outbound,142,Asia,1 -nesinemail.com,euromsg.net,inbound,142,Asia,0 -net-survey.jp,net-survey.jp,inbound,142,Asia,0 -netbk.co.jp,netbk.co.jp,inbound,142,Asia,0.010995 -next-engine.org,next-engine.org,inbound,142,Asia,1 -nextdoor.com,nextdoor.com,inbound,142,Asia,1 -nic.in,relayout.nic.in,inbound,142,Asia,0 -nicovideo.jp,nicovideo.jp,inbound,142,Asia,0 -nifty.com,nifty.com,inbound,142,Asia,0.762068 -nikkei.com,nikkei.co.jp,inbound,142,Asia,0 -nikkeibp.co.jp,nikkeibp.co.jp,inbound,142,Asia,4.9e-05 -nissen.jp,nissen.jp,inbound,142,Asia,0 -ocn.ad.jp,ocn.ad.jp,inbound,142,Asia,0 -ocn.ne.jp,ocn.ad.jp,inbound,142,Asia,0 -p-world.co.jp,p-world.co.jp,inbound,142,Asia,0 -panasonic.jp,panasonic.jp,inbound,142,Asia,0 -pepabo.com,pepabo.com,inbound,142,Asia,0 -pia.jp,pia.jp,inbound,142,Asia,0 -playstation.com,playstation.com,inbound,142,Asia,0 -pointtown.com,gmo-media.jp,inbound,142,Asia,0 -pttplc.com,pttgrp.com,inbound,142,Asia,0 -publicators.com,publicators.com,inbound,142,Asia,0 -qoo10.jp,qoo10.jp,inbound,142,Asia,0 -qq.com,qq.com,inbound,142,Asia,0.999973 -quickbooks.com,intuit.com,inbound,142,Asia,0.99908 -rakuten.co.jp,rakuten.co.jp,inbound,142,Asia,0 -rakuten.co.jp,yahoo.co.jp,inbound,142,Asia,0 -rakuten.ne.jp,rakuten.co.jp,inbound,142,Asia,0 -realus.co.jp,realus.co.jp,inbound,142,Asia,0 -recochoku.jp,recochoku.jp,inbound,142,Asia,0 -rediffmail.com,akadns.net,outbound,142,Asia,0 -rediffmail.com,rediffmail.com,inbound,142,Asia,0 -relianceada.com,relianceada.com,inbound,142,Asia,0.276515 -research-panel.jp,research-panel.jp,inbound,142,Asia,0 -responder.co.il,responder.co.il,inbound,142,Asia,1 -runnet.jp,runnet.jp,inbound,142,Asia,0 -rutenmail.com.tw,rutenmail.com.tw,inbound,142,Asia,0 -sahibinden.com,sahibinden.com,inbound,142,Asia,0 -saisoncard.co.jp,saisoncard.co.jp,inbound,142,Asia,0 -sakura.ne.jp,sakura.ne.jp,inbound,142,Asia,0.673305 -samsung.com,samsung.com,inbound,142,Asia,0.008685 -saramin.co.kr,saramin.co.kr,inbound,142,Asia,0 -sbi.co.in,sbi.co.in,inbound,142,Asia,0 -sbr-inc.co.jp,hdemail.jp,inbound,142,Asia,0 -sc.com,sc.com,inbound,142,Asia,0.99683 -secure.ne.jp,secure.ne.jp,inbound,142,Asia,3.7e-05 -secureserver.net,secureserver.net,inbound,142,Asia,0.000386 -shinseibank.com,shinseibank.com,inbound,142,Asia,0 -shukatsu.jp,shukatsu.jp,inbound,142,Asia,0 -simplymarry.com,tbsl.in,inbound,142,Asia,0 -smartphoneexperts.com,mailgun.net,inbound,142,Asia,1 -smp.ne.jp,smp.ne.jp,inbound,142,Asia,0 -snapdealmail.in,snapdealmail.in,inbound,142,Asia,0 -sofmap.com,sofmap.com,inbound,142,Asia,0.0092 -softbank.jp,softbank.jp,inbound,142,Asia,0 -sony.jp,sony.jp,inbound,142,Asia,0.000654 -sourcenext.info,sourcenext.info,inbound,142,Asia,0 -surfmandelivery.com,surfmandelivery.com,inbound,142,Asia,0 -synergy360.jp,crmstyle.com,inbound,142,Asia,0 -taipeifubon.com.tw,taipeifubon.com.tw,inbound,142,Asia,0 -techgig.com,tbsl.in,inbound,142,Asia,0 -thinkvidya.com,thinkvidya.com,inbound,142,Asia,0 -ticketmonster.co.kr,ticketmonster.co.kr,inbound,142,Asia,0 -timesjobs.com,tbsl.in,inbound,142,Asia,0 -timesjobsmail.com,tbsl.in,inbound,142,Asia,0 -timesofindia.com,indiatimes.com,inbound,142,Asia,0 -tobizaru.jp,tobizaru.jp,inbound,142,Asia,0 -tocoo.jp,aics.ne.jp,inbound,142,Asia,0 -tower.jp,tower.jp,inbound,142,Asia,0 -treemall.com.tw,symphox.com,inbound,142,Asia,0 -tsite.jp,tsite.jp,inbound,142,Asia,0 -tsutaya.co.jp,tsutaya.co.jp,inbound,142,Asia,0 -turkcell.com.tr,turkcell.com.tr,inbound,142,Asia,0.043492 -type.jp,type.jp,inbound,142,Asia,0 -u-shopping.com.tw,u-shopping.com.tw,inbound,142,Asia,0 -udnpaper.com,udnpaper.com,inbound,142,Asia,0.998858 -udnshopping.com,udnshopping.com,inbound,142,Asia,0 -vodafone.com,vodafone.in,inbound,142,Asia,0.617518 -vpass.ne.jp,clickmailer.jp,inbound,142,Asia,0 -vpcontact.com,vpcontact.com,inbound,142,Asia,0 -way2sms.biz,way2sms.biz,inbound,142,Asia,0 -way2sms.in,way2sms.in,inbound,142,Asia,0 -webcas.net,webcas.net,inbound,142,Asia,0 -wechat.com,qq.com,inbound,142,Asia,1 -www.gov.tw,hinet.net,inbound,142,Asia,8e-05 -yahoo.co.jp,yahoo.co.jp,inbound,142,Asia,8e-06 -yahoo.co.jp,yahoo.co.jp,outbound,142,Asia,0 -yahoo.{...},yahoo.co.jp,inbound,142,Asia,0 -yakala.co,euromsg.net,inbound,142,Asia,0 -yesbank.in,yesbank.in,inbound,142,Asia,0.241591 -yodobashi.com,yodobashi.com,inbound,142,Asia,0 -zizigo.com,euromsg.net,inbound,142,Asia,0 -3suisses.be,3suisses.be,inbound,150,Europe,0 -3suisses.fr,3suisses.fr,inbound,150,Europe,0 -aanotifier.nl,aanotifier.nl,inbound,150,Europe,0 -adidas.com,neolane.net,inbound,150,Europe,0 -admail.hu,sanomaonline.hu,inbound,150,Europe,0 -adsender.us,adsender.us,inbound,150,Europe,0 -adverts.ie,adverts.ie,inbound,150,Europe,1 -advfn.com,advfn.com,inbound,150,Europe,0.635382 -agnitas.de,agnitas.de,inbound,150,Europe,0.999265 -airliquide.com,airliquide.com,inbound,150,Europe,1 -alerteimmo.com,alerteimmo.com,inbound,150,Europe,0 -alinea.fr,bp06.net,inbound,150,Europe,0 -allegro.pl,allegro.pl,inbound,150,Europe,0 -allegroup.hu,allegroup.hu,inbound,150,Europe,0 -alza.cz,alza.cz,inbound,150,Europe,0.039449 -alza.sk,alza.cz,inbound,150,Europe,0.027725 -andrewchristian.com,emv8.com,inbound,150,Europe,0 -anpasia.com,anpasia.com,inbound,150,Europe,0 -aprovaconcursos.com.br,eadunicid.com.br,inbound,150,Europe,0 -aruba.it,aruba.it,inbound,150,Europe,0.055666 -ashampoo.com,ashampoo.com,inbound,150,Europe,1 -aswatson.com,emarsys.net,inbound,150,Europe,0 -avira.com,avira.com,inbound,150,Europe,0.042528 -avito.ru,avito.ru,inbound,150,Europe,0.000631 -badoo.com,monopost.com,inbound,150,Europe,1 -balsamik.fr,balsamik.fr,inbound,150,Europe,0 -bancomer.com,postini.com,inbound,150,Europe,1e-05 -bbvacompass.com,postini.com,inbound,150,Europe,0.997006 -be2.com,nmp1.net,inbound,150,Europe,0 -biglion.ru,biglion.ru,inbound,150,Europe,0.999775 -bigmailsender.com,bigmailsender.com,inbound,150,Europe,0 -bioagri.com.br,postini.com,inbound,150,Europe,0.991765 -biomedcentral.com,emv5.com,inbound,150,Europe,0 -blackberry.com,blackberry.com,inbound,150,Europe,0 -blinkboxmusic.com,mediagraft.com,inbound,150,Europe,1 -blogtrottr.com,blogtrottr.com,inbound,150,Europe,0 -blue-compass.com,blue-compass.com,inbound,150,Europe,0 -bmdeda99.com,bmdeda99.com,inbound,150,Europe,0 -bolsfr.fr,colt.net,inbound,150,Europe,0 -bonuszbrigad.hu,bonuszbrigad.hu,inbound,150,Europe,0 -boohooemail.com,smartfocusdigital.net,inbound,150,Europe,0 -booking.com,booking.com,inbound,150,Europe,1 -bouncemanager.it,musvc.com,inbound,150,Europe,0.362026 -br.com,cmailsys.com,inbound,150,Europe,0 -brandalley.com,brandalley.com,inbound,150,Europe,0 -brands4friends.de,emv5.com,inbound,150,Europe,0 -brandsvillage.net,brandsvillage.net,inbound,150,Europe,0 -bt.com,bt.com,inbound,150,Europe,0.409404 -bweeble.com,adlabsinc.com,inbound,150,Europe,0 -cabestan.com,cab07.net,inbound,150,Europe,0 -cadremploi.fr,cadremploi.fr,inbound,150,Europe,0 -cardsys.at,cardsys.at,inbound,150,Europe,1 -carmamail.com,carmamail.com,inbound,150,Europe,0 -casasbahia.com.br,casasbahia.com.br,inbound,150,Europe,0 -catchoftheday.com.au,inxserver.de,inbound,150,Europe,1 -catererglobal.com,madgexjb.com,inbound,150,Europe,0 -caterermail.com,totaljobsmail.co.uk,inbound,150,Europe,0 -cathkidston.com,cathkidston.co.uk,inbound,150,Europe,0 -cccampaigns.com,emv5.com,inbound,150,Europe,0 -cccampaigns.com,emv8.com,inbound,150,Europe,0 -cccampaigns.net,01net.com,inbound,150,Europe,0 -cccampaigns.net,cccampaigns.net,inbound,150,Europe,0 -cccampaigns.net,emv4.net,inbound,150,Europe,0 -ccmbg.com,benchmark.fr,inbound,150,Europe,0 -cdongroup.com,cdongroup.com,inbound,150,Europe,0.000147 -cheapflights.co.uk,cheapflights.co.uk,inbound,150,Europe,0 -cheapflights.com,cheapflights.com,inbound,150,Europe,0 -chelseafc.com,chelseafc.com,inbound,150,Europe,0.003566 -cinesa.es,cccampaigns.com,inbound,150,Europe,0 -cipherzone.com,infimail.com,inbound,150,Europe,0 -citybrands.hu,webinform.hu,inbound,150,Europe,1 -clickon.com.br,clickon.com.br,inbound,150,Europe,0 -clicplan.com,dmdelivery.com,inbound,150,Europe,0 -cobone.com,emarsys.net,inbound,150,Europe,0 -communicatoremail.com,communicatoremail.com,inbound,150,Europe,0 -compute.internal,amazonaws.com,inbound,150,Europe,0.81478 -confirmedoptin.com,confirmedoptin.com,inbound,150,Europe,0 -continente.pt,1-hostingservice.com,inbound,150,Europe,0 -cratusservices.in,ramcorp.in,inbound,150,Europe,0 -cricinfo.com,cricinfo.com,inbound,150,Europe,0 -critsend.com,critsend.com,inbound,150,Europe,0 -crsend.com,crsend.com,inbound,150,Europe,0.008688 -csas.cz,csas.cz,inbound,150,Europe,0.999971 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,150,Europe,1 -cv-library.co.uk,cv-library.co.uk,inbound,150,Europe,0 -cvbankas.lt,efadm.eu,inbound,150,Europe,0 -cwjobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 -d-reizen.nl,dmdelivery.com,inbound,150,Europe,0 -dafiti.com.br,fagms.de,inbound,150,Europe,0 -datingfactory.com,caerussolutions.net,inbound,150,Europe,0 -dbgi.co.uk,emc1.co.uk,inbound,150,Europe,0 -deal.com.sg,emarsys.net,inbound,150,Europe,0 -debian.org,debian.org,inbound,150,Europe,1 -deezer.com,dms30.com,inbound,150,Europe,0 -directcrm.ru,directcrm.ru,inbound,150,Europe,0 -disney.co.uk,emv9.com,inbound,150,Europe,0 -dominosemail.co.uk,dominosemail.co.uk,inbound,150,Europe,0 -doodle.com,doodle.com,inbound,150,Europe,1 -dotmailer-email.com,dotmailer.com,inbound,150,Europe,0 -dotmailer.co.uk,dotmailer.com,inbound,150,Europe,0 -dpapp.nl,prikbordmailer.nl,inbound,150,Europe,0 -dpapp.nl,sslsecuref.nl,inbound,150,Europe,0 -dreivip.com,dreivip.com,inbound,150,Europe,0.000301 -dress-for-less.de,privalia.com,inbound,150,Europe,0 -drweb.com,drweb.com,inbound,150,Europe,0.969315 -e-boks.dk,e-boks.dk,inbound,150,Europe,1 -e-ebuyer.com,e-ebuyer.com,inbound,150,Europe,0 -e-mark.nl,e-mark.nl,inbound,150,Europe,0 -e-ngine.nl,e-ngine.nl,inbound,150,Europe,0 -ebay-kleinanzeigen.de,mobile.de,inbound,150,Europe,1 -ecommzone.com,ecommzone.com,inbound,150,Europe,0 -edarling.fr,fagms.de,inbound,150,Europe,0 -edima.hu,edima.hu,inbound,150,Europe,0 -efox-shop.com,dmdelivery.com,inbound,150,Europe,0 -ejobs.ro,ejobs.ro,inbound,150,Europe,8.5e-05 -elaine-asp.de,artegic.net,inbound,150,Europe,0.999997 -elcorteingles.es,elcorteingles.es,inbound,150,Europe,0 -elektronskaposta.si,eprvak.si,inbound,150,Europe,0 -elettershop.de,servicemail24.de,inbound,150,Europe,1 -emag.ro,emag.ro,inbound,150,Europe,0.005013 -email-comparethemarket.com,smartfocusdigital.net,inbound,150,Europe,0 -email360api.com,email360api.com,inbound,150,Europe,0 -emarsys.net,emarsys.net,inbound,150,Europe,0 -embluejet.com,emblueuser.com,inbound,150,Europe,0 -emsecure.net,emsecure.net,inbound,150,Europe,0 -emsmtp.com,emsmtp.com,inbound,150,Europe,0 -enewsletter.pl,enewsletter.pl,inbound,150,Europe,0.007349 -enewsletter.pl,sare25.com,inbound,150,Europe,0 -espmp-agfr.net,bp06.net,inbound,150,Europe,0 -esprit-friends.com,esprit-friends.com,inbound,150,Europe,0 -evanscycles.com,msgfocus.com,inbound,150,Europe,0 -experteer.com,experteer.com,inbound,150,Europe,0 -extra.com.br,emv8.com,inbound,150,Europe,0 -eyepin.com,eyepin.com,inbound,150,Europe,0 -fabfurnish.com,fagms.de,inbound,150,Europe,0 -facilisimo.com,facilisimo.com,inbound,150,Europe,1 -fagms.net,fagms.de,inbound,150,Europe,0 -finn.no,schibsted-it.no,inbound,150,Europe,0.002893 -fixeads.com,fixeads.com,inbound,150,Europe,0 -flirchi.com,flirchi.com,inbound,150,Europe,1.0 -flymonarchemail.com,flymonarchemail.com,inbound,150,Europe,0 -follow-up.se,follow-up.se,inbound,150,Europe,1 -fotocasa.es,fotocasa.es,inbound,150,Europe,0 -fotostrana.ru,fotocdn.net,inbound,150,Europe,3.4e-05 -free-lance.ru,free-lance.ru,inbound,150,Europe,0 -free.fr,free.fr,inbound,150,Europe,0.984012 -free.fr,free.fr,outbound,150,Europe,6.9e-05 -freecycle.org,freecycle.org,inbound,150,Europe,0.999865 -freemail.hu,freemail.hu,outbound,150,Europe,0 -freshmail.pl,freshmail.pl,inbound,150,Europe,0 -gardeningclubmail.co.uk,msgfocus.com,inbound,150,Europe,0 -giffgaff.com,giffgaff.com,inbound,150,Europe,0 -globasemail.com,globasemail.com,inbound,150,Europe,0.951088 -gmail.com,02.net,inbound,150,Europe,0.999617 -gmail.com,as13285.net,inbound,150,Europe,0.999943 -gmail.com,bbox.fr,inbound,150,Europe,0.919849 -gmail.com,belgacom.be,inbound,150,Europe,0.999444 -gmail.com,blackberry.com,inbound,150,Europe,0.998923 -gmail.com,bluewin.ch,inbound,150,Europe,0.93294 -gmail.com,btcentralplus.com,inbound,150,Europe,0.999969 -gmail.com,chello.nl,inbound,150,Europe,0.999951 -gmail.com,fastwebnet.it,inbound,150,Europe,0.984706 -gmail.com,jazztel.es,inbound,150,Europe,0.999701 -gmail.com,net24.it,inbound,150,Europe,0.999964 -gmail.com,netcabo.pt,inbound,150,Europe,0.998164 -gmail.com,numericable.fr,inbound,150,Europe,0.999723 -gmail.com,ono.com,inbound,150,Europe,0.995991 -gmail.com,orange.es,inbound,150,Europe,0.998742 -gmail.com,orange.fr,inbound,150,Europe,0 -gmail.com,otenet.gr,inbound,150,Europe,0.957917 -gmail.com,postini.com,inbound,150,Europe,0.924066 -gmail.com,proxad.net,inbound,150,Europe,0.998094 -gmail.com,rima-tde.net,inbound,150,Europe,0.99915 -gmail.com,sfr.net,inbound,150,Europe,0.999877 -gmail.com,skybroadband.com,inbound,150,Europe,0.999854 -gmail.com,t-ipconnect.de,inbound,150,Europe,0.999841 -gmail.com,tdc.net,inbound,150,Europe,0.999591 -gmail.com,telecomitalia.it,inbound,150,Europe,0.997 -gmail.com,telekom.hu,inbound,150,Europe,0.999977 -gmail.com,telenet.be,inbound,150,Europe,1 -gmail.com,telepac.pt,inbound,150,Europe,0.999584 -gmail.com,telia.com,inbound,150,Europe,1 -gmail.com,threembb.co.uk,inbound,150,Europe,1 -gmail.com,tpnet.pl,inbound,150,Europe,0.999761 -gmail.com,virginm.net,inbound,150,Europe,0.99661 -gmail.com,vodafone-ip.de,inbound,150,Europe,1 -gmail.com,vodafone.pt,inbound,150,Europe,0.999563 -gmail.com,vodafonedsl.it,inbound,150,Europe,0.999006 -gmail.com,wanadoo.fr,inbound,150,Europe,0.999752 -gmail.com,ziggo.nl,inbound,150,Europe,1 -gmx.de,gmx.net,inbound,150,Europe,1 -gmx.de,gmx.net,outbound,150,Europe,1 -gmx.net,gmx.net,inbound,150,Europe,1 -goalunited.org,ccmdcampaigns.net,inbound,150,Europe,0 -gog.com,gog.com,inbound,150,Europe,0 -gogroopie.com,gogroopie.com,inbound,150,Europe,0.000283 -goldenline.pl,goldenline.pl,inbound,150,Europe,1 -goodgame.com,emsmtp.com,inbound,150,Europe,0 -goodlife.pt,emv8.com,inbound,150,Europe,0 -google.com,postini.com,inbound,150,Europe,0.810567 -googlemail.com,t-ipconnect.de,inbound,150,Europe,0.999957 -gumtree.com,marktplaats.nl,inbound,150,Europe,0 -gumtree.com.au,kijiji.com,inbound,150,Europe,0 -gymglish.com,gymglish.com,inbound,150,Europe,0.000788 -haskell.org,haskell.org,inbound,150,Europe,0.000703 -hazteoir.org,hazteoir.org,inbound,150,Europe,0.31478 -hh.ru,hh.ru,inbound,150,Europe,0.996236 -hotel.de,emp-mail.de,inbound,150,Europe,0 -hotornot.com,monopost.com,inbound,150,Europe,0.983953 -hotukdeals.com,hotukdeals.com,inbound,150,Europe,1 -hpnotifier.nl,hpnotifier.nl,inbound,150,Europe,0 -icelandmail.co.uk,emsg-live.co.uk,inbound,150,Europe,0 -idealista.com,idealista.com,inbound,150,Europe,0.001627 -ideascost.com,ramcorp.in,inbound,150,Europe,0 -inboxair.com,inboxair.com,inbound,150,Europe,0 -infoempleo.com,infoempleo.com,inbound,150,Europe,0 -infojobs.it,infojobs.it,inbound,150,Europe,0 -infojobs.net,infojobs.net,inbound,150,Europe,0 -infopraca.pl,careesma.com,inbound,150,Europe,0 -ingdirect.es,ingdirect.es,inbound,150,Europe,1 -inter-chat.com,inter-chat.com,inbound,150,Europe,0 -interdatesa.com,fagms.net,inbound,150,Europe,0 -internations.org,internations.org,inbound,150,Europe,1 -inx1and1.de,1and1.com,inbound,150,Europe,1 -inxserver.com,inxserver.de,inbound,150,Europe,0.952863 -inxserver.de,inxserver.de,inbound,150,Europe,0.980838 -itms.in.ua,itms.in.ua,inbound,150,Europe,0 -jiscmail.ac.uk,lsoft.se,inbound,150,Europe,0 -jobisjob.com,jobisjob.com,inbound,150,Europe,0 -jobrapidoalert.com,jobrapidoalert.com,inbound,150,Europe,0 -jobs2web.com,ondemand.com,inbound,150,Europe,1 -jobserve.com,jobserve.com,inbound,150,Europe,0 -joobmailer.com,joobmailer.com,inbound,150,Europe,0 -jumia.com.ng,fagms.de,inbound,150,Europe,0 -justclick.ru,justclick.ru,inbound,150,Europe,0 -kalunga.com.br,kalunga.com.br,inbound,150,Europe,0 -kiabi.com,dms-02.net,inbound,150,Europe,0 -kijiji.ca,kijiji.com,inbound,150,Europe,0 -kismia.com,kismia.com,inbound,150,Europe,1 -kiwari.com,kiwari.com,inbound,150,Europe,9e-06 -kundenserver.de,kundenserver.de,inbound,150,Europe,1 -laposte.net,laposte.net,inbound,150,Europe,0.271722 -laposte.net,laposte.net,outbound,150,Europe,0 -laredoute.fr,laredoute.fr,inbound,150,Europe,0 -laterooms.com,laterooms.com,inbound,150,Europe,0.014746 -leboncoin.fr,leboncoin.fr,inbound,150,Europe,0 -leparisien.fr,leparisien.fr,inbound,150,Europe,0 -lexpress.fr,bp06.net,inbound,150,Europe,0 -libero.it,libero.it,inbound,150,Europe,0.00024 -libero.it,libero.it,outbound,150,Europe,0 -lifecooler.com,1-hostingservice.com,inbound,150,Europe,0 -listadventure.com,adlabsinc.com,inbound,150,Europe,0 -listjoe.com,adlabsinc.com,inbound,150,Europe,0 -litres.ru,litres.ru,inbound,150,Europe,1 -loccitane.com,neolane.net,inbound,150,Europe,0 -logentries.com,logentries.com,inbound,150,Europe,0 -loveplanet.ru,pochta.ru,inbound,150,Europe,0.640035 -lowcostholidays.co.uk,communicatoremail.com,inbound,150,Europe,0 -lua.org,pepperfish.net,inbound,150,Europe,1 -ludokados.com,ludokado.com,inbound,150,Europe,0 -mail-cdiscount.com,mail-cdiscount.com,inbound,150,Europe,0 -mail-mbank.pl,mail-mbank.pl,inbound,150,Europe,0 -mail.ru,mail.ru,inbound,150,Europe,0.991269 -mail.ru,mail.ru,outbound,150,Europe,0.006918 -mailer-service.de,mailer-service.de,inbound,150,Europe,4.9e-05 -mailersend.com,mailersend.com,inbound,150,Europe,0 -mailing-list.it,mailing-list.it,inbound,150,Europe,0 -mailjet.com,mailjet.com,inbound,150,Europe,0 -mailplus.nl,brightbase.net,inbound,150,Europe,1 -mailpv.net,pvmailer.net,inbound,150,Europe,1 -maisonsdumonde.com,bp06.net,inbound,150,Europe,0 -makro.nl,srv2.de,inbound,150,Europe,0.888692 -mapfre.com,emv5.com,inbound,150,Europe,0 -marktplaats.nl,marktplaats.nl,inbound,150,Europe,0 -matchwereld.nl,matchwereld.nl,inbound,150,Europe,0 -maxpark.com,gidepark.ru,inbound,150,Europe,1 -mdirector.com,mdrctr.com,inbound,150,Europe,0 -mecumauction.com,mecumauction.com,inbound,150,Europe,0 -meetic.com,meetic.com,inbound,150,Europe,0 -mequedouno.com,mequedouno.com,inbound,150,Europe,0 -metrodeal.com,fagms.de,inbound,150,Europe,0 -mightydeals.co.uk,mightydeals.co.uk,inbound,150,Europe,0 -mirtesen.ru,mtml.ru,inbound,150,Europe,0 -mitula.net,mitula.org,inbound,150,Europe,0 -mitula.org,mitula.org,inbound,150,Europe,0 -mixcloudmail.com,mixcloudmail.com,inbound,150,Europe,0.995482 -mlgns.com,mlgns.com,inbound,150,Europe,0 -mlgnserv.com,mlgnserv.com,inbound,150,Europe,0 -mmks.it,mail-maker.it,inbound,150,Europe,0 -mmsecure.nl,donenad.nl,inbound,150,Europe,0 -modnakasta.ua,emv5.com,inbound,150,Europe,0 -mooply.co,mailendo.com,inbound,150,Europe,0 -moviestarplanet.com,moviestarplanet.com,inbound,150,Europe,0 -mrc.org,msgfocus.com,inbound,150,Europe,0 -msdp1.com,msdp1.com,inbound,150,Europe,0 -msgfocus.com,msgfocus.com,inbound,150,Europe,0.011658 -mymms.com,fagms.de,inbound,150,Europe,0 -namorico.me,namorico.me,inbound,150,Europe,1 -nasza-klasa.pl,nasza-klasa.pl,inbound,150,Europe,0 -nationalexpress.com,nationalexpress.com,inbound,150,Europe,0 -neolane.net,neolane.net,inbound,150,Europe,0 -netopia.pt,netopia.pt,inbound,150,Europe,0.028429 -nhs.jobs,nhscareersjobs.co.uk,inbound,150,Europe,0 -nieuwsblad.be,vummail.be,inbound,150,Europe,0 -nmp1.com,nmp1.net,inbound,150,Europe,0 -nos.pt,netcabo.pt,inbound,150,Europe,6.3e-05 -noticiasaominuto.com,ccmdcampaigns.net,inbound,150,Europe,0 -noticiasaominuto.com,noticiasaominuto.com,inbound,150,Europe,0 -nrholding.net,nrholding.net,inbound,150,Europe,0 -odisseias.com,emv4.net,inbound,150,Europe,0 -odnoklassniki.ru,odnoklassniki.ru,inbound,150,Europe,0 -offerum.com,cccampaigns.com,inbound,150,Europe,0 -offerum.com,ccemails.com,inbound,150,Europe,0 -olx.pt,fixeads.com,inbound,150,Europe,0 -orange.fr,orange.fr,inbound,150,Europe,0 -orange.fr,orange.fr,outbound,150,Europe,0 -oroscopofree.com,adsender.us,inbound,150,Europe,0 -oroscopofree.com,oroscopofree.com,inbound,150,Europe,0 -orsay.com,emp-mail.de,inbound,150,Europe,0 -ovh.net,ovh.net,inbound,150,Europe,0.165188 -oxfam.org.uk,msgfocus.com,inbound,150,Europe,0 -payback.de,artegic.net,inbound,150,Europe,0.999986 -pccomponentes.com,pccomponentes.com,inbound,150,Europe,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,150,Europe,0 -peperoni.de,peperoni.de,inbound,150,Europe,1 -peytz.dk,peytz.dk,inbound,150,Europe,4e-06 -photobox.com,photobox.com,inbound,150,Europe,0 -photoprintit.com,photoprintit.com,inbound,150,Europe,0 -pixum.com,pixum.com,inbound,150,Europe,0 -placedestendances.com,placedestendances.com,inbound,150,Europe,0 -plaisio.gr,fagms.de,inbound,150,Europe,0 -planeo.com,planeo.com,inbound,150,Europe,0 -planeo.pt,planeo.pt,inbound,150,Europe,0 -playtika.com,emv8.com,inbound,150,Europe,0 -poinx.com,poinx.com,inbound,150,Europe,0 -pokerstars.com,pokerstars.eu,inbound,150,Europe,0 -pokerstars.eu,pokerstars.eu,inbound,150,Europe,0 -pontofrio.com.br,emv8.com,inbound,150,Europe,0 -postgresql.org,postgresql.org,inbound,150,Europe,1 -praca.pl,praca.pl,inbound,150,Europe,1 -pracuj.pl,pracuj.pl,inbound,150,Europe,0 -printvenue.com,fagms.de,inbound,150,Europe,0 -privalia.com,privalia.com,inbound,150,Europe,3.94913202027326e-07 -promod-news.fr,promod-news.com,inbound,150,Europe,0 -protopmail.com,protopmail.com,inbound,150,Europe,0 -pur3.net,pur3.net,inbound,150,Europe,0.001123 -python.org,python.org,inbound,150,Europe,1 -quality.net.ua,quality.net.ua,inbound,150,Europe,0 -r-project.org,ethz.ch,inbound,150,Europe,0.99999 -r51.it,musvc.com,inbound,150,Europe,0.092498 -r52.it,musvc.com,inbound,150,Europe,0.000221 -r57.it,musvc.com,inbound,150,Europe,0.139425 -r67.it,musvc.com,inbound,150,Europe,0.199845 -r70.it,musvc.com,inbound,150,Europe,0.311983 -rakuten.co.jp,shareee.jp,inbound,150,Europe,0 -rambler.ru,rambler.ru,inbound,150,Europe,0.08173 -ratedpeople.com,ratedpeople.com,inbound,150,Europe,0.000979 -rax.ru,rax.ru,inbound,150,Europe,0.000258 -regie11.net,odiso.net,inbound,150,Europe,0 -relax7.hu,gruppi.hu,inbound,150,Europe,0 -richersoundsvip.com,ibwmail.com,inbound,150,Europe,0 -rightmove.com,rightmove.com,inbound,150,Europe,0 -roulartamail.be,roulartamail.be,inbound,150,Europe,0 -rueducommerce.com,groupe-rueducommerce.fr,inbound,150,Europe,0 -runtastic.com,runtastic.com,inbound,150,Europe,0 -ryanairmail.com,ryanairmail.com,inbound,150,Europe,0 -rzone.de,rzone.de,inbound,150,Europe,1 -saimails.in,infimail.com,inbound,150,Europe,0 -sainsburys.co.uk,emv5.com,inbound,150,Europe,0 -salesmanago.pl,salesmanago.pl,inbound,150,Europe,0 -samsung.ru,samsung.ru,inbound,150,Europe,0 -sapnetworkmail.com,sap-ag.de,inbound,150,Europe,1 -sapo.pt,sapo.pt,inbound,150,Europe,0.242839 -sapo.pt,sapo.pt,outbound,150,Europe,0 -savingdeals.in,infimail.com,inbound,150,Europe,0 -scmp.com,emarsys.net,inbound,150,Europe,0 -scoop.it,scoop.it,inbound,150,Europe,0 -scoopon.com.au,inxserver.de,inbound,150,Europe,1 -screwfix.info,fwdto.net,inbound,150,Europe,0 -secureserver.net,secureserver.net,inbound,150,Europe,0 -selection-priceminister.com,selection-priceminister.com,inbound,150,Europe,0 -sender.lt,sritis.lt,inbound,150,Europe,0.000339 -sendsmaily.info,sendsmaily.info,inbound,150,Europe,0 -seniorplanet.fr,seniorplanet.fr,inbound,150,Europe,0 -seznam.cz,seznam.cz,inbound,150,Europe,0.001781 -seznam.cz,seznam.cz,outbound,150,Europe,0.001741 -sfr.fr,sfr.fr,inbound,150,Europe,0.236539 -sfr.fr,sfr.fr,outbound,150,Europe,0.004753 -shopto.net,shopto.net,inbound,150,Europe,1 -skype.com,skype.com,inbound,150,Europe,0 -solveerrors.com,infimail.com,inbound,150,Europe,0 -spareroom.co.uk,spareroom.co.uk,inbound,150,Europe,1 -sqlservercentral.com,sqlservercentral.com,inbound,150,Europe,0 -staples-pt.com,1-hostingservice.com,inbound,150,Europe,0 -stepstone.de,stepstone.com,inbound,150,Europe,0.001689 -studentbeans.com,emv8.com,inbound,150,Europe,0 -subito.it,subito.it,inbound,150,Europe,0 -subscribe.ru,subscribe.ru,inbound,150,Europe,0 -superdrug.com,superdrug.com,inbound,150,Europe,0 -superjob.ru,superjob.ru,inbound,150,Europe,0 -support-love.com,support-love.com,inbound,150,Europe,0 -sut1.co.uk,sut1.co.uk,inbound,150,Europe,0.001789 -sut5.co.uk,sut5.co.uk,inbound,150,Europe,0 -swanson-vitamins.com,emv5.com,inbound,150,Europe,0 -t-online.de,t-online.de,inbound,150,Europe,1 -t-online.de,t-online.de,outbound,150,Europe,0.999939 -tatrabanka.sk,tatrabanka.sk,inbound,150,Europe,0 -tchibo.de,srv2.de,inbound,150,Europe,0.980447 -teamo.ru,teamo.ru,inbound,150,Europe,0 -teamviewer.com,teamviewer.com,inbound,150,Europe,0 -teapartyinfo.org,teapartyinfo.org,inbound,150,Europe,0 -telenet.be,telenet-ops.be,inbound,150,Europe,1.5e-05 -teleportmyjob.com,clara.net,inbound,150,Europe,0 -theleadmagnet.com,your-server.de,inbound,150,Europe,1 -timeweb.ru,timeweb.ru,inbound,150,Europe,1 -tiscali.it,tiscali.it,outbound,150,Europe,0 -totaljobsmail.co.uk,totaljobsmail.co.uk,inbound,150,Europe,0 -transversal.net,transversal.net,inbound,150,Europe,1 -travisperkins.co.uk,travisperkins.co.uk,inbound,150,Europe,0 -trovit.com,trovit.com,inbound,150,Europe,0 -tucasa.com,grupodtm.com,inbound,150,Europe,0 -twoomail.com,twoomail.com,inbound,150,Europe,0 -ubuntu.com,canonical.com,inbound,150,Europe,0 -ukr.net,fwdcdn.com,inbound,150,Europe,1 -ukr.net,ukr.net,outbound,150,Europe,0.999992 -ulteem.com,ulteem.com,inbound,150,Europe,0 -usndr.com,usndr.com,inbound,150,Europe,1 -venere.com,kiwari.com,inbound,150,Europe,0 -venteprivee.com,venteprivee.com,inbound,150,Europe,0 -virgilio.it,virgilio.net,inbound,150,Europe,0 -visualsoft.co.uk,visualsoft.co.uk,inbound,150,Europe,0 -vouchercloud.com,vouchercloud.com,inbound,150,Europe,0 -voyageprive.com,cccampaigns.net,inbound,150,Europe,0 -voyageprive.es,ccmdcampaigns.net,inbound,150,Europe,0 -voyageprive.it,ccmdcampaigns.net,inbound,150,Europe,0 -voyages-sncf.com,neolane.net,inbound,150,Europe,0 -vueling.com,vueling.com,inbound,150,Europe,0 -wanadoo.fr,orange.fr,inbound,150,Europe,0 -wanadoo.fr,orange.fr,outbound,150,Europe,0 -waves-audio.com,emv8.com,inbound,150,Europe,0 -web.de,web.de,inbound,150,Europe,0.999986 -web.de,web.de,outbound,150,Europe,1 -whereareyounow.com,wayn.net,inbound,150,Europe,0 -wiggle.com,wiggle.com,inbound,150,Europe,0 -william-reed.com,neolane.net,inbound,150,Europe,0 -williamhill.com,williamhill.com,inbound,150,Europe,0 -wldemail.com,emarsys.net,inbound,150,Europe,0 -wmtransfer.com,wmtransfer.com,inbound,150,Europe,0 -wnd.com,emv4.net,inbound,150,Europe,0 -wnd.com,worldnetdaily.com,inbound,150,Europe,0 -work.ua,work.ua,inbound,150,Europe,1 -workcircle.com,workcircle.net,inbound,150,Europe,0 -wp.pl,wp.pl,inbound,150,Europe,0.998027 -wp.pl,wp.pl,outbound,150,Europe,1 -xmailix.com,xmailix.com,inbound,150,Europe,0 -yandex.ru,yandex.net,inbound,150,Europe,0.999658 -yandex.ru,yandex.ru,outbound,150,Europe,1 -ymlpserver.net,ymlpserver.net,inbound,150,Europe,0 -ymlpsrv.net,ymlpsrv.net,inbound,150,Europe,0 -zalando.be,fagms.de,inbound,150,Europe,0 -zalando.dk,fagms.de,inbound,150,Europe,0 -zalando.fi,fagms.de,inbound,150,Europe,0 -zalando.it,fagms.de,inbound,150,Europe,0 -zalando.nl,fagms.de,inbound,150,Europe,0 -zalando.pl,fagms.de,inbound,150,Europe,0 -zumzi.com,neogen.ro,inbound,150,Europe,0 -zumzi.com,zumzi.com,inbound,150,Europe,0 -0bz.biz,hmts.jp,inbound,ZZ,Unknown Region,0 -104.com.tw,104.com.tw,inbound,ZZ,Unknown Region,0.091133 -1105info.com,1105info.com,inbound,ZZ,Unknown Region,0 -1111.com.tw,1111.com.tw,inbound,ZZ,Unknown Region,0 -160by2inbox.com,160by2inbox.com,inbound,ZZ,Unknown Region,0 -160by2invite.com,160by2invite.com,inbound,ZZ,Unknown Region,0 -160by2mail.com,160by2mail.com,inbound,ZZ,Unknown Region,0 -163.com,163.com,inbound,ZZ,Unknown Region,0.996685 -1800flowersinc.com,1800flowersinc.com,inbound,ZZ,Unknown Region,0 -1sale.com,1sale.com,inbound,ZZ,Unknown Region,0 -1v1y.com,euromsg.net,inbound,ZZ,Unknown Region,0 -4shared.com,4shared.com,inbound,ZZ,Unknown Region,1 -6pm.com,6pm.com,inbound,ZZ,Unknown Region,1 -6pm.com,zappos.com,inbound,ZZ,Unknown Region,0.782581 -a8.net,a8.net,inbound,ZZ,Unknown Region,0 -aaa.com,nextjump.com,inbound,ZZ,Unknown Region,0 -aaas-science.org,aaas-science.org,inbound,ZZ,Unknown Region,0 -aarp.org,aarp.org,inbound,ZZ,Unknown Region,0 -about.com,about.com,inbound,ZZ,Unknown Region,8.2e-05 -academy-enews.com,academy-enews.com,inbound,ZZ,Unknown Region,0 -accenture.com,outlook.com,inbound,ZZ,Unknown Region,1 -accountonline.com,accountonline.com,inbound,ZZ,Unknown Region,0.348991 -acehelpfulemails.com,teradatadmc.com,inbound,ZZ,Unknown Region,0 -actorsaccess.com,nonfatmedia.com,inbound,ZZ,Unknown Region,0 -adminforfree.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -adminforfree.net,adminforfree.com,inbound,ZZ,Unknown Region,1 -administrativejobinsider.com,administrativejobinsider.com,inbound,ZZ,Unknown Region,0 -adobe.com,obsmtp.com,inbound,ZZ,Unknown Region,1 -adobesystems.com,adobesystems.com,inbound,ZZ,Unknown Region,0 -adoreme.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -adp.com,adp.com,inbound,ZZ,Unknown Region,1 -adsender.us,adsender.us,inbound,ZZ,Unknown Region,0 -adsolutionline.com,adsolutionline.com,inbound,ZZ,Unknown Region,0 -adultfriendfinder.com,friendfinder.com,inbound,ZZ,Unknown Region,0 -advanceauto.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -advantagebusinessmedia.com,advantagebusinessmedia.com,inbound,ZZ,Unknown Region,0 -af.mil,af.mil,inbound,ZZ,Unknown Region,0.997561 -agoda-emails.com,agoda-emails.com,inbound,ZZ,Unknown Region,0 -agrupemonos.cl,agrupemonos.cl,inbound,ZZ,Unknown Region,1 -albertsonsemail.com,email4-mywebgrocer.com,inbound,ZZ,Unknown Region,0 -alibaba.com,alibaba.com,inbound,ZZ,Unknown Region,0 -alice.it,alice.it,inbound,ZZ,Unknown Region,0 -alice.it,aliceposta.it,outbound,ZZ,Unknown Region,0 -aliexpress.com,alibaba.com,inbound,ZZ,Unknown Region,0 -allegro.pl,allegro.pl,inbound,ZZ,Unknown Region,0 -allegrogroup.ua,allegrogroup.ua,inbound,ZZ,Unknown Region,0 -allsaints.com,allsaints.com,inbound,ZZ,Unknown Region,0 -ama-assn.org,elabs10.com,inbound,ZZ,Unknown Region,0 -amadeus.com,amadeus.net,inbound,ZZ,Unknown Region,0 -amazon.{...},amazon.{...},inbound,ZZ,Unknown Region,0.021685 -amazon.{...},amazonses.com,inbound,ZZ,Unknown Region,0.999971 -amazonses.com,amazonses.com,inbound,ZZ,Unknown Region,0.997919 -amazonses.com,postini.com,inbound,ZZ,Unknown Region,0.924067 -amctheatres.com,amctheatres.com,inbound,ZZ,Unknown Region,0 -americanpublicmediagroup.org,americanpublicmediagroup.org,inbound,ZZ,Unknown Region,0 -ancestry.com,ancestry.com,inbound,ZZ,Unknown Region,0 -angieslist.com,angieslist.com,inbound,ZZ,Unknown Region,0 -anntaylor.com,anntaylor.com,inbound,ZZ,Unknown Region,0 -anpdm.com,anpdm.com,inbound,ZZ,Unknown Region,4e-06 -aol.com,aol.com,outbound,ZZ,Unknown Region,1 -apple.com,apple.com,inbound,ZZ,Unknown Region,0.915839 -argos.co.uk,argos.co.uk,inbound,ZZ,Unknown Region,1 -argos.co.uk,exacttarget.com,inbound,ZZ,Unknown Region,1 -artists-hub.com,artists-hub.com,inbound,ZZ,Unknown Region,0 -asda.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -ask.fm,ask.fm,inbound,ZZ,Unknown Region,1e-05 -askmen.com,askmen.com,inbound,ZZ,Unknown Region,0 -astrocenter.com,center.com,inbound,ZZ,Unknown Region,0 -athleta.com,athleta.com,inbound,ZZ,Unknown Region,0 -atlassian.net,uc-inf.net,inbound,ZZ,Unknown Region,1 -att.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999956 -auctionzip-email.com,email-auctionholdings.com,inbound,ZZ,Unknown Region,0 -auinmeio.com.br,fnac.com.br,inbound,ZZ,Unknown Region,0 -authorize.net,authorize.net,inbound,ZZ,Unknown Region,0 -authorize.net,visa.com,inbound,ZZ,Unknown Region,0.993555 -autoreply.com,autoreply.com,inbound,ZZ,Unknown Region,0 -avomail.com,avomail.com,inbound,ZZ,Unknown Region,0 -avon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 -avon.com,postdirect.com,inbound,ZZ,Unknown Region,0 -aweber.com,aweber.com,inbound,ZZ,Unknown Region,3e-06 -ayi.com,ayi.com,inbound,ZZ,Unknown Region,0 -backcountry.com,backcountry.com,inbound,ZZ,Unknown Region,0 -backlog.jp,backlog.jp,inbound,ZZ,Unknown Region,0 -bagitgetitmailer.in,emce2.in,inbound,ZZ,Unknown Region,0 -banamex.com,citi.com,inbound,ZZ,Unknown Region,0.999958 -bananarepublic.com,bananarepublic.com,inbound,ZZ,Unknown Region,3.48869418874254e-07 -bancochile.cl,bancochile.cl,inbound,ZZ,Unknown Region,0.999504 -bancofalabella.com,bancofalabella.com,inbound,ZZ,Unknown Region,0 -banesco.com,banesco.com,inbound,ZZ,Unknown Region,0 -bankofamerica.com,bankofamerica.com,inbound,ZZ,Unknown Region,0.971142 -banorte.com,gfnorte.com.mx,inbound,ZZ,Unknown Region,0.994999 -barclaycardus.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -basecamp.com,basecamp.com,inbound,ZZ,Unknown Region,1 -basecamphq.com,basecamphq.com,inbound,ZZ,Unknown Region,1 -baskinrobbins.com,baskinrobbins.com,inbound,ZZ,Unknown Region,0 -bazarchic-invitations.com,bazarchic-emstech.com,inbound,ZZ,Unknown Region,0 -bcbg.com,bcbg.com,inbound,ZZ,Unknown Region,0 -beatport-email.com,beatport-email.com,inbound,ZZ,Unknown Region,0 -beautylish.com,beautylish.com,inbound,ZZ,Unknown Region,1 -bebe.com,ed10.com,inbound,ZZ,Unknown Region,0 -befrugal.com,befrugal.com,inbound,ZZ,Unknown Region,0 -belkemail.com,belkemail.com,inbound,ZZ,Unknown Region,0 -bellsouth.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999951 -benihana-news.com,benihana-news.com,inbound,ZZ,Unknown Region,0 -bestbuy.com,bestbuy.com,inbound,ZZ,Unknown Region,0.003289 -beta.lt,mailersend3.com,inbound,ZZ,Unknown Region,0 -beyondtherack.com,beyondtherack.com,inbound,ZZ,Unknown Region,0 -bigfishgames.com,bigfishgames.com,inbound,ZZ,Unknown Region,0 -biglots.com,biglots.com,inbound,ZZ,Unknown Region,0 -bjsrestaurants.com,bjsrestaurants.com,inbound,ZZ,Unknown Region,0 -blackberry.com,blackberry.com,inbound,ZZ,Unknown Region,0 -blackboard.com,notification.com,inbound,ZZ,Unknown Region,0 -blackpeoplemeet.com,blackpeoplemeet.com,inbound,ZZ,Unknown Region,0 -bloglovin.com,bloglovin.com,inbound,ZZ,Unknown Region,0 -blogtrottr.com,blogtrottr.com,inbound,ZZ,Unknown Region,0 -bloomberg.com,bloomberg.com,inbound,ZZ,Unknown Region,0.001463 -bloomberg.net,bloomberg.net,inbound,ZZ,Unknown Region,1 -bluediamondhost3.com,web-hosting.com,inbound,ZZ,Unknown Region,1 -bluehornet.com,bluehornet.com,inbound,ZZ,Unknown Region,0 -bluenile.com,bluenile.com,inbound,ZZ,Unknown Region,0 -bluestatedigital.com,bluestatedigital.com,inbound,ZZ,Unknown Region,0 -bm05.net,bm05.net,inbound,ZZ,Unknown Region,0 -bmnt.jp,bmnt.jp,inbound,ZZ,Unknown Region,0 -bmsend.com,bmsend.com,inbound,ZZ,Unknown Region,0 -bn.com,bn.com,inbound,ZZ,Unknown Region,0 -bncollegemail.com,bncollegemail.com,inbound,ZZ,Unknown Region,0 -booking.com,booking.com,inbound,ZZ,Unknown Region,1 -bookmyshow.com,eccluster.com,inbound,ZZ,Unknown Region,0 -boots.com,boots.com,inbound,ZZ,Unknown Region,0 -box.com,box.com,inbound,ZZ,Unknown Region,0.955607 -brierleycrm.com,brierleycrm.com,inbound,ZZ,Unknown Region,0 -brooksbrothers.com,brooksbrothers.com,inbound,ZZ,Unknown Region,0 -btinternet.com,cpcloud.co.uk,inbound,ZZ,Unknown Region,0 -btinternet.com,cpcloud.co.uk,outbound,ZZ,Unknown Region,0 -btinternet.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -budgettravel.com,email-budgettravel.com,inbound,ZZ,Unknown Region,0 -buyinvite.com.au,buyinvite.com.au,inbound,ZZ,Unknown Region,0 -bv.com.br,bv.com.br,inbound,ZZ,Unknown Region,0 -byway.it,byway.it,inbound,ZZ,Unknown Region,0 -cabelas.com,cabelas.com,inbound,ZZ,Unknown Region,0 -californiajobdepartment.com,californiajobdepartment.com,inbound,ZZ,Unknown Region,0 -callcommand.com,callcommand.com,inbound,ZZ,Unknown Region,0 -calottery.com,calottery.com,inbound,ZZ,Unknown Region,1 -canadiantire.ca,canadiantire.ca,inbound,ZZ,Unknown Region,0 -canalplus.es,canalplus.es,inbound,ZZ,Unknown Region,0 -capitalone.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -capitaloneemail.com,capitaloneemail.com,inbound,ZZ,Unknown Region,0 -career-hub.net,career-hub.net,inbound,ZZ,Unknown Region,1 -careerbuilder-email.com,careerbuilder-email.com,inbound,ZZ,Unknown Region,0 -careerbuilder.com,careerbuilder.com,inbound,ZZ,Unknown Region,0 -careerflash.net,careerflash.net,inbound,ZZ,Unknown Region,1 -carrefour.fr,carrefour.fr,inbound,ZZ,Unknown Region,0 -carters.com,carters.com,inbound,ZZ,Unknown Region,2e-06 -caseyresearch.com,caseyresearch.com,inbound,ZZ,Unknown Region,1 -catchyfreebies.net,mmsend53.com,inbound,ZZ,Unknown Region,0 -cccampaigns.net,emv9.net,inbound,ZZ,Unknown Region,0 -cecentertainment.com,cecentertainment.com,inbound,ZZ,Unknown Region,0 -celebritycruises.com,celebritycruises.com,inbound,ZZ,Unknown Region,0 -centaur.co.uk,centaur.co.uk,inbound,ZZ,Unknown Region,0 -centauro.com.br,centauro.com.br,inbound,ZZ,Unknown Region,0 -centerparcs.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -cfmailer.com,elabs11.com,inbound,ZZ,Unknown Region,0 -change.org,change.org,inbound,ZZ,Unknown Region,1 -channel4.com,channel4.com,inbound,ZZ,Unknown Region,0 -chase.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -cheaperthandirt.com,cheaperthandirt.com,inbound,ZZ,Unknown Region,0 -chefscatalog.com,chefscatalog.com,inbound,ZZ,Unknown Region,0 -chemistdirect.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -chemistry.com,chemistry.com,inbound,ZZ,Unknown Region,0 -chick-fil-ainsiders.com,chick-fil-ainsiders.com,inbound,ZZ,Unknown Region,0 -chopra.com,chopra.com,inbound,ZZ,Unknown Region,1 -christianmingle.com,postdirect.com,inbound,ZZ,Unknown Region,0 -cir.ca,cir.ca,inbound,ZZ,Unknown Region,1 -citi.com,citi.com,inbound,ZZ,Unknown Region,0.999941 -citibank.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -citibank.com,citi.com,inbound,ZZ,Unknown Region,0.999997 -citicorp.com,citi.com,inbound,ZZ,Unknown Region,0.999999 -citruslane.com,citruslane.com,inbound,ZZ,Unknown Region,5e-06 -clarisonic.com,clarisonic.com,inbound,ZZ,Unknown Region,0 -classmates.com,classmates.com,inbound,ZZ,Unknown Region,0 -clickon.com.ar,clickon.com.ar,inbound,ZZ,Unknown Region,0 -cmail1.com,createsend.com,inbound,ZZ,Unknown Region,0 -cmail2.com,createsend.com,inbound,ZZ,Unknown Region,0 -cmrfalabella.com,cmrfalabella.com,inbound,ZZ,Unknown Region,0 -codebreak.info,codebreak.info,inbound,ZZ,Unknown Region,1 -comcast.net,comcast.net,outbound,ZZ,Unknown Region,1 -confirmsignup.com,mmsend53.com,inbound,ZZ,Unknown Region,0 -constantcontact.com,postini.com,inbound,ZZ,Unknown Region,0.139551 -constantcontact.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999541 -contact-darty.com,mm-send.com,inbound,ZZ,Unknown Region,0 -contactlab.it,contactlab.it,inbound,ZZ,Unknown Region,0 -converse.com,converse.com,inbound,ZZ,Unknown Region,1 -cookingchanneltv.com,cookingchanneltv.com,inbound,ZZ,Unknown Region,0 -copernica.nl,picsrv.net,inbound,ZZ,Unknown Region,0.011507 -copernica.nl,vicinity.nl,inbound,ZZ,Unknown Region,0.011753 -coppel.com,coppel.com,inbound,ZZ,Unknown Region,0 -corporateperks.com,nextjump.com,inbound,ZZ,Unknown Region,0 -countrycurtainscatalog.com,countrycurtainscatalog.com,inbound,ZZ,Unknown Region,0 -coupondunia.in,coupondunia.in,inbound,ZZ,Unknown Region,1 -crabtree-evelyn.com,crabtree-evelyn.com,inbound,ZZ,Unknown Region,0 -crainnewsalerts.com,crainnewsalerts.com,inbound,ZZ,Unknown Region,0 -crashlytics.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -cricut.com,elabs12.com,inbound,ZZ,Unknown Region,0 -criticalimpactinc.com,criticalimpactinc.com,inbound,ZZ,Unknown Region,0 -critsend.com,critsend.com,inbound,ZZ,Unknown Region,0 -crocs-email.com,crocs-email.com,inbound,ZZ,Unknown Region,0 -crunchyroll.com,crunchyroll.com,inbound,ZZ,Unknown Region,0 -cudo.com.au,exacttarget.com,inbound,ZZ,Unknown Region,0 -cuenote.jp,cuenote.jp,inbound,ZZ,Unknown Region,0 -cupomturbinado.com.br,cupomnaweb.com.br,inbound,ZZ,Unknown Region,1 -cuppon.pl,cuppon.pl,inbound,ZZ,Unknown Region,0 -curriculum.com.br,curriculum.com.br,inbound,ZZ,Unknown Region,0 -currys.co.uk,currys.co.uk,inbound,ZZ,Unknown Region,0 -custom-emailing.com,elabs12.com,inbound,ZZ,Unknown Region,0 -cvent-planner.com,cvent-planner.com,inbound,ZZ,Unknown Region,0 -cyberdiet.com.br,allinmedia.com.br,inbound,ZZ,Unknown Region,0 -dabmail.com,iaires.com,inbound,ZZ,Unknown Region,0 -dailyom.com,dailyom.com,inbound,ZZ,Unknown Region,1 -dairyqueen.com,dairyqueen.com,inbound,ZZ,Unknown Region,0 -datadrivenemail.com,datadrivenemail.com,inbound,ZZ,Unknown Region,0 -daveramsey.com,daveramsey.com,inbound,ZZ,Unknown Region,0 -ddc-emails.com,ddc-emails.com,inbound,ZZ,Unknown Region,0 -dealchicken.com,dealchicken.com,inbound,ZZ,Unknown Region,0 -dealchicken.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -dealfind.com,dealfind.com,inbound,ZZ,Unknown Region,0 -dealnews.com,dealnews.com,inbound,ZZ,Unknown Region,0 -dealsdirect.com.au,dealsdirect.com.au,inbound,ZZ,Unknown Region,0 -dealspl.us,dealspl.us,inbound,ZZ,Unknown Region,0 -delta.com,delta.com,inbound,ZZ,Unknown Region,0.040177 -dermstore.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -dhl.com,dhl.com,inbound,ZZ,Unknown Region,0.994107 -dietaesaude.com.br,dietaesaude.com.br,inbound,ZZ,Unknown Region,0 -dinda.com.br,dinda.com.br,inbound,ZZ,Unknown Region,0 -directvla.com,directvla.com,inbound,ZZ,Unknown Region,0 -discover.com,discover.com,inbound,ZZ,Unknown Region,0 -disneydestinations.com,disneyparks.com,inbound,ZZ,Unknown Region,0 -disneydestinations.com,disneyworld.com,inbound,ZZ,Unknown Region,0 -diynetwork.com,diynetwork.com,inbound,ZZ,Unknown Region,0 -doctoroz.com,email-sharecare2.com,inbound,ZZ,Unknown Region,0 -dollartree.com,email-dollartree.com,inbound,ZZ,Unknown Region,0 -dominos.com,dominos.com,inbound,ZZ,Unknown Region,0.011861 -dominos.com.au,dominos.com.au,inbound,ZZ,Unknown Region,0 -donuts.ne.jp,dnuts.jp,inbound,ZZ,Unknown Region,0 -dotz.com.br,dotz.com.br,inbound,ZZ,Unknown Region,0 -doubletakeoffers.com,doubletakeoffers.com,inbound,ZZ,Unknown Region,0 -downlinebuilderdirect.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 -dptagent.net,dptagent.net,inbound,ZZ,Unknown Region,0 -draftkings.com,draftkings.com,inbound,ZZ,Unknown Region,0 -dreamhost.com,dreamhost.com,inbound,ZZ,Unknown Region,0 -driftem.com,emce2.in,inbound,ZZ,Unknown Region,0 -drjays-mail.com,drjays-mail.com,inbound,ZZ,Unknown Region,0 -dromadaire-news.com,ecmcluster.com,inbound,ZZ,Unknown Region,0 -dukecareers.com,dukecareers.com,inbound,ZZ,Unknown Region,1 -duluthtradingemail.com,email-duluthtrading.com,inbound,ZZ,Unknown Region,0 -dynect-mailer.net,dynect.net,inbound,ZZ,Unknown Region,0 -dynect-mailer.net,sendlabs.com,inbound,ZZ,Unknown Region,0 -e-bodyc.com,email-bodycentral.com,inbound,ZZ,Unknown Region,0 -e-jobs-ville.com,e-jobs-ville.com,inbound,ZZ,Unknown Region,1 -e-leclerc.com,e-leclerc.com,inbound,ZZ,Unknown Region,0 -easycanvasprints.com,easycanvasprints.com,inbound,ZZ,Unknown Region,0 -easyhits4u.com,easyhits4u.com,inbound,ZZ,Unknown Region,1 -easyhits4u.com,relmax.net,inbound,ZZ,Unknown Region,0 -ebags.com,ebags.com,inbound,ZZ,Unknown Region,0 -ebay-kleinanzeigen.de,mobile.de,inbound,ZZ,Unknown Region,1 -ebay.{...},ebay.{...},inbound,ZZ,Unknown Region,0.999648 -ebay.{...},emarsys.net,inbound,ZZ,Unknown Region,0 -ebay.{...},postdirect.com,inbound,ZZ,Unknown Region,0 -ebizac3.com,ebizac3.com,inbound,ZZ,Unknown Region,0 -ebuildabear.com,ebuildabear.com,inbound,ZZ,Unknown Region,8e-06 -ed10.net,ed10.com,inbound,ZZ,Unknown Region,0 -eddiebauer.com,eddiebauer.com,inbound,ZZ,Unknown Region,0 -eduk.com,eduk.com,inbound,ZZ,Unknown Region,0 -efamilydollar.com,efamilydollar.com,inbound,ZZ,Unknown Region,0 -elabs10.com,elabs10.com,inbound,ZZ,Unknown Region,0 -elabs12.com,elabs12.com,inbound,ZZ,Unknown Region,0 -elistas.net,elistas.net,inbound,ZZ,Unknown Region,0 -elkjop.no,ec-cluster.com,inbound,ZZ,Unknown Region,0 -elkjop.no,eccluster.com,inbound,ZZ,Unknown Region,0 -elo7.com.br,elo7.com.br,inbound,ZZ,Unknown Region,0 -email-1800contacts.com,email-1800contacts.com,inbound,ZZ,Unknown Region,0 -email-aaa.com,email-aaa.com,inbound,ZZ,Unknown Region,0 -email-aeriagames.com,email-aeriagames.com,inbound,ZZ,Unknown Region,0 -email-dressbarn.com,email-dressbarn.com,inbound,ZZ,Unknown Region,0 -email-firestone.com,reminder-firestone.com,inbound,ZZ,Unknown Region,0 -email-honest.com,email-honest.com,inbound,ZZ,Unknown Region,0 -email-od.com,email-od.com,inbound,ZZ,Unknown Region,0.999752 -email-petsmart.com,email-petsmart.com,inbound,ZZ,Unknown Region,0 -email-sportchalet.com,email-sportchalet.com,inbound,ZZ,Unknown Region,0 -email-telekom.de,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -email-totalwine.com,email-totalwine.com,inbound,ZZ,Unknown Region,0 -email-wildstar-online.com,email-carbine.com,inbound,ZZ,Unknown Region,0 -email2-beyond.com,messagebus.com,inbound,ZZ,Unknown Region,0 -email360api.com,email360api.com,inbound,ZZ,Unknown Region,0 -email365inc.com,email365inc.com,inbound,ZZ,Unknown Region,0 -email3m.com,email3m.com,inbound,ZZ,Unknown Region,0 -emailrestaurant.com,emailrestaurant.com,inbound,ZZ,Unknown Region,0 -emailtoryburch.com,emailtoryburch.com,inbound,ZZ,Unknown Region,0 -emarsys.net,emarsys.net,inbound,ZZ,Unknown Region,0.265197 -embarqmail.com,centurylink.net,inbound,ZZ,Unknown Region,0.999918 -emergencyemail.org,emergencyemail.org,inbound,ZZ,Unknown Region,0 -eminentinc.com,eminentinc.com,inbound,ZZ,Unknown Region,0 -emktviajarbarato.com.br,splio.com.br,inbound,ZZ,Unknown Region,0.875902 -employboard.com,employboard.com,inbound,ZZ,Unknown Region,1 -emsecure.net,emsecure.net,inbound,ZZ,Unknown Region,0 -emsmtp.com,emsmtp.com,inbound,ZZ,Unknown Region,0.44284 -en25.com,kqed.org,inbound,ZZ,Unknown Region,0 -enewscartes.net,bp06.net,inbound,ZZ,Unknown Region,0 -enewsletter.pl,mydeal.pl,inbound,ZZ,Unknown Region,0 -enewsletter.pl,sare25.com,inbound,ZZ,Unknown Region,0 -enplenitud.com,enplenitud.com,inbound,ZZ,Unknown Region,0 -entrepreneur.com,entrepreneur.com,inbound,ZZ,Unknown Region,0 -ethingsremembered.com,ethingsremembered.com,inbound,ZZ,Unknown Region,0 -etrade.com,etrade.com,inbound,ZZ,Unknown Region,0 -etransmail.com,etransmail.com,inbound,ZZ,Unknown Region,0 -etransmail.com,ptransmail.com,inbound,ZZ,Unknown Region,0 -etrmailbox.com,etrmailbox.com,inbound,ZZ,Unknown Region,0 -etsy.com,etsy.com,inbound,ZZ,Unknown Region,0.020849 -evernote.com,evernote.com,inbound,ZZ,Unknown Region,1 -everydayfamily.com,everydayfamily.com,inbound,ZZ,Unknown Region,0 -everytown.org,everytown.org,inbound,ZZ,Unknown Region,1 -exacttarget.com,bazaarvoice.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,booksamillion.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,exacttarget.com,inbound,ZZ,Unknown Region,0.000325 -exacttarget.com,msg.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,redboxinstant.com,inbound,ZZ,Unknown Region,0 -exacttarget.com,skylinetechnologies.com,inbound,ZZ,Unknown Region,0 -expediamail.com,airasiago.com,inbound,ZZ,Unknown Region,1 -expediamail.com,exacttarget.com,inbound,ZZ,Unknown Region,1 -expediamail.com,expediamail.com,inbound,ZZ,Unknown Region,0.839662 -expediamail.com,quotitmail.com,inbound,ZZ,Unknown Region,0 -experteer.com,experteer.com,inbound,ZZ,Unknown Region,0 -express.com,expressfashion.com,inbound,ZZ,Unknown Region,0 -facebook.com,facebook.com,inbound,ZZ,Unknown Region,0 -facebook.com,facebook.com,outbound,ZZ,Unknown Region,1 -facebookmail.com,postini.com,inbound,ZZ,Unknown Region,0.918705 -facebookmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.999846 -falabella.com,falabella.com,inbound,ZZ,Unknown Region,0 -fanatics.com,fanatics.com,inbound,ZZ,Unknown Region,0 -fanaticsretailgroup.com,fanaticsretailgroup.com,inbound,ZZ,Unknown Region,0 -fanbridge.com,fanbridge.com,inbound,ZZ,Unknown Region,0 -fanofannas.com,fanofannas.com,inbound,ZZ,Unknown Region,0 -fansedge.com,fansedge.com,inbound,ZZ,Unknown Region,0 -farmers.com,farmers.com,inbound,ZZ,Unknown Region,0.999984 -fastcompany.com,fastcompany.com,inbound,ZZ,Unknown Region,0 -fedex.com,fedex.com,inbound,ZZ,Unknown Region,0.919932 -feld-ent.com,postdirect.com,inbound,ZZ,Unknown Region,0 -fellowshiponemail.com,fellowshiponemail.com,inbound,ZZ,Unknown Region,0 -fetlifemail.com,fetlifemail.com,inbound,ZZ,Unknown Region,0 -fidelity.com,fidelity.com,inbound,ZZ,Unknown Region,1 -financialfreedommail.com,financialfreedommail.com,inbound,ZZ,Unknown Region,0 -fingerhut.com,fingerhut.com,inbound,ZZ,Unknown Region,0 -flets.com,flets.com,inbound,ZZ,Unknown Region,0 -flirtlocal.com,flirtlocal.com,inbound,ZZ,Unknown Region,1 -flmsecure.com,fling.com,inbound,ZZ,Unknown Region,0 -flmsecure.com,flmsecure.com,inbound,ZZ,Unknown Region,0 -floridajobdepartment.com,floridajobdepartment.com,inbound,ZZ,Unknown Region,0 -fnac.com,fnac.com,inbound,ZZ,Unknown Region,0.025375 -fnb.co.za,fnb.co.za,inbound,ZZ,Unknown Region,0.325259 -foodnetwork.com,foodnetwork.com,inbound,ZZ,Unknown Region,0 -forcemail.in,iaires.com,inbound,ZZ,Unknown Region,0 -foreseegame.com,iaires.com,inbound,ZZ,Unknown Region,0 -fotffamily.com,fotffamily.com,inbound,ZZ,Unknown Region,0 -fotolivro.com.br,fotolivro.com.br,inbound,ZZ,Unknown Region,0 -fpmailerbr.com,fpmailerbr.com,inbound,ZZ,Unknown Region,0 -freecycle.org,freecycle.org,inbound,ZZ,Unknown Region,1 -freelancer.com,getafreelancer.com,inbound,ZZ,Unknown Region,0 -freesafelistmailer.com,waters-advertising.com,inbound,ZZ,Unknown Region,0 -freshmail.pl,freshmail.pl,inbound,ZZ,Unknown Region,0 -fridays.com,fridays.com,inbound,ZZ,Unknown Region,0 -frk.com,frk.com,inbound,ZZ,Unknown Region,1 -frontdoor.com,frontdoor.com,inbound,ZZ,Unknown Region,0 -frontgate-email.com,frontgate-email.com,inbound,ZZ,Unknown Region,0 -fuckbooknet.net,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -fundplaza.co.in,arrowsignindia.com,inbound,ZZ,Unknown Region,0 -fundplaza.in,fundplaza.in,inbound,ZZ,Unknown Region,0 -funonthenet.in,funonthenet.in,inbound,ZZ,Unknown Region,1 -futurmailer.pt,futurmailer.pt,inbound,ZZ,Unknown Region,0 -gabbar.info,gabbar.info,inbound,ZZ,Unknown Region,1 -gamefly.com,gamefly.com,inbound,ZZ,Unknown Region,0.014317 -gamingmails.com,gamingmails.com,inbound,ZZ,Unknown Region,0 -gap.com,gap.com,inbound,ZZ,Unknown Region,0 -gap.eu,gap.eu,inbound,ZZ,Unknown Region,0 -gapcanada.ca,gapcanada.ca,inbound,ZZ,Unknown Region,0 -garanti.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 -garnethill-email.com,garnethill-email.com,inbound,ZZ,Unknown Region,0 -gaylordalert.com,gaylordalert.com,inbound,ZZ,Unknown Region,0 -gcast.com.au,systemsserver.net,inbound,ZZ,Unknown Region,0 -gdtsuccess.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 -geico.com,geico.com,inbound,ZZ,Unknown Region,0.096795 -gene.com,roche.com,inbound,ZZ,Unknown Region,1 -geocaching.com,groundspeak.com,inbound,ZZ,Unknown Region,1 -getinbox.net,getinbox.net,inbound,ZZ,Unknown Region,0 -getkeepsafe.com,getkeepsafe.com,inbound,ZZ,Unknown Region,1 -getmein.com,getmein.com,inbound,ZZ,Unknown Region,0 -getresponse.com,getresponse.com,inbound,ZZ,Unknown Region,0 -gfsmarketplace-email.com,gfsmarketplace-email.com,inbound,ZZ,Unknown Region,0 -gilt.com,gilt.com,inbound,ZZ,Unknown Region,1e-06 -github.com,github.net,inbound,ZZ,Unknown Region,1 -glassdoor.com,glassdoor.com,inbound,ZZ,Unknown Region,0 -globalmembersupport.com,globalmembersupport.com,inbound,ZZ,Unknown Region,0 -globalspec.com,globalspec.com,inbound,ZZ,Unknown Region,0 -gmail.com,blackberry.com,inbound,ZZ,Unknown Region,0.998326 -gmail.com,postini.com,inbound,ZZ,Unknown Region,0.891175 -gmail.com,yahoo.{...},inbound,ZZ,Unknown Region,0.998661 -go.com,starwave.com,inbound,ZZ,Unknown Region,0.006276 -godtubemail.com,godtubemail.com,inbound,ZZ,Unknown Region,0 -godvinemail.com,godvinemail.com,inbound,ZZ,Unknown Region,0 -gohappy.com.tw,gohappy.com.tw,inbound,ZZ,Unknown Region,0 -goldenbrands.gr,goldenbrands.gr,inbound,ZZ,Unknown Region,1 -golfnow.com,email-golfnow.com,inbound,ZZ,Unknown Region,0 -google.com,postini.com,inbound,ZZ,Unknown Region,0.81059 -gop.com,gop.com,inbound,ZZ,Unknown Region,0 -grabone-mail-ie.com,grabone-mail-ie.com,inbound,ZZ,Unknown Region,0 -grabone-mail.com,grabone-mail.com,inbound,ZZ,Unknown Region,0 -gratka.pl,gratka.pl,inbound,ZZ,Unknown Region,0 -grocerycouponnetwork.com,grocerycouponnetwork.com,inbound,ZZ,Unknown Region,0 -groopdealz.com,groopdealz.com,inbound,ZZ,Unknown Region,1 -groupalia.es,groupalia.es,inbound,ZZ,Unknown Region,0 -groupalia.it,groupalia.it,inbound,ZZ,Unknown Region,0 -groupon.jp,data-hotel.net,inbound,ZZ,Unknown Region,0 -groupon.{...},groupon.{...},inbound,ZZ,Unknown Region,0.990176 -grouponmail.{...},grouponmail.{...},inbound,ZZ,Unknown Region,0 -grubhubmail.com,grubhubmail.com,inbound,ZZ,Unknown Region,0 -grupanya.com,euromsg.net,inbound,ZZ,Unknown Region,0 -guruin.info,guru.net.in,inbound,ZZ,Unknown Region,1 -habitaclia.com,splio.es,inbound,ZZ,Unknown Region,0.951406 -harborfreightemail.com,harborfreightemail.com,inbound,ZZ,Unknown Region,0 -hautelook.com,hautelook.com,inbound,ZZ,Unknown Region,0 -hepsiburada.com,euromsg.net,inbound,ZZ,Unknown Region,0 -herbalifemail.com,herbalifemail.com,inbound,ZZ,Unknown Region,0 -herculist.com,herculist.com,inbound,ZZ,Unknown Region,0 -hgtv.com,hgtv.com,inbound,ZZ,Unknown Region,0 -hhgreggemail.com,hhgreggemail.com,inbound,ZZ,Unknown Region,0 -hipmunk.com,hipmunk.com,inbound,ZZ,Unknown Region,0 -hln.be,persgroep-ops.net,inbound,ZZ,Unknown Region,0 -hm-f.jp,hm-f.jp,inbound,ZZ,Unknown Region,0 -hobsonsmail.com,hobsonsmail.com,inbound,ZZ,Unknown Region,0 -homebaselife.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -homechoice.co.za,homechoice.co.za,inbound,ZZ,Unknown Region,0 -homedepotemail.com,homedepotemail.com,inbound,ZZ,Unknown Region,0 -honto.jp,honto.jp,inbound,ZZ,Unknown Region,0 -horoscope.com,center.com,inbound,ZZ,Unknown Region,0 -hotels.com,hotels.com,inbound,ZZ,Unknown Region,0 -hotelurbano.com.br,allin.com.br,inbound,ZZ,Unknown Region,0 -hotmail.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999944 -hotmail.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 -hotspotmailer.com,hotspotmailer.com,inbound,ZZ,Unknown Region,1 -hp.com,hp.com,inbound,ZZ,Unknown Region,0.198696 -hsn.com,hsn.com,inbound,ZZ,Unknown Region,0 -htcampusmailer.com,eccluster.com,inbound,ZZ,Unknown Region,0 -hubspot.com,hubspot.com,inbound,ZZ,Unknown Region,1 -huinforma.com.br,huinforma.com.br,inbound,ZZ,Unknown Region,0 -hulumail.com,hulumail.com,inbound,ZZ,Unknown Region,0 -hungryhouse.co.uk,mxmfb.com,inbound,ZZ,Unknown Region,0 -huntington.com,huntington.com,inbound,ZZ,Unknown Region,0.986937 -i-say.com,ipsos-interactive.com,inbound,ZZ,Unknown Region,1 -icloud.com,apple.com,inbound,ZZ,Unknown Region,1 -icloud.com,icloud.com,outbound,ZZ,Unknown Region,1 -icloud.com,mac.com,inbound,ZZ,Unknown Region,1 -icloud.com,me.com,inbound,ZZ,Unknown Region,0.999995 -ifttt.com,ifttt.com,inbound,ZZ,Unknown Region,1 -ign.com,ign.com,inbound,ZZ,Unknown Region,0 -ignitionsender.com,ignitionsender.com,inbound,ZZ,Unknown Region,0 -iheart.com,iheart.com,inbound,ZZ,Unknown Region,0 -immobilienscout24.de,immobilienscout24.de,inbound,ZZ,Unknown Region,1 -in-boxpays.com,in-boxpays.com,inbound,ZZ,Unknown Region,0 -indiamart.com,indiamart.com,inbound,ZZ,Unknown Region,1 -indiatimes.com,speakingtree.in,inbound,ZZ,Unknown Region,0 -indiatimeshop.com,sendpal.in,inbound,ZZ,Unknown Region,0 -indieroyale.com,desura.com,inbound,ZZ,Unknown Region,1 -infibeam.com,eccluster.com,inbound,ZZ,Unknown Region,0 -infopanel.jp,mailds.jp,inbound,ZZ,Unknown Region,0 -infos-micromania.com,infos-micromania.com,inbound,ZZ,Unknown Region,0 -infosephora.com,splio.com,inbound,ZZ,Unknown Region,0.962167 -infoworld.com,infoworld.com,inbound,ZZ,Unknown Region,0 -infusionmail.com,infusionmail.com,inbound,ZZ,Unknown Region,0 -inmotionhosting.com,inmotionhosting.com,inbound,ZZ,Unknown Region,1 -ino.com,ino.com,inbound,ZZ,Unknown Region,0.999981 -interwell.gr,interwell.gr,inbound,ZZ,Unknown Region,0 -ipcmedia.co.uk,ipcmedia.co.uk,inbound,ZZ,Unknown Region,0 -itunes.com,apple.com,inbound,ZZ,Unknown Region,0.043012 -jackwills.com,jackwills.com,inbound,ZZ,Unknown Region,0 -jalag.de,jalag.de,inbound,ZZ,Unknown Region,1 -jane.com,jane.com,inbound,ZZ,Unknown Region,1 -jango.com,jango.com,inbound,ZZ,Unknown Region,0 -jared.com,jared.com,inbound,ZZ,Unknown Region,0 -jdate.com,postdirect.com,inbound,ZZ,Unknown Region,0 -jetprivilege.com,jetprivilege.com,inbound,ZZ,Unknown Region,0 -jeuxvideo.com,jeuxvideo.com,inbound,ZZ,Unknown Region,0.148921 -jeweloscoemail.com,email-mywebgrocer2.com,inbound,ZZ,Unknown Region,0 -joann-mail.com,joann-mail.com,inbound,ZZ,Unknown Region,0 -jobinsider.com,jobinsider.com,inbound,ZZ,Unknown Region,0 -jobisjob.com,jobisjob.com,inbound,ZZ,Unknown Region,0 -jobomas.com,jobomas.com,inbound,ZZ,Unknown Region,1 -jobstreet.com,jobstreet.com,inbound,ZZ,Unknown Region,0 -jpcycles.com,jpcycles.com,inbound,ZZ,Unknown Region,0.001716 -jungleerummy.com,jungleerummy.com,inbound,ZZ,Unknown Region,1 -jusbrasil.com.br,jusbrasil.com.br,inbound,ZZ,Unknown Region,0 -just-eat.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -justclick.ru,justclick.ru,inbound,ZZ,Unknown Region,0 -justdial.com,iaires.com,inbound,ZZ,Unknown Region,0 -k1speed.com,k1speed.com,inbound,ZZ,Unknown Region,1 -kaskusnetworks.com,kaskus.com,inbound,ZZ,Unknown Region,0 -kay.com,kay.com,inbound,ZZ,Unknown Region,0 -keek.com,keek.com,inbound,ZZ,Unknown Region,1 -kgbdeals.co.uk,email1-kgbdeals.com,inbound,ZZ,Unknown Region,0 -kliksa.net,euromsg.net,inbound,ZZ,Unknown Region,0 -kliktoday.com,kliktoday.com,inbound,ZZ,Unknown Region,0 -klm-mail.com,klm-mail.com,inbound,ZZ,Unknown Region,0 -kohls.com,kohls.com,inbound,ZZ,Unknown Region,0 -kongregate.com,kongregate.com,inbound,ZZ,Unknown Region,0 -krs.bz,tricorn.net,inbound,ZZ,Unknown Region,0 -la-meteo-mail.fr,splio.com,inbound,ZZ,Unknown Region,1 -laaptuemail.com,laaptuemail.com,inbound,ZZ,Unknown Region,0 -lakewoodchurch.com,lakewoodchurch.com,inbound,ZZ,Unknown Region,0 -landsend.com,email-landsend.com,inbound,ZZ,Unknown Region,0 -landsend.com,postdirect.com,inbound,ZZ,Unknown Region,0 -laptuinvite.com,laptuinvite.com,inbound,ZZ,Unknown Region,0 -lastminute.com,lastminute.com,inbound,ZZ,Unknown Region,0 -lazerhits.com,lazerhits.com,inbound,ZZ,Unknown Region,1 -leadercontato.com.br,leadercontato.com.br,inbound,ZZ,Unknown Region,0 -lefigaro.fr,splio.com,inbound,ZZ,Unknown Region,1 -lemonde.fr,lemonde.fr,inbound,ZZ,Unknown Region,0 -life360.com,life360.com,inbound,ZZ,Unknown Region,1 -lifecare-news.com,email-lifecare.com,inbound,ZZ,Unknown Region,0 -lifemiles.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -lifescript.com,ilinkmd.com,inbound,ZZ,Unknown Region,0 -line6.com,line6.com,inbound,ZZ,Unknown Region,0 -linkedin.com,linkedin.com,inbound,ZZ,Unknown Region,0.995201 -linkedin.com,postini.com,inbound,ZZ,Unknown Region,0.940251 -liquidation.com,liquidation.com,inbound,ZZ,Unknown Region,0 -listbuildingmaximizer.com,listbuildingmaximizer.com,inbound,ZZ,Unknown Region,0.00023 -live.{...},hotmail.{...},inbound,ZZ,Unknown Region,0.999921 -live.{...},hotmail.{...},outbound,ZZ,Unknown Region,1 -livejournal.com,livejournal.com,inbound,ZZ,Unknown Region,0 -livemailservice.com,livemailservice.com,inbound,ZZ,Unknown Region,0 -livenation.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -lmlmgv.com.br,gvarev.com.br,inbound,ZZ,Unknown Region,0 -loccitane.com,neolane.net,inbound,ZZ,Unknown Region,0 -loft.com,anntaylor.com,inbound,ZZ,Unknown Region,0 -lombardipublishing.com,lombardipublishing.com,inbound,ZZ,Unknown Region,0 -lookout.com,lookout.com,inbound,ZZ,Unknown Region,1 -lsi.com,postini.com,inbound,ZZ,Unknown Region,0.981115 -lynxmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -lyris.net,lyris.net,inbound,ZZ,Unknown Region,0 -m1e.net,m1e.net,inbound,ZZ,Unknown Region,0.000342 -mac.com,icloud.com,outbound,ZZ,Unknown Region,1 -mac.com,mac.com,inbound,ZZ,Unknown Region,1 -macromill.com,macromill.com,inbound,ZZ,Unknown Region,0 -macys.com,macys.com,inbound,ZZ,Unknown Region,0 -magix.net,magix.net,inbound,ZZ,Unknown Region,0.673176 -mail-backcountry.com,email-bcmarketing.com,inbound,ZZ,Unknown Region,0 -mail.ru,mail.ru,inbound,ZZ,Unknown Region,0.986526 -mail.ru,mail.ru,outbound,ZZ,Unknown Region,0.006561 -maileclipse.com,emce2.in,inbound,ZZ,Unknown Region,0 -mailengine1.com,mailengine1.com,inbound,ZZ,Unknown Region,0 -mailer4u.in,elabs10.com,inbound,ZZ,Unknown Region,0 -mailersend.com,mailersend.com,inbound,ZZ,Unknown Region,0 -mailjet.com,mailjet.com,inbound,ZZ,Unknown Region,0.803083 -mailmailmail.net,mailmailmail.net,inbound,ZZ,Unknown Region,0 -mailoct.in,tcmailer14.in,inbound,ZZ,Unknown Region,0 -mailoct1.in,mailoct1.in,inbound,ZZ,Unknown Region,0 -mailoct1.in,myntramail2.in,inbound,ZZ,Unknown Region,0 -mailorama.fr,mailorama.fr,inbound,ZZ,Unknown Region,0 -mailpost.in,iaires.com,inbound,ZZ,Unknown Region,0 -mailquant.com,iaires.com,inbound,ZZ,Unknown Region,0 -mandrillapp.com,backpage.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,mcsignup.com,inbound,ZZ,Unknown Region,1 -mandrillapp.com,myjobhelperalerts.com,inbound,ZZ,Unknown Region,1 -mango.com,emstechnology2.net,inbound,ZZ,Unknown Region,0 -manipal.edu,iaires.com,inbound,ZZ,Unknown Region,0 -manta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -mar0.net,mar0.net,inbound,ZZ,Unknown Region,0.976409 -markavip.com,markavip.com,inbound,ZZ,Unknown Region,0 -marketingstudio.com,marketingstudio.com,inbound,ZZ,Unknown Region,0 -marksandspencer.com,marksandspencer.com,inbound,ZZ,Unknown Region,0 -maropost.com,mp2200.com,inbound,ZZ,Unknown Region,0 -maropost.com,survivallife.com,inbound,ZZ,Unknown Region,0 -marykay.com,marykay.com,inbound,ZZ,Unknown Region,0 -masivapp.com,masivapp.com,inbound,ZZ,Unknown Region,1 -match.com,match.com,inbound,ZZ,Unknown Region,0 -mbga.jp,mbga.jp,inbound,ZZ,Unknown Region,0 -mbna.co.uk,ec-cluster.com,inbound,ZZ,Unknown Region,0 -mcdlv.net,mcdlv.net,inbound,ZZ,Unknown Region,0 -mcsv.net,mcsv.net,inbound,ZZ,Unknown Region,0 -me.com,icloud.com,outbound,ZZ,Unknown Region,1 -me.com,mac.com,inbound,ZZ,Unknown Region,1 -medallia.com,medallia.com,inbound,ZZ,Unknown Region,1 -mediapost.com,mediapost.com,inbound,ZZ,Unknown Region,0 -medium.com,messagebus.com,inbound,ZZ,Unknown Region,0 -medscape.com,medscape.com,inbound,ZZ,Unknown Region,0 -meetup.com,meetup.com,inbound,ZZ,Unknown Region,0 -melaleuca.com,melaleuca.com,inbound,ZZ,Unknown Region,0 -mercadojobs.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -mercadolibre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 -mercadolivre.com,mercadolibre.com,inbound,ZZ,Unknown Region,0 -merceworld.com,merceworld.com,inbound,ZZ,Unknown Region,0.996737 -merodea.me,sendgrid.net,inbound,ZZ,Unknown Region,1 -metro.co.in,srv2.de,inbound,ZZ,Unknown Region,0.966947 -mgmresorts.com,mgmresorts.com,inbound,ZZ,Unknown Region,0 -microsoft.com,msn.com,inbound,ZZ,Unknown Region,1 -microsoftemail.com,microsoftemail.com,inbound,ZZ,Unknown Region,0 -microsoftemail.com,microsoftstoreemail.com,inbound,ZZ,Unknown Region,0 -mileageplusshoppingnews.com,mail-skymilesshoppingsupport.com,inbound,ZZ,Unknown Region,0 -minhavida.com.br,minhavida.com.br,inbound,ZZ,Unknown Region,0 -mixi.jp,mixi.jp,inbound,ZZ,Unknown Region,0 -mjinn.com,mailurja.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktdns.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktomail.com,inbound,ZZ,Unknown Region,0 -mktomail.com,mktroute.com,inbound,ZZ,Unknown Region,0 -mlsend.com,mlsend.com,inbound,ZZ,Unknown Region,0 -mlsend2.com,mlsend2.com,inbound,ZZ,Unknown Region,0 -mlssoccer.com,mlssoccer.com,inbound,ZZ,Unknown Region,0 -mmaco.net,mmaco.net,inbound,ZZ,Unknown Region,0.999998 -mmorpg.com,mmorpg.com,inbound,ZZ,Unknown Region,0.002979 -mocospace.com,mocospace.com,inbound,ZZ,Unknown Region,0 -moneyforward.com,moneyforward.com,inbound,ZZ,Unknown Region,0 -moneysupermarketmail.com,moneysupermarketmail.com,inbound,ZZ,Unknown Region,0 -monografias.com,elistas.net,inbound,ZZ,Unknown Region,0 -monster.com,monster.com,inbound,ZZ,Unknown Region,0.000503 -monsterindia.com,monster.co.in,inbound,ZZ,Unknown Region,0 -mooply.co,mailendo.com,inbound,ZZ,Unknown Region,0 -morningstar.net,morningstar.net,inbound,ZZ,Unknown Region,0 -mothercaregroup.com,neolane.net,inbound,ZZ,Unknown Region,0 -mozilla.org,mozilla.com,inbound,ZZ,Unknown Region,0.633241 -mpse.jp,mpme.jp,inbound,ZZ,Unknown Region,0 -ms00.net,ms00.net,inbound,ZZ,Unknown Region,0 -msdp1.com,msdp1.com,inbound,ZZ,Unknown Region,0 -msn.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999946 -msn.com,hotmail.{...},outbound,ZZ,Unknown Region,1 -musiciansfriend.com,musiciansfriend.com,inbound,ZZ,Unknown Region,0 -mxmfb.com,mxmfb.com,inbound,ZZ,Unknown Region,0 -mycolorscreen.com,mta4.net,inbound,ZZ,Unknown Region,0 -myfitnesspal.com,messagebus.com,inbound,ZZ,Unknown Region,0 -mygroupon.co.th,grouponmail.{...},inbound,ZZ,Unknown Region,0 -myheritage.com,myheritage.com,inbound,ZZ,Unknown Region,0 -myideeli.com,myideeli.com,inbound,ZZ,Unknown Region,0 -myntramail.com,iaires.com,inbound,ZZ,Unknown Region,0 -myntramail.com,myntramail.com,inbound,ZZ,Unknown Region,0 -myntramails.in,icubes.in,inbound,ZZ,Unknown Region,0 -myoutlets.in,trustmailer.com,inbound,ZZ,Unknown Region,0 -mypoints.com,mypoints.com,inbound,ZZ,Unknown Region,0 -mysale.my,mysale.my,inbound,ZZ,Unknown Region,0 -mysale.ph,mysale.ph,inbound,ZZ,Unknown Region,0 -mysupermarket.co.uk,mysupermarket.co.uk,inbound,ZZ,Unknown Region,0 -mysurvey.com,mysurvey.com,inbound,ZZ,Unknown Region,0 -mysurvey.eu,mysurvey.com,inbound,ZZ,Unknown Region,0 -myvegas.com,myvegas.com,inbound,ZZ,Unknown Region,1 -naaptoldeals.com,eccluster.com,inbound,ZZ,Unknown Region,0 -nanomail.com.br,araie.com.br,inbound,ZZ,Unknown Region,1 -nascar.com,nascar.com,inbound,ZZ,Unknown Region,0 -nationbuilder.com,nationbuilder.com,inbound,ZZ,Unknown Region,1 -nationwide-communications.co.uk,nationwide-communications.co.uk,inbound,ZZ,Unknown Region,0 -naukri.com,naukri.com,inbound,ZZ,Unknown Region,0 -navy.mil,navy.mil,inbound,ZZ,Unknown Region,0.000243 -nend.net,postini.com,inbound,ZZ,Unknown Region,0 -netatlantic.com,netatlantic.com,inbound,ZZ,Unknown Region,0.001085 -netflix.com,amazonses.com,inbound,ZZ,Unknown Region,0.999999 -netflix.com,netflix.com,inbound,ZZ,Unknown Region,1 -netlogmail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 -netprosoftmail.com,netprosoftmail.com,inbound,ZZ,Unknown Region,0 -netshoes.com.br,netshoes.com.br,inbound,ZZ,Unknown Region,0 -newegg.com,newegg.com,inbound,ZZ,Unknown Region,2e-06 -newmarkethealth.com,newmarkethealth.com,inbound,ZZ,Unknown Region,0 -news-h5g.com,news-h5g.com,inbound,ZZ,Unknown Region,0 -newsletter-verychic.com,splio.es,inbound,ZZ,Unknown Region,0.9804 -newsmax.com,newsmax.com,inbound,ZZ,Unknown Region,0 -nflshop.com,nflshop.com,inbound,ZZ,Unknown Region,0 -nieuwsblad.be,vummail.be,inbound,ZZ,Unknown Region,0 -nike.com,nike.com,inbound,ZZ,Unknown Region,0 -ninewestmail.com,ninewestmail.com,inbound,ZZ,Unknown Region,0 -nokia.com,nokia.com,inbound,ZZ,Unknown Region,0.001256 -npr.org,npr.org,inbound,ZZ,Unknown Region,0 -ns.nl,tripolis.com,inbound,ZZ,Unknown Region,0 -nsandi.com,mxmfb.com,inbound,ZZ,Unknown Region,0 -nytimes.com,nytimes.com,inbound,ZZ,Unknown Region,0 -nzsale.co.nz,nzsale.co.nz,inbound,ZZ,Unknown Region,0 -oakley.com,oakley.com,inbound,ZZ,Unknown Region,0 -ocmail1.in,tcmail.in,inbound,ZZ,Unknown Region,0 -ocmail14.in,tcmailer5.in,inbound,ZZ,Unknown Region,0 -ocmail22.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 -ocmail22.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 -ocmail40.in,tcmailer15.in,inbound,ZZ,Unknown Region,0 -ocmail40.in,tcmailer4.in,inbound,ZZ,Unknown Region,0 -ocnmail.in,ocmail6.in,inbound,ZZ,Unknown Region,0 -ocnmail.in,tcmail3.in,inbound,ZZ,Unknown Region,0 -ofertasbmc.com.br,ofertasbmc.com.br,inbound,ZZ,Unknown Region,0 -ofertix.com,ofertix.com,inbound,ZZ,Unknown Region,0 -offers.com,offers.com,inbound,ZZ,Unknown Region,1 -officedepot.com,officedepot.com,inbound,ZZ,Unknown Region,0.019269 -officemax.com,officemax.com,inbound,ZZ,Unknown Region,0 -officemax.com,officemaxworkplace.com,inbound,ZZ,Unknown Region,0 -ofsys.com,bulletin-metro.ca,inbound,ZZ,Unknown Region,0 -oknotify2.com,oknotify2.com,inbound,ZZ,Unknown Region,0 -oldnavy.ca,oldnavy.ca,inbound,ZZ,Unknown Region,0 -oldnavy.com,oldnavy.com,inbound,ZZ,Unknown Region,0 -omahasteaks.com,omahasteaks.com,inbound,ZZ,Unknown Region,0 -oneindia.in,mailurja.com,inbound,ZZ,Unknown Region,0 -onekingslane.com,onekingslane.com,inbound,ZZ,Unknown Region,0 -onlive.com,ipost.com,inbound,ZZ,Unknown Region,0 -onmicrosoft.com,outlook.com,inbound,ZZ,Unknown Region,1 -optimusmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -orderscatalog.com,orderscatalog.com,inbound,ZZ,Unknown Region,0 -oroscopofree.com,adsender.us,inbound,ZZ,Unknown Region,0 -os-email.com,os-email.com,inbound,ZZ,Unknown Region,0 -oshkoshbgosh.com,oshkoshbgosh.com,inbound,ZZ,Unknown Region,6e-06 -osu.edu,outlook.com,inbound,ZZ,Unknown Region,1 -otto.de,eccluster.com,inbound,ZZ,Unknown Region,1 -ouffer.com,ouffer.com,inbound,ZZ,Unknown Region,0 -ourtime.com,seniorpeoplemeet.com,inbound,ZZ,Unknown Region,0 -outback.com,outback.com,inbound,ZZ,Unknown Region,0 -outlook.com,hotmail.{...},inbound,ZZ,Unknown Region,0.999843 -outlook.com,hotmail.{...},outbound,ZZ,Unknown Region,1 -outspot.be,teneo.be,inbound,ZZ,Unknown Region,0 -outspot.nl,teneo.be,inbound,ZZ,Unknown Region,0 -ovenmail.com,iaires.com,inbound,ZZ,Unknown Region,0 -overstock.com,overstock.com,inbound,ZZ,Unknown Region,0 -ovh.net,ovh.net,inbound,ZZ,Unknown Region,0.352531 -ozsale.com.au,ozsale.com.au,inbound,ZZ,Unknown Region,0.000192 -panelplace.com,smtp.com,inbound,ZZ,Unknown Region,0 -panerabreadnews.com,panerabreadnews.com,inbound,ZZ,Unknown Region,0 -pantaloondirect.net,iaires.com,inbound,ZZ,Unknown Region,0 -papajohns-specials.com,papajohns-specials.com,inbound,ZZ,Unknown Region,0 -paradisepublishers.com,paradisepublishers.com,inbound,ZZ,Unknown Region,0.99957 -path.com,path.com,inbound,ZZ,Unknown Region,1 -payback.in,eccluster.com,inbound,ZZ,Unknown Region,0 -payback.in,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -payback.in,ecmcluster.com,inbound,ZZ,Unknown Region,0 -paypal.co.uk,paypal.com,inbound,ZZ,Unknown Region,1 -paypal.com,paypal.com,inbound,ZZ,Unknown Region,0.608834 -paypal.com.au,paypal.com,inbound,ZZ,Unknown Region,1 -paypal.de,paypal.com,inbound,ZZ,Unknown Region,1 -pd25.com,pd25.com,inbound,ZZ,Unknown Region,1 -peanuthome.info,adopterc.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,aguitytr.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bevest.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bluester.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,burror.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,bursion.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cantexi.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,caserhi.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,celect.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,chintone.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,citery.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cleathal.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,colicom.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,complec.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,deciarc.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,declaws.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,dewest.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,epconce.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,fnotec.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,folkswor.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,forepert.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,gurgaro.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,heallyps.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,holeph.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,homewor.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,hydroni.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,ingenbu.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 -peanuthome.info,unglibshudder.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,adcrent.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,addmiel.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,agilhe.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,allegap.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,andvore.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,angogl.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,animass.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,arettery.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,aribank.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,arkefoc.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,avenog.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,barrave.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bindowmo.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bitravit.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,borsand.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,branti.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,breaserp.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bredogly.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,briantra.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,bridea.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,carial.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,castac.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,chedoner.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,chiquent.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cinchoi.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cliate.info,inbound,ZZ,Unknown Region,0 -peanutwebmaster.info,cognn.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,abjibbin.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,audiette.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,blancer.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,bluester.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,burror.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,bursion.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,caserhi.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,celect.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,cleathal.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,coherentrequittal.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,complec.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,condost.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,cyprinoidkaiserdom.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,deciarc.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,declaws.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,epconce.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ferrayer.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,fnotec.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,folkswor.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,forepert.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,gurgaro.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,holeph.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,homewor.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,hydroni.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ingenbu.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,kinklybotaurus.info,inbound,ZZ,Unknown Region,0 -peanutwebsite.info,ninetiethwhiffet.info,inbound,ZZ,Unknown Region,0 -peixeurbano.com.br,peixeurbano.com.br,inbound,ZZ,Unknown Region,0 -pennwell.com,pennwell.com,inbound,ZZ,Unknown Region,0 -perfectworld.com,perfectworld.com,inbound,ZZ,Unknown Region,0 -personare.com.br,personare.com.br,inbound,ZZ,Unknown Region,0 -pge.com,pge.com,inbound,ZZ,Unknown Region,0.149792 -philosophy.com,philosophy.com,inbound,ZZ,Unknown Region,0 -phpclasses.org,phpclasses.org,inbound,ZZ,Unknown Region,0 -pinger.com,pinger.com,inbound,ZZ,Unknown Region,0 -pinterest.com,pinterest.com,inbound,ZZ,Unknown Region,1 -pizzahutoffers.com,pizzahutoffers.com,inbound,ZZ,Unknown Region,0 -playstationmail.net,playstationmail.net,inbound,ZZ,Unknown Region,0 -plumdistrict.com,plumdistrict.com,inbound,ZZ,Unknown Region,1 -politicoemail.com,politicoemail.com,inbound,ZZ,Unknown Region,0 -polyvore.com,polyvore.com,inbound,ZZ,Unknown Region,1 -popsugar.com,popsugar.com,inbound,ZZ,Unknown Region,0 -priceline.com,priceline.com,inbound,ZZ,Unknown Region,1 -princess.com,princess.com,inbound,ZZ,Unknown Region,0 -privoscite.si,privoscite.si,inbound,ZZ,Unknown Region,0 -profitcenteronline.com,groupdealtools.com,inbound,ZZ,Unknown Region,1 -progressive.com,progressive.com,inbound,ZZ,Unknown Region,0.814003 -publix.com,publix.com,inbound,ZZ,Unknown Region,0 -puritan.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 -qoinpro.com,qoinpro.com,inbound,ZZ,Unknown Region,1 -qoo10.jp,qoo10.jp,inbound,ZZ,Unknown Region,4e-06 -qoo10.sg,qoo10.co.id,inbound,ZZ,Unknown Region,1.9e-05 -qoo10.sg,qoo10.com,inbound,ZZ,Unknown Region,0 -qoo10.sg,qoo10.my,inbound,ZZ,Unknown Region,2.2e-05 -qoo10.sg,qoo10.sg,inbound,ZZ,Unknown Region,8e-06 -quinstreet.com,neoquin.com,inbound,ZZ,Unknown Region,0 -quora.com,quora.com,inbound,ZZ,Unknown Region,1 -rabota.ua,rabota.ua,inbound,ZZ,Unknown Region,0 -rackroom-email.com,rackroom-email.com,inbound,ZZ,Unknown Region,0 -railcard-daysoutguide.co.uk,railcard-daysoutguide.co.uk,inbound,ZZ,Unknown Region,0 -rakuten.co.jp,rakuten.co.jp,inbound,ZZ,Unknown Region,0 -rakuten.com,rakuten.com,inbound,ZZ,Unknown Region,0 -reactiveadz.com,downlinebuilderdirect.com,inbound,ZZ,Unknown Region,0 -realage-mail.com,postdirect.com,inbound,ZZ,Unknown Region,0 -redbox.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -redcross.org.uk,redcross.org.uk,inbound,ZZ,Unknown Region,0 -rediffmail.com,rediffmail.com,inbound,ZZ,Unknown Region,0 -reebonz.com,reebonz.com,inbound,ZZ,Unknown Region,0.759458 -reed.co.uk,reed.co.uk,inbound,ZZ,Unknown Region,0 -regie11.net,odiso.net,inbound,ZZ,Unknown Region,0 -regionalhelpwanted.com,regionalhelpwanted.com,inbound,ZZ,Unknown Region,1 -registrar-servers.com,registrar-servers.com,inbound,ZZ,Unknown Region,0.05163 -repica.jp,kakaku.com,inbound,ZZ,Unknown Region,0 -repica.jp,repica.jp,inbound,ZZ,Unknown Region,0 -republicwireless.com,republicwireless.com,inbound,ZZ,Unknown Region,0.033564 -retailjobinsider.com,retailjobinsider.com,inbound,ZZ,Unknown Region,0 -retailmenot.com,retailmenot.com,inbound,ZZ,Unknown Region,4.68294583517529e-07 -reverbnation.com,reverbnation.com,inbound,ZZ,Unknown Region,0 -revolutiongolf.com,revolutiongolf.com,inbound,ZZ,Unknown Region,0 -reyrey.net,reyrey.net,inbound,ZZ,Unknown Region,0.999904 -richyrichmailer.com,maddog-productions.info,inbound,ZZ,Unknown Region,1 -rikunabi.com,rikunabi.com,inbound,ZZ,Unknown Region,0 -ringcentral.com,ringcentral.com,inbound,ZZ,Unknown Region,1 -rmtr.de,rapidmail.de,inbound,ZZ,Unknown Region,0 -rnmk.com,rnmk.com,inbound,ZZ,Unknown Region,0 -rockpath.info,rockpath.info,inbound,ZZ,Unknown Region,1 -rogers.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -rogers.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -rookiestewsemails.com,rookiestewsemails.com,inbound,ZZ,Unknown Region,0 -royalcaribbeanmarketing.com,royalcaribbeanmarketing.com,inbound,ZZ,Unknown Region,0 -rpo9usa.email,rpo9usa.email,inbound,ZZ,Unknown Region,0 -rr.com,rr.com,inbound,ZZ,Unknown Region,2e-06 -rr.com,rr.com,outbound,ZZ,Unknown Region,0 -rsgsv.net,rsgsv.net,inbound,ZZ,Unknown Region,0 -rummycirclemails.com,eccluster.com,inbound,ZZ,Unknown Region,0 -runkeeper.com,runkeeper.com,inbound,ZZ,Unknown Region,0 -s3s-br1.net,splio.com.br,inbound,ZZ,Unknown Region,0.908187 -s3s-main.net,splio.com,inbound,ZZ,Unknown Region,0.970428 -s4s-pl1.pl,splio.com,inbound,ZZ,Unknown Region,0.939032 -safe-sender.net,safe-sender.net,inbound,ZZ,Unknown Region,0 -safeway.com,safeway.com,inbound,ZZ,Unknown Region,0.000382 -salesforce.com,postini.com,inbound,ZZ,Unknown Region,0.915635 -salesforce.com,salesforce.com,inbound,ZZ,Unknown Region,0.952721 -samsungusa.com,samsungusa.com,inbound,ZZ,Unknown Region,0 -sanmina-sci.com,postini.com,inbound,ZZ,Unknown Region,0.999991 -sanmina.com,postini.com,inbound,ZZ,Unknown Region,0.999828 -saturday.com,saturday.com,inbound,ZZ,Unknown Region,0 -sbcglobal.net,yahoo.{...},inbound,ZZ,Unknown Region,0.999985 -sears.ca,sears.ca,inbound,ZZ,Unknown Region,0.005447 -searscard.com,searscard.com,inbound,ZZ,Unknown Region,1 -secretescapes.com,secretescapes.com,inbound,ZZ,Unknown Region,0 -secure.ne.jp,secure.ne.jp,inbound,ZZ,Unknown Region,0.000618 -secureserver.net,secureserver.net,inbound,ZZ,Unknown Region,0 -seek.com.au,seek.com.au,inbound,ZZ,Unknown Region,0 -selectacast.net,selectacast.net,inbound,ZZ,Unknown Region,0.000561 -semana.com,semana.com,inbound,ZZ,Unknown Region,0 -sendgrid.info,sendgrid.net,inbound,ZZ,Unknown Region,0.999896 -sendgrid.me,sendgrid.net,inbound,ZZ,Unknown Region,1 -sendpal.in,sendpal.in,inbound,ZZ,Unknown Region,0 -serviciobancomer.com,serviciobancomer.com,inbound,ZZ,Unknown Region,0 -sfid01.com,sfid01.com,inbound,ZZ,Unknown Region,0 -shaadi.com,shaadi.com,inbound,ZZ,Unknown Region,0 -shop2gether.com.br,shop2gether.com.br,inbound,ZZ,Unknown Region,0 -shopjustice.com,shopjustice.com,inbound,ZZ,Unknown Region,0 -shopnineteenmails.in,iaires.com,inbound,ZZ,Unknown Region,0 -shoppersstop.com,shoppersstop.com,inbound,ZZ,Unknown Region,0 -shoprite-email.com,email-mywebgrocer.com,inbound,ZZ,Unknown Region,0 -showingtime.com,showingtime.com,inbound,ZZ,Unknown Region,0 -showroomprive.com,showroomprive.be,inbound,ZZ,Unknown Region,0 -showroomprive.com,showroomprive.nl,inbound,ZZ,Unknown Region,0 -showroomprive.es,showroomprive.es,inbound,ZZ,Unknown Region,0 -showroomprive.it,showroomprive.pt,inbound,ZZ,Unknown Region,0 -showroomprive.pt,showroomprive.co.uk,inbound,ZZ,Unknown Region,0 -shtyle.fm,shtyle.fm,inbound,ZZ,Unknown Region,0 -simplesafelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -simpletextadz.com,web-hosting.com,inbound,ZZ,Unknown Region,1 -singsale.com.sg,singsale.com.sg,inbound,ZZ,Unknown Region,0 -skillpages-mailer.com,dynect.net,inbound,ZZ,Unknown Region,0 -skymall.com,skymall.com,inbound,ZZ,Unknown Region,0 -skynet.be,belgacom.be,inbound,ZZ,Unknown Region,0 -skype.com,skype.com,inbound,ZZ,Unknown Region,0 -skyscanner.net,skyscanner.net,inbound,ZZ,Unknown Region,1 -slickdeals.net,slickdeals.net,inbound,ZZ,Unknown Region,0 -slidesharemail.com,slideshare.net,inbound,ZZ,Unknown Region,1 -smartresponder.ru,smartresponder.ru,inbound,ZZ,Unknown Region,1 -snagajob-email.com,snagajob-email.com,inbound,ZZ,Unknown Region,0 -snapdeal.com,snapdeal.com,inbound,ZZ,Unknown Region,0 -socialsex.biz,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -softbank.jp,softbank.jp,outbound,ZZ,Unknown Region,0 -softbank.ne.jp,softbank.ne.jp,inbound,ZZ,Unknown Region,0 -softbank.ne.jp,softbank.ne.jp,outbound,ZZ,Unknown Region,0 -sony.com,sony.com,inbound,ZZ,Unknown Region,0 -sonyrewards.com,sonyrewards.com,inbound,ZZ,Unknown Region,0 -soundcloudmail.com,soundcloudmail.com,inbound,ZZ,Unknown Region,0.999996 -spanishdict.com,spanishdict.com,inbound,ZZ,Unknown Region,0 -sparklist.com,sparklist.com,inbound,ZZ,Unknown Region,0 -spartoo.com,spartoo.com,inbound,ZZ,Unknown Region,0 -speedyrewards-email.com,speedyrewards-email.com,inbound,ZZ,Unknown Region,0 -spotifymail.com,spotifymail.com,inbound,ZZ,Unknown Region,1 -ssgadm.com,ssg.com,inbound,ZZ,Unknown Region,0 -staffeazymailers.com,iaires.com,inbound,ZZ,Unknown Region,0 -stakemail.com,iaires.com,inbound,ZZ,Unknown Region,0 -stampmail.in,iaires.com,inbound,ZZ,Unknown Region,0 -standaard.be,vummail.be,inbound,ZZ,Unknown Region,0 -stansberryresearch.com,stansberry-re.net,inbound,ZZ,Unknown Region,0 -stansberryresearch.com,stansberryresearch.com,inbound,ZZ,Unknown Region,0 -staples.co.uk,ncrwebhost.de,inbound,ZZ,Unknown Region,0 -starsports.com,eccluster.com,inbound,ZZ,Unknown Region,0 -startwire.com,jobsreport.com,inbound,ZZ,Unknown Region,1 -starwoodhotels.com,outlook.com,inbound,ZZ,Unknown Region,1 -state-of-the-art-mailer.com,futurebanners.net,inbound,ZZ,Unknown Region,0 -stayfriends.de,stayfriends.de,inbound,ZZ,Unknown Region,0 -steampowered.com,steampowered.com,inbound,ZZ,Unknown Region,1 -stelladot.com,stelladot.com,inbound,ZZ,Unknown Region,0 -stjobs.sg,st701.com,inbound,ZZ,Unknown Region,1 -stjude.org,stjude.org,inbound,ZZ,Unknown Region,0.006723 -strava.com,strava.com,inbound,ZZ,Unknown Region,1 -streeteasy.com,streeteasy.com,inbound,ZZ,Unknown Region,0 -stylecareers.com,stylecareers.com,inbound,ZZ,Unknown Region,0 -subtend.info,subtend.info,inbound,ZZ,Unknown Region,1 -subway.com,subway.com,inbound,ZZ,Unknown Region,0 -surveyspot.com,ssisurveys.com,inbound,ZZ,Unknown Region,0 -sweepstakesalerts.com,sweepstakesalerts.com,inbound,ZZ,Unknown Region,0 -swimoutlet.com,isport.com,inbound,ZZ,Unknown Region,0 -sympatico.ca,hotmail.{...},inbound,ZZ,Unknown Region,1 -synchronyfinancial.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tadtopmails.com,tadtopmails.com,inbound,ZZ,Unknown Region,0 -taishinbank.com.tw,taishinbank.com.tw,inbound,ZZ,Unknown Region,0 -take2games.com,take2games.com,inbound,ZZ,Unknown Region,0 -talkmatch.com,talkmatch.com,inbound,ZZ,Unknown Region,0 -tanga.com,tanga.com,inbound,ZZ,Unknown Region,0 -tanningmail.com,tanningmail.com,inbound,ZZ,Unknown Region,0 -tappingsolutionemail.com,tappingsolutionemail.com,inbound,ZZ,Unknown Region,0 -target.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tasteofhome.com,tasteofhome.com,inbound,ZZ,Unknown Region,0 -tchibo.com.tr,euromsg.net,inbound,ZZ,Unknown Region,0 -teambuymail.com,teambuymail.com,inbound,ZZ,Unknown Region,0 -teamsnap.com,teamsnap.com,inbound,ZZ,Unknown Region,1 -technolutions.net,technolutions.net,inbound,ZZ,Unknown Region,1 -telegraph.co.uk,telegraph.co.uk,inbound,ZZ,Unknown Region,0 -telus.com,telus.com,inbound,ZZ,Unknown Region,0 -templeandwebster.com.au,templeandwebster.com.au,inbound,ZZ,Unknown Region,0 -texasjobdepartment.com,texasjobdepartment.com,inbound,ZZ,Unknown Region,0 -thebodyshop-usa.com,email-bodyshop.com,inbound,ZZ,Unknown Region,0 -thebodyshop-usa.com,postdirect.com,inbound,ZZ,Unknown Region,0 -thecarousell.com,thecarousell.com,inbound,ZZ,Unknown Region,1 -theguardian.com,theguardian.com,inbound,ZZ,Unknown Region,0 -thepamperedchef.com,thepamperedchef.com,inbound,ZZ,Unknown Region,0 -thephonehouse.es,splio.com,inbound,ZZ,Unknown Region,0.983578 -thephonehouse.es,splio.es,inbound,ZZ,Unknown Region,0.983266 -therealreal.com,email-realreal.com,inbound,ZZ,Unknown Region,0 -thesource.ca,thesource.ca,inbound,ZZ,Unknown Region,0 -thewarehouse.co.nz,thewarehouse.co.nz,inbound,ZZ,Unknown Region,0 -thinkgeek.com,thinkgeek.com,inbound,ZZ,Unknown Region,1e-06 -thirtyonegifts.com,thirtyonegifts.com,inbound,ZZ,Unknown Region,0 -thomascook.com,eccluster.com,inbound,ZZ,Unknown Region,0 -ticketmaster.com,ticketmaster.com,inbound,ZZ,Unknown Region,0.001001 -ticketmasterbiletix.com,ticketmasterbiletix.com,inbound,ZZ,Unknown Region,0 -timehop.com,timehop.com,inbound,ZZ,Unknown Region,1 -timeout.com,ec-cluster.com,inbound,ZZ,Unknown Region,0 -timewarnercable.com,bigfootinteractive.com,inbound,ZZ,Unknown Region,0 -tinyletterapp.com,tinyletterapp.com,inbound,ZZ,Unknown Region,0 -tobi.com,messagebus.com,inbound,ZZ,Unknown Region,0 -topface.com,topface.com,inbound,ZZ,Unknown Region,1e-06 -topica.com,topica-silver-y.com,inbound,ZZ,Unknown Region,0 -topqpon.si,topqpon.si,inbound,ZZ,Unknown Region,0 -touchbase2.com,mailurja.com,inbound,ZZ,Unknown Region,0 -townnews-mail.com,townnews-mail.com,inbound,ZZ,Unknown Region,0 -trabajar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999997 -trabalhar.com,trabajo.org,inbound,ZZ,Unknown Region,0.999994 -tradeloop.com,tradeloop.com,inbound,ZZ,Unknown Region,1 -trafficwave.net,trafficwave.net,inbound,ZZ,Unknown Region,0 -transittraveljobinsider.com,transittraveljobinsider.com,inbound,ZZ,Unknown Region,0 -transportexchangegroup.com,transportexchangegroup.com,inbound,ZZ,Unknown Region,0.998825 -travelchannel.com,travelchannel.com,inbound,ZZ,Unknown Region,0 -travelocity.com,travelocity.com,inbound,ZZ,Unknown Region,0.000195 -travelzoo.com,travelzoo.com,inbound,ZZ,Unknown Region,0 -trclient.com,trclient.com,inbound,ZZ,Unknown Region,0 -trello.com,mandrillapp.com,inbound,ZZ,Unknown Region,1 -triongames.com,triongames.com,inbound,ZZ,Unknown Region,0.085718 -tripadvisor.com,tripadvisor.com,inbound,ZZ,Unknown Region,0.000588 -tripolis.com,tripolis.com,inbound,ZZ,Unknown Region,0 -trulia.com,trulia.com,inbound,ZZ,Unknown Region,0 -tsmmail.com,tsmmail.com,inbound,ZZ,Unknown Region,0 -tumblr.com,tumblr.com,inbound,ZZ,Unknown Region,1 -turbine.com,turbine.com,inbound,ZZ,Unknown Region,0 -turner.com,cnn.com,inbound,ZZ,Unknown Region,0 -twe-safelist.com,adminforfree.com,inbound,ZZ,Unknown Region,1 -twitter.com,twitter.com,inbound,ZZ,Unknown Region,0.999969 -twoomail.com,netlogmail.com,inbound,ZZ,Unknown Region,0 -ubivox.com,ubivox.com,inbound,ZZ,Unknown Region,0.964085 -uga.edu,outlook.com,inbound,ZZ,Unknown Region,1 -uhcmedicaresolutions.com,uhcmedicaresolutions.com,inbound,ZZ,Unknown Region,0 -ulta.com,exacttarget.com,inbound,ZZ,Unknown Region,0 -ulta.com,ulta.com,inbound,ZZ,Unknown Region,0 -uniqlo-usa.com,uniqlo-usa.com,inbound,ZZ,Unknown Region,0 -unitedrepublic.org,unitedrepublic.org,inbound,ZZ,Unknown Region,1 -unosinsidersclub.com,unosinsidersclub.com,inbound,ZZ,Unknown Region,0 -urx.com.br,urx.com.br,inbound,ZZ,Unknown Region,0 -usaa.com,usaa.com,inbound,ZZ,Unknown Region,0.999997 -usahockey-email.com,usahockey-email.com,inbound,ZZ,Unknown Region,0 -usndr.com,usndr.com,inbound,ZZ,Unknown Region,1 -usx.com.br,uqx.com.br,inbound,ZZ,Unknown Region,0 -usx.com.br,usx.com.br,inbound,ZZ,Unknown Region,0 -usx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 -utilitiesjobinsider.com,utilitiesjobinsider.com,inbound,ZZ,Unknown Region,0 -utx.com.br,utx.com.br,inbound,ZZ,Unknown Region,0 -uvarosa.com.br,uvarosa.com.br,inbound,ZZ,Unknown Region,0 -vakifbank.com.tr,vakifbank.com.tr,inbound,ZZ,Unknown Region,1 -vd.nl,emsecure.net,inbound,ZZ,Unknown Region,0 -venca.es,eccluster.com,inbound,ZZ,Unknown Region,0 -verizon.com,verizon.com,inbound,ZZ,Unknown Region,0.999816 -vfoutletvip.com,vfoutletvip.com,inbound,ZZ,Unknown Region,0 -vicinity.nl,picsrv.net,inbound,ZZ,Unknown Region,0.022107 -vietcombank.com.vn,vietcombank.com.vn,inbound,ZZ,Unknown Region,1 -viralsender.com,viralsender.com,inbound,ZZ,Unknown Region,0 -vistaprint.com,vistaprint.com,inbound,ZZ,Unknown Region,0 -vistaprint.com.au,vistaprint.com.au,inbound,ZZ,Unknown Region,0 -vitaminworld.com,email-nbtyinc.com,inbound,ZZ,Unknown Region,0 -vk.com,vkontakte.ru,inbound,ZZ,Unknown Region,0 -vocus.com,vocus.com,inbound,ZZ,Unknown Region,0 -vovici.com,vovici.com,inbound,ZZ,Unknown Region,0 -vresp.com,verticalresponse.com,inbound,ZZ,Unknown Region,0 -vudu.com,vudu.com,inbound,ZZ,Unknown Region,0 -walmart.ca,walmart.ca,inbound,ZZ,Unknown Region,0 -walmart.com,walmart.com,inbound,ZZ,Unknown Region,0.315059 -warehouselogisticsjobinsider.com,warehouselogisticsjobinsider.com,inbound,ZZ,Unknown Region,0 -way2sms.biz,way2sms.biz,inbound,ZZ,Unknown Region,0 -way2sms.in,way2sms.in,inbound,ZZ,Unknown Region,0 -way2smsemail.com,way2smsemail.com,inbound,ZZ,Unknown Region,0 -way2smsemails.com,way2smsemails.com,inbound,ZZ,Unknown Region,0 -way2smsmail.in,way2smsmail.in,inbound,ZZ,Unknown Region,0 -way2smsmails.com,way2smsmails.com,inbound,ZZ,Unknown Region,0 -wealthyaffiliate.com,wealthyaffiliate.com,inbound,ZZ,Unknown Region,1 -webmd.com,webmd.com,inbound,ZZ,Unknown Region,0 -wegottickets.com,wegottickets.com,inbound,ZZ,Unknown Region,0 -weheartit.com,weheartit.com,inbound,ZZ,Unknown Region,1 -wehkamp.nl,wehkamp.nl,inbound,ZZ,Unknown Region,0 -wellsfargo.com,wellsfargo.com,inbound,ZZ,Unknown Region,1 -wemakeprice.com,wemakeprice.com,inbound,ZZ,Unknown Region,0 -westwing.com.br,cust-cluster.com,inbound,ZZ,Unknown Region,0 -westwing.es,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -westwing.ru,ecm-cluster.com,inbound,ZZ,Unknown Region,0 -wgbh.org,wgbh.org,inbound,ZZ,Unknown Region,0 -whaakky.com,whaakky.com,inbound,ZZ,Unknown Region,0 -whereareyounow.com,wayn.net,inbound,ZZ,Unknown Region,0 -whitehouse.gov,whitehouse.gov,inbound,ZZ,Unknown Region,0 -whitelabelpros.com,whitelabelpros.com,inbound,ZZ,Unknown Region,0 -wikia.com,wikia.com,inbound,ZZ,Unknown Region,0.289222 -wisdomitservices.com,infimail.com,inbound,ZZ,Unknown Region,0 -wolfmedia.us,wolfmedia.us,inbound,ZZ,Unknown Region,0 -wordfly.com,wordfly.com,inbound,ZZ,Unknown Region,0 -workhunter.net,workhunter.net,inbound,ZZ,Unknown Region,1 -worldwinner.com,worldwinner.com,inbound,ZZ,Unknown Region,0 -wowcher.co.uk,wowcher.co.uk,inbound,ZZ,Unknown Region,0 -wp.com,wordpress.com,inbound,ZZ,Unknown Region,0 -wpengine.com,wpengine.com,inbound,ZZ,Unknown Region,1 -writers-community.com,writers-community.com,inbound,ZZ,Unknown Region,0 -writersstore.com,writersstore.com,inbound,ZZ,Unknown Region,0 -wsjemail.com,wsjemail.com,inbound,ZZ,Unknown Region,0 -wyndhamhotelgroup.com,wyndhamhotelgroup.com,inbound,ZZ,Unknown Region,0 -xbox.com,xbox.com,inbound,ZZ,Unknown Region,0 -xcelenergy-emailnews.com,xcelenergy-emailnews.com,inbound,ZZ,Unknown Region,0 -xing.com,xing.com,inbound,ZZ,Unknown Region,0 -xxxconnect.com,infinitypersonals.com,inbound,ZZ,Unknown Region,0 -yahoo-inc.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -yahoo.{...},yahoo.{...},inbound,ZZ,Unknown Region,0.999989 -yahoo.{...},yahoodns.net,outbound,ZZ,Unknown Region,1 -yahoogroups.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -yammer.com,yammer.com,inbound,ZZ,Unknown Region,1 -yapikredi.com.tr,yapikredi.com.tr,inbound,ZZ,Unknown Region,1 -yapstone.com,yapstone.com,inbound,ZZ,Unknown Region,0 -yesbank.in,yesbank.in,inbound,ZZ,Unknown Region,0 -yipit.com,yipit.com,inbound,ZZ,Unknown Region,1 -ymail.com,yahoo.{...},inbound,ZZ,Unknown Region,1 -ymail.com,yahoodns.net,outbound,ZZ,Unknown Region,1 -youravon.com,email-avonglobal.com,inbound,ZZ,Unknown Region,0 -yournewsletters.net,everydayhealth.com,inbound,ZZ,Unknown Region,0 -youversion.com,youversion.com,inbound,ZZ,Unknown Region,1 -zappos.com,zappos.com,inbound,ZZ,Unknown Region,0.625377 -zattoo.com,sendnode.com,inbound,ZZ,Unknown Region,0 -zelonews.com.br,zelonews.com.br,inbound,ZZ,Unknown Region,0 -zendesk.com,zdsys.com,inbound,ZZ,Unknown Region,1 -zibmail.info,zibmail.info,inbound,ZZ,Unknown Region,0 -zillow.com,zillow.com,inbound,ZZ,Unknown Region,2.82543102656668e-07 -zinio.net,zinio.com,inbound,ZZ,Unknown Region,1 -zipalerts.com,sendgrid.net,inbound,ZZ,Unknown Region,1 -zipalerts.com,zipalerts.com,inbound,ZZ,Unknown Region,1 -zlavadna.sk,zlavadna.sk,inbound,ZZ,Unknown Region,0 -zoom.com.br,zoom.com.br,inbound,ZZ,Unknown Region,0 -zoominternet.net,synacor.com,inbound,ZZ,Unknown Region,0 -zoosk.com,zoosk.com,inbound,ZZ,Unknown Region,0 -zorpia.com,zorpia.com,inbound,ZZ,Unknown Region,0.520147 -zovifashion.com,eccluster.com,inbound,ZZ,Unknown Region,0 -zyngamail.com,zyngamail.com,inbound,ZZ,Unknown Region,0 \ No newline at end of file diff --git a/tools/CheckSTARTTLS.py b/tools/CheckSTARTTLS.py deleted file mode 100755 index ef0bf2e5c..000000000 --- a/tools/CheckSTARTTLS.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import errno -import smtplib -import socket -import subprocess -import re -import json -import collections - -import dns.resolver -from M2Crypto import X509 -from publicsuffix import PublicSuffixList - -public_suffix_list = PublicSuffixList() -CERTS_OBSERVED = 'certs-observed' - -def mkdirp(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: raise - -def extract_names(pem): - """Return a set of DNS subject names from PEM-encoded leaf cert.""" - leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) - - subj = leaf.get_subject() - # Certs have a "subject" identified by a Distingushed Name (DN). - # Host certs should also have a Common Name (CN) with a DNS name. - common_names = subj.get_entries_by_nid(subj.nid['CN']) - common_names = [name.get_data().as_text() for name in common_names] - try: - # The SAN extension allows one cert to cover multiple domains - # and permits DNS wildcards. - # http://www.digicert.com/subject-alternative-name.htm - # The field is a comma delimited list, e.g.: - # >>> twitter_cert.get_ext('subjectAltName').get_value() - # 'DNS:www.twitter.com, DNS:twitter.com' - alt_names = leaf.get_ext('subjectAltName').get_value() - alt_names = alt_names.split(', ') - alt_names = [name.partition(':') for name in alt_names] - alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] - except: - alt_names = [] - return set(common_names + alt_names) - -def tls_connect(mx_host, mail_domain): - """Attempt a STARTTLS connection with openssl and save the output.""" - if supports_starttls(mx_host): - # smtplib doesn't let us access certificate information, - # so shell out to openssl. - try: - output = subprocess.check_output( - """openssl s_client \ - -starttls smtp -connect %s:25 -showcerts /dev/null - """ % mx_host, shell = True) - except subprocess.CalledProcessError: - print "Failed s_client" - return - - # Save a copy of the certificate for later analysis - with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f: - f.write(output) - -def valid_cert(filename): - """Return true if the certificate is valid. - - Note: CApath must have hashed symlinks to the trust roots. - TODO: Include the -attime flag based on file modification time.""" - - if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: - return False - try: - # The file contains both the leaf cert and any intermediates, so we pass it - # as both the cert to validate and as the "untrusted" chain. - output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ - -untrusted "%s" \ - "%s" - """ % (filename, filename), shell = True) - return True - except subprocess.CalledProcessError: - return False - -def check_certs(mail_domain): - """ - Return "" if any certs for any mx domains pointed to by mail_domain - were invalid, and a public suffix for one if they were all valid - """ - dir = os.path.join(CERTS_OBSERVED, mail_domain) - if not os.path.exists(dir): - collect(mail_domain) - names = set() - for mx_hostname in os.listdir(dir): - filename = os.path.join(dir, mx_hostname) - if not valid_cert(filename): - return "" - else: - new_names = extract_names_from_openssl_output(filename) - new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names) - names.update(new_names) - if len(names) >= 1: - # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. - return names.pop() - else: - return "" - -def common_suffix(hosts): - num_components = min(len(h.split(".")) for h in hosts) - longest_suffix = "" - for i in range(1, num_components + 1): - suffixes = set(".".join(h.split(".")[-i:]) for h in hosts) - if len(suffixes) == 1: - longest_suffix = suffixes.pop() - else: - return longest_suffix - return longest_suffix - -def extract_names_from_openssl_output(certificates_file): - openssl_output = open(certificates_file, "r").read() - cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", openssl_output, flags = re.DOTALL) - return extract_names(cert[0]) - -def supports_starttls(mx_host): - try: - smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) - smtpserver.ehlo() - smtpserver.starttls() - return True - print "Success: %s" % mx_host - except socket.error as e: - print "Connection to %s failed: %s" % (mx_host, e.strerror) - return False - except smtplib.SMTPException, e: - # In order to talk to some hosts, you need to run this from a host that has a - # reverse DNS entry. AWS instances all have reverse DNS, as an example. - if e[0] == 554: - print e[1] - else: - print "No STARTTLS support on %s" % mx_host, e[0] - return False - -def min_tls_version(mail_domain): - protocols = [] - for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)): - filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname) - contents = open(filename).read() - protocol = re.findall("Protocol : (.*)", contents)[0] - protocols.append(protocol) - return min(protocols) - -def collect(mail_domain): - """ - Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS. - Store the output in a directory with the same name as mail_domain to make - subsequent analysis faster. - """ - print "Checking domain %s" % mail_domain - mkdirp(os.path.join(CERTS_OBSERVED, mail_domain)) - answers = dns.resolver.query(mail_domain, 'MX') - for rdata in answers: - mx_host = str(rdata.exchange).rstrip(".") - tls_connect(mx_host, mail_domain) - -if __name__ == '__main__': - """Consume a target list of domains and output a configuration file for those domains.""" - if len(sys.argv) < 2: - print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") - - config = collections.defaultdict(dict) - - for input in sys.argv[1:]: - for domain in open(input).readlines(): - domain = domain.strip() - suffix = check_certs(domain) - if suffix != "": - min_version = min_tls_version(domain) - suffix_match = "." + suffix - config["acceptable-mxs"][domain] = { - "accept-mx-domains": [suffix_match] - } - config["tls-policies"][suffix_match] = { - "require-tls": True, - "min-tls-version": min_version - } - - print json.dumps(config, indent=2, sort_keys=True) diff --git a/tools/ProcessGoogleSTARTTLSDomains.py b/tools/ProcessGoogleSTARTTLSDomains.py deleted file mode 100755 index 815ec5d4e..000000000 --- a/tools/ProcessGoogleSTARTTLSDomains.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Process Google's TLS delivery data from -https://www.google.com/transparencyreport/saferemail/data/?hl=en -to look for outbound domains that can negotiate an encrypted -connection >99% of the time. - -Usage: - ./ProcessGoogleSTARTTLSDomains.py google-starttls-domains.csv -""" -import csv -import codecs -import sys -from collections import defaultdict - -csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') -d = defaultdict(set) -# Google's report doesn't include gmail.com because it's local delivery, but we -# know they support STARTTLS, so manually include them. -d["gmail.com"] = set([1]) -for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader: - if direction == "outbound": - # Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}. - # We're tryingto get a solid list of the relevant TLDs, but in the meantime - # just use .com. - address_suffix = address_suffix.replace("{...}", "com") - try: - d[address_suffix].add(float(fraction_encrypted)) - except ValueError: - pass - -for address_suffix, fraction_encrypted in d.iteritems(): - if min(fraction_encrypted) >= 0.99: - print address_suffix diff --git a/vagrant-bootstrap.sh b/vagrant-bootstrap.sh deleted file mode 100755 index 82622edea..000000000 --- a/vagrant-bootstrap.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -export DEBIAN_FRONTEND=noninteractive - -apt-get update -q -apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - postfix dnsmasq mutt vim - -# Provide hostnames so the boxes can talk to each other. DNSMasq will also serve -# results to each box based on these contents. -cat >> /etc/hosts < /etc/dnsmasq.conf -# Not sure why restart is necessary, but otherwise dnsmasq doesn't use -# /etc/hosts to answer queries. -/etc/init.d/dnsmasq restart - -if [ "`hostname`" = "sender" ]; then - (while sleep 10; do - echo -e 'Subject: hi\n\nhi' | sendmail vagrant@valid-example-recipient.com - done) & - #ln -sf "/vagrant/postfix-config-sender-tls_policy.cf" /etc/postfix/tls_policy -fi - -#ln -sf "/vagrant/postfix-config-`hostname`.cf" /etc/postfix/main.cf -#ln -sf "/vagrant/certificates" /etc/certificates -postfix reload diff --git a/vagrant-shared/certificates/ca.crt b/vagrant-shared/certificates/ca.crt deleted file mode 100644 index dce966d32..000000000 --- a/vagrant-shared/certificates/ca.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDmzCCAoOgAwIBAgIJAIheS+k2UDobMA0GCSqGSIb3DQEBBQUAMGQxCzAJBgNV -BAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxDDAKBgNVBAoMA0VGRjEt -MCsGA1UECwwkU1RBUlRUTFMgRXZlcnl3aGVyZSB0ZXN0IGNlcnRpZmljYXRlMB4X -DTE0MDYxMDE4MDg0OVoXDTE5MDYxMDE4MDg1MFowZDELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRT -VEFSVFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDDYmUQUjaX6L3n9fNks2yQUFhKUBR1NYenx3w2 -DnaZiwpLzI3igJPMQfBdyJGLM1jXZZcpvpgt6yN4OMOLNS2QKBY20gDoQIh0Jmaj -KCoXUbX30H1FfTn+pyU02UpuFsFN3TAk5bQ/BQUYOlMouCowyZ25mnEzzHLeRHKH -Gi2uCH59T53rcgDwjq88pKMVUlndixkOKpeXZkTL++Edg0b5SUpRuzMs6kFmwLoQ -x4xG5lgaAHyu2/9KXhcqielE95s5FNGfi9U2q3nkmpa4dM266DWfkibfvCBP3hiO -Ks+IN+Hyohi2SY7NoDN6hMKcuUNlAK/xD+foA4Ck3R45HII3AgMBAAGjUDBOMB0G -A1UdDgQWBBRPlrBeFMPb27oWESmXykOW1XWLLDAfBgNVHSMEGDAWgBRPlrBeFMPb -27oWESmXykOW1XWLLDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBY -lBQEuRKo927jJFzgvdTKJGIAzxBnC5vg+qKeZkduwxeVEwB13HPP1syvdpAK5dkl -JGuHCF/Yc39HX1OZv7huVRMnyrSKpMt25uqUHirH6Db17HRCS4ZA6rARJfxS6RWu -J31lfXvGRr0hI4yw3XpKGM/c8Gkzji6PsYn4TCPnXjJbRj1GjHFaAuIeyoO0zQjo -OuHnzxs4bIDaV32NHNetuDKSO1GNbenxRiiN1HvQ1vfhzpqerRRaPCHrW4eUcynQ -AAeuC+Ek925t9mH7Ni/kZ0eN7XUwvg3c2lm+LeV+ICP+NWv9r92kfXDZNMhlhNsf -Y8+m1Y9WL3CLj99voNQS ------END CERTIFICATE----- diff --git a/vagrant-shared/certificates/ca.key b/vagrant-shared/certificates/ca.key deleted file mode 100644 index 3ebf4159e..000000000 --- a/vagrant-shared/certificates/ca.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAw2JlEFI2l+i95/XzZLNskFBYSlAUdTWHp8d8Ng52mYsKS8yN -4oCTzEHwXciRizNY12WXKb6YLesjeDjDizUtkCgWNtIA6ECIdCZmoygqF1G199B9 -RX05/qclNNlKbhbBTd0wJOW0PwUFGDpTKLgqMMmduZpxM8xy3kRyhxotrgh+fU+d -63IA8I6vPKSjFVJZ3YsZDiqXl2ZEy/vhHYNG+UlKUbszLOpBZsC6EMeMRuZYGgB8 -rtv/Sl4XKonpRPebORTRn4vVNqt55JqWuHTNuug1n5Im37wgT94YjirPiDfh8qIY -tkmOzaAzeoTCnLlDZQCv8Q/n6AOApN0eORyCNwIDAQABAoIBAQCwNXASDSNBU1zZ -8v3kZtDVQjCuLJSWtIU4cnd6RQb/KN9LRxr7GJyyzREbc4SXduJ7uBphQov6daMS -jJcGWBpUdWK7ZB//Vhv6LJvKL7HuP/oNmhEwd2SzXkj25bTznkANmhsOW794Sm2y -0P8orRcX0u0Vc8z+Ozepby+e2qQx29FXznberRv2rXmMeeQqF+sc2MBu34SlcLnK -KVSe7SmDc+DhWJ3XiPoEpiOTbv4EynndXC85owdse/eN/EahFtrfzqm6jDWVga8A -xA/7RD/Urc2L2IsZOB92xOk/tGs5Df8ZrHavzbSo+i0pzYAKvIxr+G2Krl1Tl/Lh -IXwjWPLZAoGBAOsbdxeZUHH6pk497ICIcmW/NDhAY/mSA6vBWZWLcYF7OU/wMzZ6 -Wlx9v6oz8tBTzQj/Xkpv0ZLIeQqGTuSQzQ7EcrOiTz4PIpM5y/nf/S0oSXwJ7mJw -Sx8w3XHqHEPCT9G83o7EV7xHuRy8/zQzJ9dkFxznA9sRcO7RhE/tsrMNAoGBANS/ -Ql140oyDqyABIlxswyvfJ8Ll4kmb52yNGaCJPSfrAVmSoN15AkbDkpVKFb6hsL6u -xqVPyeVxard4twddrV3GSvTibkGw4fZ8x0FDgDGvCgJ36e4NHb7jQVwwGfNXKMAP -qvYtnE28eAM+zrDHryEhgp4k59zApphr+IU7JQlTAoGAc1h5ODm+rvzTBMX6tyC6 -R1Lkcsicg//wDx8ALY9JM8ZZ2u80oQCsPn5vPzjXYwAKMuTexNRRVJtITzKPmDG2 -eQ1GXP0/tWnFg8eyXDhZRQNj8hgJPYBsSrQ1oMLD9TZq5LKt2gtYJAZoOkI7Tsfe -Px1a/ZIVYTAQYQqnyHMM3i0CgYAQNirWeJiCwJ3PqIZ3yInu0+hxv5bIySqPaQkk -5JBWdF/79WJwvgHgZpLK8YRKrIONZEAa5MObylK5fGdmFktZs/yOQJrqQpJVeBiu -7nfcUVxP59dZnoI/w419euTfWCrwx8DdVYhtnAkBJk4VxoGf4q/TYTiR59RKFSAw -9trRpQKBgDjFQ9mXJUUKoc61P3PIrSYOlnx89c0sznrwm2eeLLAwr/VQ+vNDGRkC -6kXbVLuTxRaFnLSZTlUd0nSXPbJEM5GII48zbGZ7t/ysPtA9390xk1efTO/ryf/j -pn4FBvpgXaneRs4ExyknLgFfJRUEu17fRbyRPZr0bUlTkKg/ZS5M ------END RSA PRIVATE KEY----- diff --git a/vagrant-shared/certificates/certificates b/vagrant-shared/certificates/certificates deleted file mode 120000 index d162b42af..000000000 --- a/vagrant-shared/certificates/certificates +++ /dev/null @@ -1 +0,0 @@ -/vagrant/certificates \ No newline at end of file diff --git a/vagrant-shared/certificates/valid.crt b/vagrant-shared/certificates/valid.crt deleted file mode 100644 index f5fe4c213..000000000 --- a/vagrant-shared/certificates/valid.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDbTCCAlUCAQEwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCVVMxCzAJBgNV -BAgMAkNBMQswCQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMS0wKwYDVQQLDCRTVEFS -VFRMUyBFdmVyeXdoZXJlIHRlc3QgY2VydGlmaWNhdGUwHhcNMTQwNjEwMTgxMTQ3 -WhcNMTkwNjEwMTgxMTQ3WjCBlDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQsw -CQYDVQQHDAJTRjEMMAoGA1UECgwDRUZGMTQwMgYDVQQLDCtTVEFSVFRMUyBFdmVy -eXdoZXJlIHRlc3QgY2VydGlmaWNhdGUgKGxlYWYpMScwJQYDVQQDDB5teC52YWxp -ZC1leGFtcGxlLXJlY2lwaWVudC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQDZ2NdAwI0LUmXrV7bItKhP8NRzypkYHgApXaoZ6iB6TJnYb1FZyv2Y -wyBdsQQ8zcIgKRyl0rKvNPgHt3riM7nSoB16II6fIQDuR2UnXkhtDOfUd7Ye0wKX -l3A+qoEge42/TfRvzPC6IMa1KH7+dwayprIjLxFUzfMl6GqA/5auLZZtSsV6Pix3 -jXZYFPUHPoBJEyNo/bizcdvSZS/7Kwzfc64l7JLA/OGtRbQpcMAOpRRXCx32nt3N -2L1OXz8Q4It+J20oN8Vfin1a7GbAGdktRbz2bV8bmb0ux1NOnddigPUHRRYMIaKN -W7Nrxp2YQpqmsqBmVEVPA903yRc0ZvuRAgMBAAEwDQYJKoZIhvcNAQEFBQADggEB -AMJ3neFM+to/tFhTiAfUnIrQOKyfk+zYN8gC99HD2SUVbu+Cu1qCNJWxbE3bdxxX -y960yX+Or8E4nG9sWbFbzlGEzz0F8DAPEh4UCtb/5MRkhW158Y9LtbheQAQpZolQ -Sma6lnngCmSDr/OpDW34oM8S7+3VOawYv1e1ruCMqizBwilgcsGq2hY1hWca6LMP -X2FHQ+m4mYBV9wJ9WuKMs9uADz9cfMYuA/wMzNeMZyM82w+nSQGZ+mX7YPu+WBJM -k9OcFrUWxrC/uOQfsyqdjLvFjg4/b49jnusn9mZ/KbtLVmkz5P+5kwSn4kcd5Rlw -LLmEGiXiyEjo8UmEVyumrIQ= ------END CERTIFICATE----- diff --git a/vagrant-shared/certificates/valid.csr b/vagrant-shared/certificates/valid.csr deleted file mode 100644 index a58aab677..000000000 --- a/vagrant-shared/certificates/valid.csr +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIC2jCCAcICAQAwgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UE -BwwCU0YxDDAKBgNVBAoMA0VGRjE0MDIGA1UECwwrU1RBUlRUTFMgRXZlcnl3aGVy -ZSB0ZXN0IGNlcnRpZmljYXRlIChsZWFmKTEnMCUGA1UEAwwebXgudmFsaWQtZXhh -bXBsZS1yZWNpcGllbnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9RWcr9mMMgXbEE -PM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2HtMCl5dwPqqB -IHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrFej4sd412WBT1 -Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd9p7dzdi9Tl8/ -EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UWDCGijVuza8ad -mEKaprKgZlRFTwPdN8kXNGb7kQIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAAhe -pMLNDAaA9fXZ/yqW4ov1pBKYZL1R1RX+YgZqUPoAlrTCiJ0UGIuPcTDGJDMdpZ7x -9gSqwsXrvzXafuHI4GuFjnbY5SIv8zvc+nMXib7IMlyMUcuSBZP8W0sl3ZGWnnpk -legC10c3+I9TfQ7Tl0mpdyf/6yLhM1plxLcIy5bguLJjbBK9JhKNfc84rivqrxUI -+cUqWU13WjMzWdKS6rK5m/Bfleg+jyZ11xYY0QfwNGwuPjfiEBjCs2iqJvdLFRem -FoDq3XBsrH5XohSwWZ6UrZY7wARkmHsYJeYTHulb3MkDPCQKlbU+2SMIpbNk43+/ -fN/ctxWXg3vd/q6eJiE= ------END CERTIFICATE REQUEST----- diff --git a/vagrant-shared/certificates/valid.key b/vagrant-shared/certificates/valid.key deleted file mode 100644 index b5e8371cd..000000000 --- a/vagrant-shared/certificates/valid.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA2djXQMCNC1Jl61e2yLSoT/DUc8qZGB4AKV2qGeogekyZ2G9R -Wcr9mMMgXbEEPM3CICkcpdKyrzT4B7d64jO50qAdeiCOnyEA7kdlJ15IbQzn1He2 -HtMCl5dwPqqBIHuNv030b8zwuiDGtSh+/ncGsqayIy8RVM3zJehqgP+Wri2WbUrF -ej4sd412WBT1Bz6ASRMjaP24s3Hb0mUv+ysM33OuJeySwPzhrUW0KXDADqUUVwsd -9p7dzdi9Tl8/EOCLfidtKDfFX4p9WuxmwBnZLUW89m1fG5m9LsdTTp3XYoD1B0UW -DCGijVuza8admEKaprKgZlRFTwPdN8kXNGb7kQIDAQABAoIBAHAZgUq0yt+UmxWr -oUdOj33zc5/SFU2vwm2G4U1MiUHlwRT602Xdavn9Dt6nhIK1bruV7EP4VDKMk0WF -SRq1e13DPuflcP65wPzciFTl02cqSPGwWGssMh1HtF7K5n+MlLhoqOwPDaD51MbL -++192lh8Jxar1cNJ52EOZB/VZfhiUZ88JAITWam6clId8KS4cy7SvQ9haptcPlfl -wIeCti2eI4nvC0IR0SCEqWAax6XqypA5k2fjQJklcnBK1R+H1muc00J5lMIz4U8q -qFb5RgxIDMcEeQZWmzfBjU5hO0q/51QyWFFFuBL00MkT4djCHjO/IPYc6DQoAoZR -TbMCWwECgYEA+i3W44tSY96Fdn1gDyYhDor9GPN0XwmRBHpl44TxBLG48I6cGLYI -l0InEh7pu7/0fw+cWylx2OLrl2HeGlWilEOpP2ucJcswcNibATtgSdnu7PjC4eAW -46a5+2hbFk/NONU5VTGClJHCvKzYPWMrAIkRT//WdkIVs23i7bRcOjkCgYEA3upr -cnMhnwVmZrGAIm+DtiDJCc4uVXeqLD2j9csCVNCw/CQAjJ3/5VlQDSjzerjijw/V -uoA+Su8xMoMCg6mAihqdxBMrbwdlh7vje23RaxbeQkIY5JIVAsdDr+UtwofaX3Zx -j3RW5jNeouVlvtExyKZZhyMwuh8d59m4V65/rBkCgYBH+f40EvZOQ0v0jhef5CFo -lLZCgnB9kzwEpM5BihLpfdQuaWkhduW71s102i321UAbejtKwv69HnQXZpHG09Jl -g53i4CvZd77lCHx3+0Q1mxyxUtSGtbkAIAyr9xcVsTni2v2WtBrUcacsLzI7XxeV -HNo9QObLuTGTIM9EAjryiQKBgQCIZEBX56/jn6c3IFX5O+gH8OlxEXFyI+TAavq+ -Mnd7s7EGpXSclTP0fYAofSz0otkklZi9Iyh6Kv4cHOLV8klOtthfFyeVKJ5rvX+D -jv76miRlwBGBEQzABXIZ1oz4IK1xiYQUNSfSdA3sd5WYemEOlxHiSJrQ1qcyrBlJ -tOAzSQKBgQDjLQIZ0rmSAMUCGi7WlDle/pnf6sUEFUPWUqzHOJbJk/CZIIf1IT6o -Evep9eeP6QjiupusY/uSlgjjjj7yxZU5q3VV/0cQl6QhwxG9wqpa8x32wwaM1WBZ -rJgsuSjcEoluAqfX/bqRtumNtQ213DSADzSCpOetlJb+ARANk7YULg== ------END RSA PRIVATE KEY----- diff --git a/vagrant-shared/postfix-config-sender-tls_policy b/vagrant-shared/postfix-config-sender-tls_policy deleted file mode 100644 index f4f1e80c5..000000000 --- a/vagrant-shared/postfix-config-sender-tls_policy +++ /dev/null @@ -1 +0,0 @@ -valid-example-recipient.com secure match=valid-example-recipient.com:.valid-example-recipient.com diff --git a/vagrant-shared/postfix-config-sender.cf b/vagrant-shared/postfix-config-sender.cf deleted file mode 100644 index b9f265058..000000000 --- a/vagrant-shared/postfix-config-sender.cf +++ /dev/null @@ -1,39 +0,0 @@ -# See /usr/share/postfix/main.cf.dist for a commented, more complete version - - -# Debian specific: Specifying a file name will cause the first -# line of that file to be used as the name. The Debian default -# is /etc/mailname. -#myorigin = /etc/mailname - -smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) -biff = no - -# appending .domain is the MUA's job. -append_dot_mydomain = no - -# Uncomment the next line to generate "delayed mail" warnings -#delay_warning_time = 4h - -readme_directory = no - -# TLS parameters -smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key -smtpd_use_tls=yes -smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache - -# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for -# information on enabling SSL in the smtp client. - -myhostname = sender.example.com -alias_maps = hash:/etc/aliases -alias_database = hash:/etc/aliases -myorigin = /etc/mailname -mydestination = sender.example.com, localhost.example.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 -mailbox_size_limit = 0 -recipient_delimiter = + -inet_interfaces = all diff --git a/vagrant-shared/postfix-config-valid-example-recipient.cf b/vagrant-shared/postfix-config-valid-example-recipient.cf deleted file mode 100644 index 1d08a20cd..000000000 --- a/vagrant-shared/postfix-config-valid-example-recipient.cf +++ /dev/null @@ -1,46 +0,0 @@ -# See /usr/share/postfix/main.cf.dist for a commented, more complete version - - -# Debian specific: Specifying a file name will cause the first -# line of that file to be used as the name. The Debian default -# is /etc/mailname. -#myorigin = /etc/mailname - -smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) -biff = no - -# appending .domain is the MUA's job. -append_dot_mydomain = no - -# Uncomment the next line to generate "delayed mail" warnings -#delay_warning_time = 4h - -readme_directory = no - -# TLS parameters -smtpd_use_tls=yes -smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache - -# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for -# information on enabling SSL in the smtp client. - -myhostname = valid-example-recipient.com -alias_maps = hash:/etc/aliases -alias_database = hash:/etc/aliases -myorigin = /etc/mailname -mydestination = valid-example-recipient.com, localhost.example.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 -mailbox_size_limit = 0 -recipient_delimiter = + -inet_interfaces = all - -# STARTLS Everywhere recommended best-practice settings -smtpd_tls_session_cache_timeout = 3600s -smtpd_tls_received_header = yes - -#STARTTLS EVERYWHERE MAGIC STARTS HERE -smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy -smtpd_tls_cert_file=/etc/certificates/valid.crt -smtpd_tls_key_file=/etc/certificates/valid.key From 5ca1b49e2ee1a368b296e209baaa87f4d30a2b6a Mon Sep 17 00:00:00 2001 From: ynasser Date: Thu, 2 Nov 2017 06:23:38 +0000 Subject: [PATCH 204/639] revoke accepts --cert-name too now; from the livestram --- certbot/cli.py | 8 ++++---- certbot/main.py | 9 ++++++++- certbot/tests/main_test.py | 26 +++++++++++++++++++++----- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 880ffd543..47d8df3f4 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -90,7 +90,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -371,9 +371,9 @@ VERB_HELP = [ "usage": "\n\n certbot delete --cert-name CERTNAME\n\n" }), ("revoke", { - "short": "Revoke a certificate specified with --cert-path", + "short": "Revoke a certificate specified with --cert-path or --cert-name", "opts": "Options for revocation of certificates", - "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem [options]\n\n" + "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem --cert-name example.com [options]\n\n" }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", @@ -1262,7 +1262,7 @@ def _paths_parser(helpful): add(section, "--cert-path", type=os.path.abspath, default=flag_default("auth_cert_path"), help=cph) elif verb == "revoke": - add(section, "--cert-path", type=read_file, required=True, help=cph) + add(section, "--cert-path", type=read_file, required=False, help=cph) else: add(section, "--cert-path", type=os.path.abspath, help=cph, required=(verb == "install")) diff --git a/certbot/main.py b/certbot/main.py index 9e2850891..c2a6a7bb2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -655,6 +655,14 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction config.installer = config.authenticator = "None" + + if config.cert_path is None and config.certname: + config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + elif not config.cert_path or (config.cert_path and config.certname): + # intentionally not supporting --cert-path & --cert-name together, + # to avoid dealing with mismatched values + raise errors.Error("Error! Exactly one of --cert-path or --cert-name must be specified!") + if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) @@ -667,7 +675,6 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config acme = client.acme_from_config_key(config, key) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) - try: acme.revoke(jose.ComparableX509(cert), config.reason) _delete_if_appropriate(config) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..5995f77ad 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -224,6 +224,8 @@ class RevokeTest(test_util.TempDirTestCase): shutil.copy(CERT_PATH, self.tempdir) self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) + with open(self.tmp_cert_path, 'r') as f: + self.tmp_cert = (self.tmp_cert_path, f.read()) self.patches = [ mock.patch('acme.client.Client', autospec=True), @@ -253,9 +255,10 @@ class RevokeTest(test_util.TempDirTestCase): for patch in self.patches: patch.stop() - def _call(self, extra_args=""): - args = 'revoke --cert-path={0} ' + extra_args - args = args.format(self.tmp_cert_path).split() + def _call(self, args=[]): + if not args: + args = 'revoke --cert-path={0} ' + args = args.format(self.tmp_cert_path).split() plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -271,12 +274,25 @@ class RevokeTest(test_util.TempDirTestCase): mock_revoke = mock_acme_client.Client().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): - self._call("--reason " + reason) + args = 'revoke --cert-path={0} --reason {1}'.format(self.tmp_cert_path, reason).split() + self._call(args) expected.append(mock.call(mock.ANY, code)) - self._call("--reason " + reason.upper()) + args = 'revoke --cert-path={0} --reason {1}'.format(self.tmp_cert_path, + reason.upper()).split() + self._call(args) expected.append(mock.call(mock.ANY, code)) self.assertEqual(expected, mock_revoke.call_args_list) + @mock.patch('certbot.main._delete_if_appropriate') + @mock.patch('certbot.storage.cert_path_for_cert_name') + def test_revoke_by_certname(self, mock_cert_path_for_cert_name, + mock_delete_if_appropriate): + args = 'revoke --cert-name=example.com'.split() + mock_cert_path_for_cert_name.return_value = self.tmp_cert + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + @mock.patch('certbot.main._delete_if_appropriate') def test_revocation_success(self, mock_delete_if_appropriate): self._call() From c443db0618310c1f743b5d9bb0399d407212d295 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Thu, 12 Apr 2018 16:33:10 -0700 Subject: [PATCH 205/639] Update constants.py On FreeBSD or MacOS, "certbot --nginx" fails. The reason is, at these op. systems, nginx directory is different than linux. --- certbot-nginx/certbot_nginx/constants.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 3f263fea3..8422ab3cd 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -1,9 +1,14 @@ """nginx plugin constants.""" import pkg_resources +import platform - +if(platform.system() == ('FreeBSD' or 'Darwin')): + server_root_tmp = "/usr/local/etc/nginx" +else: + server_root_tmp = "/etc/nginx" + CLI_DEFAULTS = dict( - server_root="/etc/nginx", + server_root=server_root_tmp ctl="nginx", ) """CLI defaults.""" From b39507c5af53bd8511dec554a653cd7a4d3e8267 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Tue, 17 Apr 2018 09:09:27 -0700 Subject: [PATCH 206/639] Update constants.py Fixed comma. --- certbot-nginx/certbot_nginx/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 8422ab3cd..72fc5d4de 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -8,7 +8,7 @@ else: server_root_tmp = "/etc/nginx" CLI_DEFAULTS = dict( - server_root=server_root_tmp + server_root=server_root_tmp, ctl="nginx", ) """CLI defaults.""" From 20418cdd6899b0ac5c0e7669bdc69ed135594276 Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Thu, 17 May 2018 09:52:11 -0400 Subject: [PATCH 207/639] Fixed #5859 (#6011) --- tests/boulder-fetch.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index fc9cbaae7..d513ec064 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,12 +17,6 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -# If we're testing against ACMEv2, we need to use a newer boulder config for -# now. See https://github.com/letsencrypt/boulder#quickstart. -if [ "$BOULDER_INTEGRATION" = "v2" ]; then - sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml -fi - docker-compose up -d set +x # reduce verbosity while waiting for boulder From 1be1bd92115101e019d71a23ee14e7bfadf72226 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 May 2018 09:23:05 -0700 Subject: [PATCH 208/639] remove PYTHONPATH (#6016) --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 38d4e6ae1..8c4d6c38d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,6 @@ commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py setenv = - PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 [testenv:py27-oldest] From 9b2862ebb0f1618161fcf5726ac4de812b959ac9 Mon Sep 17 00:00:00 2001 From: TyrannosourceExe <38411281+TyrannosourceExe@users.noreply.github.com> Date: Thu, 17 May 2018 19:03:01 -0400 Subject: [PATCH 209/639] 3692 --dry-run expiration emails (#6015) * If --dry-run is used and there exists no staging account, create account with no email * added unit testing of dry-run to ensure certbot does not ask the user to create an email, and that certbot creates an account with no email --- certbot/client.py | 4 ++++ certbot/tests/client_test.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 45dc9c63b..cdfbdc252 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -155,6 +155,10 @@ def register(config, account_storage, tos_cb=None): if not config.dry_run: logger.info("Registering without email!") + # If --dry-run is used, and there is no staging account, create one with no email. + if config.dry_run: + config.email = None + # Each new registration shall use a fresh new key key = jose.JWKRSA(key=jose.ComparableRSAKey( rsa.generate_private_key( diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6add141d4..a4425bca9 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -12,7 +12,6 @@ from certbot import util import certbot.tests.util as test_util - KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") @@ -92,6 +91,20 @@ class RegisterTest(test_util.ConfigTestCase): mock_logger.info.assert_called_once_with(mock.ANY) self.assertTrue(mock_handle.called) + @mock.patch("certbot.account.report_new_account") + @mock.patch("certbot.client.display_ops.get_email") + def test_dry_run_no_staging_account(self, _rep, mock_get_email): + """Tests dry-run for no staging account, expect account created with no email""" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + with mock.patch("certbot.eff.handle_subscription"): + with mock.patch("certbot.account.report_new_account"): + self.config.dry_run = True + self._call() + # check Certbot did not ask the user to provide an email + self.assertFalse(mock_get_email.called) + # check Certbot created an account with no email. Contact should return empty + self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) + def test_unsupported_error(self): from acme import messages msg = "Test" @@ -105,6 +118,7 @@ class RegisterTest(test_util.ConfigTestCase): class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot.client.Client tests.""" + def setUp(self): super(ClientTestCommon, self).setUp() self.config.no_verify_ssl = False @@ -124,6 +138,7 @@ class ClientTestCommon(test_util.ConfigTestCase): class ClientTest(ClientTestCommon): """Tests for certbot.client.Client.""" + def setUp(self): super(ClientTest, self).setUp() @@ -286,10 +301,10 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, - mock_storage, mock_obtain_certificate): + mock_storage, mock_obtain_certificate): domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), - mock.MagicMock(), mock.MagicMock(), None) + mock.MagicMock(), mock.MagicMock(), None) self.client.config.dry_run = False self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) @@ -318,8 +333,8 @@ class ClientTest(ClientTestCommon): candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") mock_parser.verb = "certonly" mock_parser.args = ["--cert-path", candidate_cert_path, - "--chain-path", candidate_chain_path, - "--fullchain-path", candidate_fullchain_path] + "--chain-path", candidate_chain_path, + "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, @@ -407,6 +422,7 @@ class ClientTest(ClientTestCommon): class EnhanceConfigTest(ClientTestCommon): """Tests for certbot.client.Client.enhance_config.""" + def setUp(self): super(EnhanceConfigTest, self).setUp() From 1239d7a881f7ad88c736aac526a821658cea0f96 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 17 May 2018 20:02:27 -0700 Subject: [PATCH 210/639] check platform with correct python --- certbot-nginx/certbot_nginx/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 72fc5d4de..dfc451202 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -2,11 +2,11 @@ import pkg_resources import platform -if(platform.system() == ('FreeBSD' or 'Darwin')): +if platform.system() in ('FreeBSD', 'Darwin'): server_root_tmp = "/usr/local/etc/nginx" else: server_root_tmp = "/etc/nginx" - + CLI_DEFAULTS = dict( server_root=server_root_tmp, ctl="nginx", From 94bf97b812fac7b56541992ef6ab81a6abedec49 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 02:26:10 -0700 Subject: [PATCH 211/639] Add remaining DNS plugins to mypy.ini and sort (#6018) --- mypy.ini | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy.ini b/mypy.ini index 506c253a8..d00c21ae7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,15 @@ check_untyped_defs = True [mypy-certbot_apache.*] check_untyped_defs = True +[mypy-certbot_dns_cloudflare.*] +check_untyped_defs = True + +[mypy-certbot_dns_cloudxns.*] +check_untyped_defs = True + +[mypy-certbot_dns_digitalocean.*] +check_untyped_defs = True + [mypy-certbot_dns_dnsimple.*] check_untyped_defs = True @@ -29,8 +38,5 @@ check_untyped_defs = True [mypy-certbot_dns_route53.*] check_untyped_defs = True -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - [mypy-certbot_nginx.*] check_untyped_defs = True From 250c0d6691f8fce9568fa13df0dc256181403cfe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 06:05:26 -0700 Subject: [PATCH 212/639] cd before running tests (#6017) When importing a module, Python first searches the current directory. See https://docs.python.org/3/tutorial/modules.html#the-module-search-path. This means that running something like `import certbot` from the root of the Certbot repo will use the local Certbot files regardless of the version installed on the system or virtual environment. Normally this behavior is fine because the local files are what we want to test, however, during our "oldest" tests, we test against older versions of our packages to make sure we're keeping compatibility. To make sure our tests use the correct versions, this commit has our tests cd to an empty temporary directory before running tests. We also had to change the package names given to pytest to be the names used in Python to import the package rather than the name of the files locally to accommodate this. --- tools/install_and_test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 59832cbc3..819f683aa 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -12,12 +12,18 @@ else pip_install="$(dirname $0)/pip_install_editable.sh" fi +temp_cwd=$(mktemp -d) +trap "rm -rf $temp_cwd" EXIT + set -x for requirement in "$@" ; do $pip_install $requirement pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] + pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names if [ $pkg = "." ]; then pkg="certbot" fi + cd "$temp_cwd" pytest --numprocesses auto --quiet --pyargs $pkg + cd - done From 36dfd065037678bf38b7adb95ab16ed951ebbd18 Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Fri, 18 May 2018 09:28:17 -0400 Subject: [PATCH 213/639] Prepare certbot module for mypy check untyped defs (#6005) * Prepare certbot module for mypy check untyped defs * Fix #5952 * Bump mypy to version 0.600 and fix associated bugs * Fix pylint bugs after introducing mypy * Implement Brad's suggestions * Reenabling pylint and adding nginx mypy back --- acme/acme/challenges.py | 4 +- acme/acme/crypto_util.py | 22 ++-- acme/acme/magic_typing.py | 5 +- certbot/auth_handler.py | 24 +++-- certbot/cert_manager.py | 5 +- certbot/cli.py | 54 ++++++---- certbot/client.py | 16 +-- certbot/crypto_util.py | 128 ++++++++++++------------ certbot/error_handler.py | 16 +-- certbot/hooks.py | 17 ++-- certbot/log.py | 3 +- certbot/main.py | 8 +- certbot/plugins/common.py | 6 +- certbot/plugins/disco.py | 3 +- certbot/plugins/disco_test.py | 3 +- certbot/plugins/manual.py | 5 +- certbot/plugins/selection_test.py | 3 +- certbot/plugins/standalone.py | 29 ++++-- certbot/plugins/standalone_test.py | 19 ++-- certbot/plugins/storage.py | 3 +- certbot/plugins/webroot.py | 17 ++-- certbot/renewal.py | 17 ++-- certbot/reverter.py | 12 ++- certbot/tests/auth_handler_test.py | 4 +- certbot/tests/cli_test.py | 3 +- certbot/tests/display/completer_test.py | 3 +- certbot/tests/error_handler_test.py | 6 +- certbot/tests/hook_test.py | 21 ++-- certbot/tests/log_test.py | 13 +-- certbot/tests/main_test.py | 17 ++-- certbot/tests/reporter_test.py | 18 ++-- certbot/util.py | 14 ++- mypy.ini | 6 ++ setup.py | 2 +- tools/dev_constraints.txt | 2 +- tox.ini | 2 +- 36 files changed, 316 insertions(+), 214 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index b2a4882eb..674f2c38f 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -147,9 +147,9 @@ class KeyAuthorizationChallenge(_TokenChallenge): :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` that will be used to generate `response`. - + :param str typ: type of the challenge """ - + typ = NotImplemented response_cls = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index ad914ca60..d0e203417 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -12,7 +12,8 @@ import josepy as jose from acme import errors # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Callable, Text, Union +from acme.magic_typing import Callable, Union, Tuple, Optional +# pylint: enable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -135,14 +136,23 @@ def probe_sni(name, host, port=443, timeout=300, socket_kwargs = {'source_address': source_address} - host_protocol_agnostic = None if host == '::' or host == '0' else host + host_protocol_agnostic = host + if host == '::' or host == '0': + # https://github.com/python/typeshed/pull/2136 + # while PR is not merged, we need to ignore + host_protocol_agnostic = None try: # pylint: disable=star-args - logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, - " from {0}:{1}".format(source_address[0], source_address[1]) if \ - socket_kwargs else "") - sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs) + logger.debug( + "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, + " from {0}:{1}".format( + source_address[0], + source_address[1] + ) if socket_kwargs else "" + ) + socket_tuple = (host_protocol_agnostic, port) # type: Tuple[Optional[str], int] + sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 555088cf2..471b8dfa9 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -8,6 +8,9 @@ class TypingClass(object): try: # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index caf112c61..e7d658b25 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -8,7 +8,9 @@ import zope.component from acme import challenges from acme import messages - +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +# pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler @@ -117,7 +119,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] + resp = [] # type: Collection[acme.challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -133,10 +135,9 @@ class AuthHandler(object): def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" - all_achalls = [] + all_achalls = [] # type: Collection[challenges.ChallengeResponse] for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) - return all_achalls def _respond(self, aauthzrs, resp, best_effort): @@ -146,7 +147,8 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() + chall_update = dict() \ + # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... @@ -198,7 +200,7 @@ class AuthHandler(object): while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) - all_failed_achalls = set() + all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( aauthzrs, index, chall_update[index]) @@ -424,7 +426,7 @@ def _find_smart_path(challbs, preferences, combinations): # max_cost is now equal to sum(indices) + 1 - best_combo = [] + best_combo = None # Set above completing all of the available challenges best_combo_cost = max_cost @@ -479,7 +481,7 @@ def _report_no_chall_path(challbs): msg += ( " You may need to use an authenticator " "plugin that can do challenges over DNS.") - logger.fatal(msg) + logger.critical(msg) raise errors.AuthorizationError(msg) @@ -522,11 +524,11 @@ def _report_failed_challs(failed_achalls): :class:`certbot.achallenges.AnnotatedChallenge`. """ - problems = dict() + problems = collections.defaultdict(list)\ + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: - problems.setdefault(achall.error.typ, []).append(achall) - + problems[achall.error.typ].append(achall) reporter = zope.component.getUtility(interfaces.IReporter) for achalls in six.itervalues(problems): reporter.add_message( diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d841c1912..d1205835a 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -7,6 +7,7 @@ import re import traceback import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -226,7 +227,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] + acceptable_matches_rv = [] # type: List[str] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -340,7 +341,7 @@ def _report_human_readable(config, parsed_certs): def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] + out = [] # type: List[str] notify = out.append diff --git a/certbot/cli.py b/certbot/cli.py index b71d60055..8a1ad381a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -12,10 +12,14 @@ import sys import configargparse import six import zope.component +import zope.interface from zope.interface import interfaces as zope_interfaces from acme import challenges +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict, Optional +# pylint: enable=unused-import, no-name-in-module import certbot @@ -33,7 +37,7 @@ import certbot.plugins.selection as plugin_selection logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None +helpful_parser = None # type: Optional[HelpfulArgumentParser] # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: @@ -196,17 +200,17 @@ def set_by_cli(var): (CLI or config file) including if the user explicitly set it to the default. Returns False if the variable was assigned a default value. """ - detector = set_by_cli.detector - if detector is None: + detector = set_by_cli.detector # type: ignore + if detector is None and helpful_parser is not None: # Setup on first run: `detector` is a weird version of config in which # the default value of every attribute is wrangled to be boolean-false plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = helpful_parser.args + [helpful_parser.verb] - detector = set_by_cli.detector = prepare_and_parse_args( + detector = set_by_cli.detector = prepare_and_parse_args( # type: ignore plugins, reconstructed_args, detect_defaults=True) # propagate plugin requests: eg --standalone modifies config.authenticator - detector.authenticator, detector.installer = ( + detector.authenticator, detector.installer = ( # type: ignore plugin_selection.cli_plugin_requests(detector)) if not isinstance(getattr(detector, var), _Default): @@ -220,7 +224,10 @@ def set_by_cli(var): return True return False + # static housekeeping var +# functions attributed are not supported by mypy +# https://github.com/python/mypy/issues/2087 set_by_cli.detector = None # type: ignore @@ -236,8 +243,10 @@ def has_default_value(option, value): :rtype: bool """ - return (option in helpful_parser.defaults and - helpful_parser.defaults[option] == value) + if helpful_parser is not None: + return (option in helpful_parser.defaults and + helpful_parser.defaults[option] == value) + return False def option_was_set(option, value): @@ -254,11 +263,12 @@ def option_was_set(option, value): def argparse_type(variable): - "Return our argparse type function for a config variable (default: str)" + """Return our argparse type function for a config variable (default: str)""" # pylint: disable=protected-access - for action in helpful_parser.parser._actions: - if action.type is not None and action.dest == variable: - return action.type + if helpful_parser is not None: + for action in helpful_parser.parser._actions: + if action.type is not None and action.dest == variable: + return action.type return str def read_file(filename, mode="rb"): @@ -291,10 +301,12 @@ def flag_default(name): def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" + # pylint: disable=no-member if hidden: return argparse.SUPPRESS else: - return interfaces.IConfig[name].__doc__ + field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + return field.__doc__ class HelpfulArgumentGroup(object): @@ -473,7 +485,7 @@ class HelpfulArgumentParser(object): HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"] plugin_names = list(plugins) - self.help_topics = HELP_TOPICS + plugin_names + [None] + self.help_topics = HELP_TOPICS + plugin_names + [None] # type: ignore self.detect_defaults = detect_defaults self.args = args @@ -492,8 +504,11 @@ class HelpfulArgumentParser(object): short_usage = self._usage_string(plugins, self.help_arg) self.visible_topics = self.determine_help_topics(self.help_arg) - self.groups = {} # elements are added by .add_group() - self.defaults = {} # elements are added by .parse_args() + + # elements are added by .add_group() + self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + # elements are added by .parse_args() + self.defaults = {} # type: Dict[str, Any] self.parser = configargparse.ArgParser( prog="certbot", @@ -805,7 +820,6 @@ class HelpfulArgumentParser(object): if self.help_arg: for v in verbs: self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"]) - return HelpfulArgumentGroup(self, topic) def add_plugin_args(self, plugins): @@ -1296,14 +1310,14 @@ def _paths_parser(helpful): verb = helpful.help_arg cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - section = ["paths", "install", "revoke", "certonly", "manage"] + sections = ["paths", "install", "revoke", "certonly", "manage"] if verb == "certonly": - add(section, "--cert-path", type=os.path.abspath, + add(sections, "--cert-path", type=os.path.abspath, default=flag_default("auth_cert_path"), help=cph) elif verb == "revoke": - add(section, "--cert-path", type=read_file, required=True, help=cph) + add(sections, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, help=cph) + add(sections, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): diff --git a/certbot/client.py b/certbot/client.py index cdfbdc252..1932ab83e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,7 +5,9 @@ import os import platform from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +# https://github.com/python/typeshed/blob/master/third_party/ +# 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi +from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key # type: ignore import josepy as jose import OpenSSL import zope.component @@ -160,11 +162,11 @@ def register(config, account_storage, tos_cb=None): config.email = None # Each new registration shall use a fresh new key - key = jose.JWKRSA(key=jose.ComparableRSAKey( - rsa.generate_private_key( + rsa_key = generate_private_key( public_exponent=65537, key_size=config.rsa_key_size, - backend=default_backend()))) + backend=default_backend()) + key = jose.JWKRSA(key=jose.ComparableRSAKey(rsa_key)) acme = acme_from_config_key(config, key) # TODO: add phone? regr = perform_registration(acme, config, tos_cb) @@ -609,8 +611,10 @@ def validate_key_csr(privkey, csr=None): if csr.form == "der": csr_obj = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, csr.data) - csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem") + cert_buffer = OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_obj + ) + csr = util.CSR(csr.file, cert_buffer, "pem") # If CSR is provided, it must be readable and valid. if csr.data and not crypto_util.valid_csr(csr.data): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 756bd7565..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -8,15 +8,18 @@ import hashlib import logging import os -import OpenSSL + import pyrfc3339 import six import zope.component +from OpenSSL import crypto +from OpenSSL import SSL # type: ignore from cryptography.hazmat.backends import default_backend -from cryptography import x509 # type: ignore +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util - +from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot import util @@ -47,7 +50,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): try: key_pem = make_key(key_size) except ValueError as err: - logger.exception(err) + logger.error("", exc_info=True) raise err config = zope.component.getUtility(interfaces.IConfig) @@ -111,11 +114,11 @@ def valid_csr(csr): """ try: - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) return req.verify(req.get_pubkey()) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -129,13 +132,13 @@ def csr_matches_pubkey(csr, privkey): :rtype: bool """ - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) - pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) + pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey) try: return req.verify(pkey) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -145,26 +148,26 @@ def import_csr_file(csrfile, data): :param str csrfile: CSR filename :param str data: contents of the CSR file - :returns: (`OpenSSL.crypto.FILETYPE_PEM`, + :returns: (`crypto.FILETYPE_PEM`, util.CSR object representing the CSR, list of domains requested in the CSR) :rtype: tuple """ - PEM = OpenSSL.crypto.FILETYPE_PEM - load = OpenSSL.crypto.load_certificate_request + PEM = crypto.FILETYPE_PEM + load = crypto.load_certificate_request try: # Try to parse as DER first, then fall back to PEM. - csr = load(OpenSSL.crypto.FILETYPE_ASN1, data) - except OpenSSL.crypto.Error: + csr = load(crypto.FILETYPE_ASN1, data) + except crypto.Error: try: csr = load(PEM, data) - except OpenSSL.crypto.Error: + except crypto.Error: raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) domains = _get_names_from_loaded_cert_or_req(csr) # Internally we always use PEM, so re-encode as PEM before returning. - data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr) + data_pem = crypto.dump_certificate_request(PEM, csr) return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains @@ -178,9 +181,9 @@ def make_key(bits): """ assert bits >= 1024 # XXX - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) - return OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + return crypto.dump_privatekey(crypto.FILETYPE_PEM, key) def valid_privkey(privkey): @@ -193,9 +196,9 @@ def valid_privkey(privkey): """ try: - return OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, privkey).check() - except (TypeError, OpenSSL.crypto.Error): + return crypto.load_privatekey( + crypto.FILETYPE_PEM, privkey).check() + except (TypeError, crypto.Error): return False @@ -224,13 +227,14 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain, 'rb') as chain: - chain, _ = pyopenssl_load_certificate(chain.read()) - with open(renewable_cert.cert, 'rb') as cert: - cert = x509.load_pem_x509_certificate(cert.read(), default_backend()) + with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] + chain, _ = pyopenssl_load_certificate(chain_file.read()) + with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) hash_name = cert.signature_hash_algorithm.name - OpenSSL.crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, OpenSSL.crypto.Error) as e: + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) @@ -246,11 +250,11 @@ def verify_cert_matches_priv_key(cert_path, key_path): :raises errors.Error: If they don't match. """ try: - context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + context = SSL.Context(SSL.SSLv23_METHOD) context.use_certificate_file(cert_path) context.use_privatekey_file(key_path) context.check_privatekey() - except (IOError, OpenSSL.SSL.Error) as e: + except (IOError, SSL.Error) as e: error_str = "verifying the cert located at {0} matches the \ private key located at {1} has failed. \ Details: {2}".format(cert_path, @@ -267,12 +271,12 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain) as chain: - chain = chain.read() - with open(renewable_cert.cert) as cert: - cert = cert.read() - with open(renewable_cert.fullchain) as fullchain: - fullchain = fullchain.read() + with open(renewable_cert.chain) as chain_file: # type: IO[str] + chain = chain_file.read() + with open(renewable_cert.cert) as cert_file: # type: IO[str] + cert = cert_file.read() + with open(renewable_cert.fullchain) as fullchain_file: # type: IO[str] + fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" error_str = error_str.format(renewable_cert.lineagename) @@ -294,43 +298,43 @@ def pyopenssl_load_certificate(data): openssl_errors = [] - for file_type in (OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1): + for file_type in (crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1): try: - return OpenSSL.crypto.load_certificate(file_type, data), file_type - except OpenSSL.crypto.Error as error: # TODO: other errors? + return crypto.load_certificate(file_type, data), file_type + except crypto.Error as error: # TODO: other errors? openssl_errors.append(error) raise errors.Error("Unable to load: {0}".format(",".join( str(error) for error in openssl_errors))) def _load_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): try: return load_func(typ, cert_or_req_str) - except OpenSSL.crypto.Error as error: - logger.exception(error) + except crypto.Error: + logger.error("", exc_info=True) raise def _get_sans_from_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): # pylint: disable=protected-access return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req( cert_or_req_str, load_func, typ)) -def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_sans_from_cert(cert, typ=crypto.FILETYPE_PEM): """Get a list of Subject Alternative Names from a certificate. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of Subject Alternative Names. :rtype: list """ return _get_sans_from_cert_or_req( - cert, OpenSSL.crypto.load_certificate, typ) + cert, crypto.load_certificate, typ) def _get_names_from_cert_or_req(cert_or_req, load_func, typ): @@ -343,24 +347,24 @@ def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) -def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_names_from_cert(csr, typ=crypto.FILETYPE_PEM): """Get a list of domains from a cert, including the CN if it is set. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of domain names. :rtype: list """ return _get_names_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate, typ) + csr, crypto.load_certificate, typ) -def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): +def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. - :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :param list chain: List of `crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). """ @@ -378,7 +382,7 @@ def notBefore(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notBefore) + return _notAfterBefore(cert_path, crypto.X509.get_notBefore) def notAfter(cert_path): @@ -390,15 +394,15 @@ def notAfter(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notAfter) + return _notAfterBefore(cert_path, crypto.X509.get_notAfter) def _notAfterBefore(cert_path, method): """Internal helper function for finding notbefore/notafter. :param str cert_path: path to a cert in PEM format - :param function method: one of ``OpenSSL.crypto.X509.get_notBefore`` - or ``OpenSSL.crypto.X509.get_notAfter`` + :param function method: one of ``crypto.X509.get_notBefore`` + or ``crypto.X509.get_notAfter`` :returns: the notBefore or notAfter value from the cert at cert_path :rtype: :class:`datetime.datetime` @@ -406,7 +410,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name with open(cert_path) as f: - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -443,7 +447,7 @@ def cert_and_chain_from_fullchain(fullchain_pem): :rtype: tuple """ - cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() + cert = crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) diff --git a/certbot/error_handler.py b/certbot/error_handler.py index e2737711e..5e72f8153 100644 --- a/certbot/error_handler.py +++ b/certbot/error_handler.py @@ -5,6 +5,10 @@ import os import signal import traceback +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Callable, Dict, List, Union +# pylint: enable=unused-import, no-name-in-module + from certbot import errors logger = logging.getLogger(__name__) @@ -56,9 +60,9 @@ class ErrorHandler(object): def __init__(self, func=None, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] - self.prev_handlers = {} - self.received_signals = [] + self.funcs = [] # type: List[Callable[[], Any]] + self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] + self.received_signals = [] # type: List[int] if func is not None: self.register(func, *args, **kwargs) @@ -88,6 +92,7 @@ class ErrorHandler(object): return retval def register(self, func, *args, **kwargs): + # type: (Callable, *Any, **Any) -> None """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error @@ -101,9 +106,8 @@ class ErrorHandler(object): while self.funcs: try: self.funcs[-1]() - except Exception as error: # pylint: disable=broad-except - logger.error("Encountered exception during recovery") - logger.exception(error) + except Exception: # pylint: disable=broad-except + logger.error("Encountered exception during recovery: ", exc_info=True) self.funcs.pop() def _set_signal_handlers(self): diff --git a/certbot/hooks.py b/certbot/hooks.py index b5c9046e9..d5239a437 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -6,6 +6,7 @@ import os from subprocess import Popen, PIPE +from acme.magic_typing import Set, List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util @@ -76,7 +77,8 @@ def pre_hook(config): if cmd: _run_pre_hook_if_necessary(cmd) -pre_hook.already = set() # type: ignore + +executed_pre_hooks = set() # type: Set[str] def _run_pre_hook_if_necessary(command): @@ -88,12 +90,12 @@ def _run_pre_hook_if_necessary(command): :param str command: pre-hook to be run """ - if command in pre_hook.already: + if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: logger.info("Running pre-hook command: %s", command) _run_hook(command) - pre_hook.already.add(command) + executed_pre_hooks.add(command) def post_hook(config): @@ -127,7 +129,8 @@ def post_hook(config): logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) -post_hook.eventually = [] # type: ignore + +post_hooks = [] # type: List[str] def _run_eventually(command): @@ -139,13 +142,13 @@ def _run_eventually(command): :param str command: post-hook to register to be run """ - if command not in post_hook.eventually: - post_hook.eventually.append(command) + if command not in post_hooks: + post_hooks.append(command) def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" - for cmd in post_hook.eventually: + for cmd in post_hooks: logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) diff --git a/certbot/log.py b/certbot/log.py index face93cb3..89626af99 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -191,9 +191,8 @@ class MemoryHandler(logging.handlers.MemoryHandler): only happens when flush(force=True) is called. """ - def __init__(self, target=None): + def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - capacity = float('inf') super(MemoryHandler, self).__init__(capacity, target=target) def close(self): diff --git a/certbot/main.py b/certbot/main.py index 8a9a37084..6c1d82793 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -11,6 +11,7 @@ import josepy as jose import zope.component from acme import errors as acme_errors +from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module import certbot @@ -520,8 +521,8 @@ def _determine_account(config): config, account_storage, tos_cb=_tos_cb) except errors.MissingCommandlineFlag: raise - except errors.Error as error: - logger.debug(error, exc_info=True) + except errors.Error: + logger.debug("", exc_info=True) raise errors.Error( "Unable to register an account with ACME server") @@ -1271,7 +1272,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) + displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ + # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 147d9e21a..ee1af4978 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -11,6 +11,8 @@ import zope.interface from josepy import util as jose_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import constants from certbot import crypto_util from certbot import errors @@ -331,8 +333,8 @@ class ChallengePerformer(object): def __init__(self, configurator): self.configurator = configurator - self.achalls = [] - self.indices = [] + self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.indices = [] # type: List[int] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 062c11650..c33a56785 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -10,6 +10,7 @@ from collections import OrderedDict import zope.interface import zope.interface.verify +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import interfaces @@ -189,7 +190,7 @@ class PluginsRegistry(collections.Mapping): @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} + plugins = {} # type: Dict[str, PluginEntryPoint] # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( diff --git a/certbot/plugins/disco_test.py b/certbot/plugins/disco_test.py index 220b902b3..720b90b16 100644 --- a/certbot/plugins/disco_test.py +++ b/certbot/plugins/disco_test.py @@ -8,6 +8,7 @@ import pkg_resources import six import zope.interface +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces @@ -250,7 +251,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] + order = [] # type: List[str] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 614449d34..53533d35a 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -5,7 +5,9 @@ import zope.component import zope.interface from acme import challenges +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import interfaces from certbot import errors from certbot import hooks @@ -98,7 +100,8 @@ when it receives a TLS ClientHello with the SNI extension set to super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() + self.env = dict() \ + # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None @classmethod diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index 4112810a2..ab480544a 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -6,6 +6,7 @@ import unittest import mock import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.display import util as display_util from certbot.tests import util as test_util from certbot import interfaces @@ -47,7 +48,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] + self.ifaces = [] # type: List[interfaces.IPlugin] def _call(self): from certbot.plugins.selection import pick_plugin diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 817403bd3..cb2e69511 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -3,6 +3,8 @@ import argparse import collections import logging import socket +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import OpenSSL import six @@ -10,7 +12,10 @@ import zope.interface from acme import challenges from acme import standalone as acme_standalone +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, Set, Tuple, List, Type, TYPE_CHECKING +from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces @@ -18,6 +23,11 @@ from certbot.plugins import common logger = logging.getLogger(__name__) +if TYPE_CHECKING: + ServedType = DefaultDict[ + acme_standalone.BaseDualNetworkedServers, + Set[achallenges.KeyAuthorizationAnnotatedChallenge] + ] class ServerManager(object): """Standalone servers manager. @@ -33,7 +43,7 @@ class ServerManager(object): """ def __init__(self, certs, http_01_resources): - self._instances = {} + self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] self.certs = certs self.http_01_resources = http_01_resources @@ -59,7 +69,8 @@ class ServerManager(object): address = (listenaddr, port) try: if challenge_type is challenges.TLSSNI01: - servers = acme_standalone.TLSSNI01DualNetworkedServers(address, self.certs) + servers = acme_standalone.TLSSNI01DualNetworkedServers( + address, self.certs) # type: acme_standalone.BaseDualNetworkedServers else: # challenges.HTTP01 servers = acme_standalone.HTTP01DualNetworkedServers( address, self.http_01_resources) @@ -103,7 +114,8 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] +SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \ +# type: List[Type[challenges.KeyAuthorizationChallenge]] class SupportedChallengesAction(argparse.Action): @@ -179,14 +191,15 @@ class Authenticator(common.Plugin): self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) - self.served = collections.defaultdict(set) + self.served = collections.defaultdict(set) # type: ServedType # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} - self.http_01_resources = set() + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = set() \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.servers = ServerManager(self.certs, self.http_01_resources) @@ -265,13 +278,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket.errno.EACCES: + if error.socket_error.errno == socket_errors.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - elif error.socket_error.errno == socket.errno.EADDRINUSE: + elif error.socket_error.errno == socket_errors.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 5227bc59e..47f44ff77 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -2,12 +2,18 @@ import argparse import socket import unittest +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import josepy as jose import mock import six +import OpenSSL.crypto # pylint: disable=unused-import + from acme import challenges +from acme import standalone as acme_standalone # pylint: disable=unused-import +from acme.magic_typing import Dict, Tuple, Set # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -21,8 +27,9 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot.plugins.standalone import ServerManager - self.certs = {} - self.http_01_resources = {} + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = {} \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): @@ -159,7 +166,7 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno @@ -174,7 +181,7 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE self.assertRaises(errors.PluginError, self._fail_perform, errno) self._assert_correct_yesno_call(mock_yesno) @@ -184,11 +191,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket.errno.EACCES + errno = socket_errors.EACCES self.assertRaises(errors.PluginError, self._fail_perform, errno) def test_perform_unexpected_socket_error(self): - errno = socket.errno.ENOTCONN + errno = socket_errors.ENOTCONN self.assertRaises( errors.StandaloneBindError, self._fail_perform, errno) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index a0c3f8564..9472a1ebb 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -3,6 +3,7 @@ import json import logging import os +from acme.magic_typing import Any, Dict # pylint: disable=unused-import, no-name-in-module from certbot import errors logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() + data = dict() # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 6328b16ef..5d0d7d586 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -10,8 +10,12 @@ import six import zope.component import zope.interface -from acme import challenges +from acme import challenges # pylint: disable=unused-import +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Set, DefaultDict, List +# pylint: enable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import cli from certbot import errors from certbot import interfaces @@ -64,10 +68,11 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} - self.performed = collections.defaultdict(set) + self.full_roots = {} # type: Dict[str, str] + self.performed = collections.defaultdict(set) \ + # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] # stack of dirs successfully created by this authenticator - self._created_dirs = [] + self._created_dirs = [] # type: List[str] def prepare(self): # pylint: disable=missing-docstring pass @@ -156,7 +161,6 @@ to serve all files under specified web root ({0}).""" " --help webroot for examples.") for name, path in path_map.items(): self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH) - logger.debug("Creating root challenges validation dir at %s", self.full_roots[name]) @@ -207,7 +211,6 @@ to serve all files under specified web root ({0}).""" os.umask(old_umask) self.performed[root_path].add(achall) - return response def cleanup(self, achalls): # pylint: disable=missing-docstring @@ -219,7 +222,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] + not_removed = [] # type: List[str] while len(self._created_dirs) > 0: path = self._created_dirs.pop() try: diff --git a/certbot/renewal.py b/certbot/renewal.py index 4651eeb36..0a6568426 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -11,6 +11,8 @@ import zope.component import OpenSSL +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + from certbot import cli from certbot import crypto_util from certbot import errors @@ -59,8 +61,8 @@ def _reconstitute(config, full_path): """ try: renewal_candidate = storage.RenewableCert(full_path, config) - except (errors.CertStorageError, IOError) as exc: - logger.warning(exc) + except (errors.CertStorageError, IOError): + logger.warning("", exc_info=True) logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None @@ -133,14 +135,15 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. + plugin_prefixes = [] # type: List[str] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) - plugin_prefixes = [] else: - plugin_prefixes = [renewalparams["authenticator"]] + plugin_prefixes.append(renewalparams["authenticator"]) - if renewalparams.get("installer", None) is not None: + if renewalparams.get("installer") is not None: plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') for config_item, config_value in six.iteritems(renewalparams): @@ -316,13 +319,13 @@ def report(msgs, category): def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): - out = [] + out = [] # type: List[str] notify = out.append disp = zope.component.getUtility(interfaces.IDisplay) def notify_error(err): """Notify and log errors.""" - notify(err) + notify(str(err)) logger.error(err) if config.dry_run: diff --git a/certbot/reverter.py b/certbot/reverter.py index 15ad1a987..683c0cc32 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -82,8 +82,10 @@ class Reverter(object): self._recover_checkpoint(self.config.temp_checkpoint_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for %s", - self.config.temp_checkpoint_dir) + logger.critical( + "Incomplete or failed recovery for %s", + self.config.temp_checkpoint_dir, + ) raise errors.ReverterError("Unable to revert temporary config") def rollback_checkpoints(self, rollback=1): @@ -123,7 +125,7 @@ class Reverter(object): try: self._recover_checkpoint(cp_dir) except errors.ReverterError: - logger.fatal("Failed to load checkpoint during rollback") + logger.critical("Failed to load checkpoint during rollback") raise errors.ReverterError( "Unable to load checkpoint during rollback") rollback -= 1 @@ -457,7 +459,7 @@ class Reverter(object): self._recover_checkpoint(self.config.in_progress_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for IN_PROGRESS " + logger.critical("Incomplete or failed recovery for IN_PROGRESS " "checkpoint - %s", self.config.in_progress_dir) raise errors.ReverterError( @@ -494,7 +496,7 @@ class Reverter(object): "Certbot probably shut down unexpectedly", os.linesep, path) except (IOError, OSError): - logger.fatal( + logger.critical( "Unable to remove filepaths contained within %s", file_list) raise errors.ReverterError( "Unable to remove filepaths contained within " diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 9a8a13498..76d1df90f 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -10,6 +10,7 @@ import zope.component from acme import challenges from acme import client as acme_client from acme import messages +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -354,12 +355,13 @@ class PollChallengesTest(unittest.TestCase): acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) ] - self.chall_update = {} + self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] + @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 1bba6991a..979cd97c1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -495,7 +495,8 @@ class SetByCliTest(unittest.TestCase): for v in ('manual', 'manual_auth_hook', 'manual_public_ip_logging_ok'): self.assertTrue(_call_set_by_cli(v, args, verb)) - cli.set_by_cli.detector = None + # https://github.com/python/mypy/issues/2087 + cli.set_by_cli.detector = None # type: ignore args = ['--manual-auth-hook', 'command'] for v in ('manual_auth_hook', 'manual_public_ip_logging_ok'): diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 333acf2b3..ac01103b8 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -8,6 +8,7 @@ import unittest import mock from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.tests.util import TempDirTestCase class CompleterTest(TempDirTestCase): @@ -21,7 +22,7 @@ class CompleterTest(TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] + self.paths = [] # type: List[str] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index d4c48c242..a4a65e2d4 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -6,6 +6,9 @@ import sys import unittest import mock +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Dict, Union +# pylint: enable=unused-import, no-name-in-module def get_signals(signums): @@ -23,8 +26,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = {} - prev_handlers = get_signals(signums) + prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] set_signals(dict((s, lambda s, _: signals.append(s)) for s in signums)) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 8619a1a2e..c9cfc69f9 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -5,6 +5,7 @@ import unittest import mock +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.tests import util @@ -106,8 +107,8 @@ class PreHookTest(HookTest): super(PreHookTest, self).tearDown() def _reset_pre_hook_already(self): - from certbot.hooks import pre_hook - pre_hook.already.clear() + from certbot.hooks import executed_pre_hooks + executed_pre_hooks.clear() def test_certonly(self): self.config.verb = "certonly" @@ -184,8 +185,8 @@ class PostHookTest(HookTest): super(PostHookTest, self).tearDown() def _reset_post_hook_eventually(self): - from certbot.hooks import post_hook - post_hook.eventually = [] + from certbot.hooks import post_hooks + del post_hooks[:] def test_certonly_and_run_with_hook(self): for verb in ("certonly", "run",): @@ -238,8 +239,8 @@ class PostHookTest(HookTest): self.assertEqual(self._get_eventually(), expected) def _get_eventually(self): - from certbot.hooks import post_hook - return post_hook.eventually + from certbot.hooks import post_hooks + return post_hooks class RunSavedPostHooksTest(HookTest): @@ -248,23 +249,23 @@ class RunSavedPostHooksTest(HookTest): @classmethod def _call(cls, *args, **kwargs): from certbot.hooks import run_saved_post_hooks - return run_saved_post_hooks(*args, **kwargs) + return run_saved_post_hooks() def _call_with_mock_execute_and_eventually(self, *args, **kwargs): """Call run_saved_post_hooks but mock out execute and eventually - certbot.hooks.post_hook.eventually is replaced with + certbot.hooks.post_hooks is replaced with self.eventually. The mock execute object is returned rather than the return value of run_saved_post_hooks. """ - eventually_path = "certbot.hooks.post_hook.eventually" + eventually_path = "certbot.hooks.post_hooks" with mock.patch(eventually_path, new=self.eventually): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] + self.eventually = [] # type: List[str] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 549d2c5e1..c5991347e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -10,6 +10,7 @@ import mock import six from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors @@ -21,9 +22,9 @@ class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot.log.pre_arg_parse_setup.""" @classmethod - def _call(cls, *args, **kwargs): + def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot.log import pre_arg_parse_setup - return pre_arg_parse_setup(*args, **kwargs) + return pre_arg_parse_setup() @mock.patch('certbot.log.sys') @mock.patch('certbot.log.pre_arg_parse_except_hook') @@ -38,16 +39,16 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - MemoryHandler = logging.handlers.MemoryHandler - memory_handler = None + memory_handler = None # type: Optional[logging.handlers.MemoryHandler] for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] - if memory_handler is None and isinstance(handler, MemoryHandler): + if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): memory_handler = handler + target = memory_handler.target # type: ignore else: self.assertTrue(isinstance(handler, logging.StreamHandler)) self.assertTrue( - isinstance(memory_handler.target, logging.StreamHandler)) + isinstance(target, logging.StreamHandler)) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 0986ff060..14cde27ee 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -16,12 +16,14 @@ import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli from certbot import constants from certbot import configuration from certbot import crypto_util from certbot import errors +from certbot import interfaces # pylint: disable=unused-import from certbot import main from certbot import updater from certbot import util @@ -600,14 +602,14 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met if mockisfile: orig_open = os.path.isfile - def mock_isfile(fn, *args, **kwargs): + def mock_isfile(fn, *args, **kwargs): # pylint: disable=unused-argument """Mock os.path.isfile()""" if (fn.endswith("cert") or fn.endswith("chain") or fn.endswith("privkey")): return True else: - return orig_open(fn, *args, **kwargs) + return orig_open(fn) with mock.patch("os.path.isfile") as mock_if: mock_if.side_effect = mock_isfile @@ -836,7 +838,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -851,7 +853,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, uid, strict): @@ -873,7 +875,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -891,7 +893,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -1040,9 +1042,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') - def write_msg(message, *args, **kwargs): + def write_msg(message, *args, **kwargs): # pylint: disable=unused-argument """Write message to stdout.""" - _, _ = args, kwargs stdout.write(message) try: diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 9ec8dca28..22e11e672 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -12,7 +12,7 @@ class ReporterTest(unittest.TestCase): from certbot import reporter self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) - self.old_stdout = sys.stdout + self.old_stdout = sys.stdout # type: ignore sys.stdout = six.StringIO() def tearDown(self): @@ -21,32 +21,32 @@ class ReporterTest(unittest.TestCase): def test_multiline_message(self): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("Line 1\n" in output) self.assertTrue("Line 2" in output) def test_tty_print_empty(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self.test_no_tty_print_empty() def test_no_tty_print_empty(self): self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore try: raise ValueError except ValueError: self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore def test_tty_successful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._successful_exit_common() def test_no_tty_successful_exit(self): self._successful_exit_common() def test_tty_unsuccessful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._unsuccessful_exit_common() def test_no_tty_unsuccessful_exit(self): @@ -55,7 +55,7 @@ class ReporterTest(unittest.TestCase): def _successful_exit_common(self): self._add_messages() self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" in output) @@ -67,7 +67,7 @@ class ReporterTest(unittest.TestCase): raise ValueError except ValueError: self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" not in output) diff --git a/certbot/util.py b/certbot/util.py index 55acd624f..8e84c29ba 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -20,6 +20,7 @@ from collections import OrderedDict import configargparse +from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import lock @@ -218,8 +219,12 @@ def safe_open(path, mode="w", chmod=None, buffering=None): """ # pylint: disable=star-args - open_args = () if chmod is None else (chmod,) - fdopen_args = () if buffering is None else (buffering,) + open_args = () # type: Union[Tuple[()], Tuple[int]] + if chmod is not None: + open_args = (chmod,) + fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + if buffering is not None: + fdopen_args = (buffering,) return os.fdopen( os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), mode, *fdopen_args) @@ -303,9 +308,8 @@ def get_filtered_names(all_names): for name in all_names: try: filtered_names.add(enforce_le_validity(name)) - except errors.ConfigurationError as error: - logger.debug('Not suggesting name "%s"', name) - logger.debug(error) + except errors.ConfigurationError: + logger.debug('Not suggesting name "%s"', name, exc_info=True) return filtered_names diff --git a/mypy.ini b/mypy.ini index d00c21ae7..f0c99e65f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,12 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-acme.magic_typing_test] +ignore_errors = True + +[mypy-certbot.*] +check_untyped_defs = True + [mypy-certbot_apache.*] check_untyped_defs = True diff --git a/setup.py b/setup.py index 3760fd35b..ee0470d3a 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.22.1', + 'acme>0.24.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index df13cdbef..d965d4470 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -30,7 +30,7 @@ josepy==1.0.1 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -mypy==0.580 +mypy==0.600 ndg-httpsclient==0.3.2 oauth2client==2.0.0 pathlib2==2.3.0 diff --git a/tox.ini b/tox.ini index 8c4d6c38d..2834ef9f9 100644 --- a/tox.ini +++ b/tox.ini @@ -121,8 +121,8 @@ commands = [testenv:mypy] basepython = python3 commands = - {[base]pip_install} .[dev3] {[base]install_packages} + {[base]pip_install} .[dev3] mypy {[base]source_paths} [testenv:apacheconftest] From 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Fri, 18 May 2018 09:10:41 -0700 Subject: [PATCH 214/639] switch signature verification to use pure cryptography (#6000) * switch signature verification to use pure cryptography On systems that prevent write/execute pages this prevents a segfault that is caused by pyopenssl creating a dynamic callback in the verification helper. * switch to using a verifier for older cryptography releases also add ec support, test vectors, and a test --- certbot/crypto_util.py | 36 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 ++++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 ++++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 +++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..71f6c990c 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,11 +12,16 @@ import os import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,26 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY----- From dec97fc1269636a94a9763d6a2e641d2ada3ac6f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 17:48:30 -0700 Subject: [PATCH 215/639] Revert "Add link to pycon issues (#5959)" This reverts commit 68359086fffca8805893bf6133c53b5f75357a7f. --- docs/contributing.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 52f08efe0..ed986c562 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -72,10 +72,6 @@ found in the `virtualenv docs`_. Find issues to work on ---------------------- -.. note:: If you're sprinting on Certbot at PyCon, you can find especially good - issues to work on during the event `here - `_. - You can find the open issues in the `github issue tracker`_. Comparatively easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan From c9a206ca890c3f39c5f0dffce9862cf439b9d5a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 21 May 2018 20:23:21 -0700 Subject: [PATCH 216/639] Get mypy passing with check_untyped_defs everywhere (#6021) * unchecked_typed_defs everywhere * fix mypy for lock_test * add magic_typing * fix mypy in letshelp * fix validator errors in compat test * fix mypy for test_driver.py * fix mypy in util.py * delint --- .../certbot_compatibility_test/test_driver.py | 48 ++++++++----------- .../certbot_compatibility_test/util.py | 12 ++--- .../certbot_compatibility_test/validator.py | 5 +- letshelp-certbot/letshelp_certbot/apache.py | 5 +- .../letshelp_certbot/apache_test.py | 16 +++++-- .../letshelp_certbot/magic_typing.py | 16 +++++++ .../letshelp_certbot/magic_typing_test.py | 41 ++++++++++++++++ mypy.ini | 46 ++---------------- tests/lock_test.py | 2 +- 9 files changed, 106 insertions(+), 85 deletions(-) create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing.py create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing_test.py diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 2c6c917b3..9eea95e67 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -15,6 +15,7 @@ from six.moves import xrange # pylint: disable=import-error,redefined-builtin from acme import challenges from acme import crypto_util from acme import messages +from acme.magic_typing import List, Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util @@ -52,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir): try: responses = plugin.perform(achalls) - except le_errors.Error as error: - logger.error("Performing challenges on %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Performing challenges on %s caused an error:", config, exc_info=True) return False success = True @@ -82,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir): if success: try: plugin.cleanup(achalls) - except le_errors.Error as error: - logger.error("Challenge cleanup for %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True) success = False if _dirs_are_unequal(config, backup): @@ -147,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains): try: plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path) plugin.save() # Needed by the Apache plugin - except le_errors.Error as error: - logger.error("**** Plugin failed to deploy certificate for %s:", domain) - logger.exception(error) + except le_errors.Error: + logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True) return False if not _save_and_restart(plugin, "deployed"): @@ -179,7 +177,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] + domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] for domain, info in domains_and_info: try: @@ -192,10 +190,9 @@ def test_enhancements(plugin, domains): # Don't immediately fail because a redirect may already be enabled logger.warning("*** Plugin failed to enable redirect for %s:", domain) logger.warning("%s", error) - except le_errors.Error as error: + except le_errors.Error: logger.error("*** An error occurred while enabling redirect for %s:", - domain) - logger.exception(error) + domain, exc_info=True) if not _save_and_restart(plugin, "enhanced"): return False @@ -222,9 +219,8 @@ def _save_and_restart(plugin, title=None): plugin.save(title) plugin.restart() return True - except le_errors.Error as error: - logger.error("*** Plugin failed to save and restart server:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin failed to save and restart server:", exc_info=True) return False @@ -232,9 +228,8 @@ def test_rollback(plugin, config, backup): """Tests the rollback checkpoints function""" try: plugin.rollback_checkpoints(1337) - except le_errors.Error as error: - logger.error("*** Plugin raised an exception during rollback:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin raised an exception during rollback:", exc_info=True) return False if _dirs_are_unequal(config, backup): @@ -263,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2): logger.error("The following files and directories are only " "present in one directory") if dircmp.left_only: - logger.error(dircmp.left_only) + logger.error(str(dircmp.left_only)) else: - logger.error(dircmp.right_only) + logger.error(str(dircmp.right_only)) return True elif dircmp.common_funny or dircmp.funny_files: logger.error("The following files and directories could not be " "compared:") if dircmp.common_funny: - logger.error(dircmp.common_funny) + logger.error(str(dircmp.common_funny)) else: - logger.error(dircmp.funny_files) + logger.error(str(dircmp.funny_files)) return True elif dircmp.diff_files: logger.error("The following files differ:") - logger.error(dircmp.diff_files) + logger.error(str(dircmp.diff_files)) return True for subdir in dircmp.subdirs.itervalues(): @@ -354,9 +349,8 @@ def main(): success = test_authenticator(plugin, config, temp_dir) if success and args.install: success = test_installer(args, plugin, config, temp_dir) - except errors.Error as error: - logger.error("Tests on %s raised:", config) - logger.exception(error) + except errors.Error: + logger.error("Tests on %s raised:", config, exc_info=True) success = False if success: diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index 4155944bd..6051bbc2e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -26,12 +26,12 @@ def create_le_config(parent_dir): config = copy.deepcopy(constants.CLI_DEFAULTS) le_dir = os.path.join(parent_dir, "certbot") - config["config_dir"] = os.path.join(le_dir, "config") - config["work_dir"] = os.path.join(le_dir, "work") - config["logs_dir"] = os.path.join(le_dir, "logs_dir") - os.makedirs(config["config_dir"]) - os.mkdir(config["work_dir"]) - os.mkdir(config["logs_dir"]) + os.mkdir(le_dir) + for dir_name in ("config", "logs", "work"): + full_path = os.path.join(le_dir, dir_name) + os.mkdir(full_path) + full_name = dir_name + "_dir" + config[full_name] = full_path config["domains"] = None diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 791fe0da2..fd2f95702 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -33,7 +33,7 @@ class Validator(object): try: presented_cert = crypto_util.probe_sni(name, host, port) except acme_errors.Error as error: - logger.exception(error) + logger.exception(str(error)) return False return presented_cert.digest("sha256") == cert.digest("sha256") @@ -86,8 +86,7 @@ class Validator(object): return False try: - _, max_age_value = max_age[0] - max_age_value = int(max_age_value) + max_age_value = int(max_age[0][1]) except ValueError: logger.error("Server responded with invalid HSTS header field") return False diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index f77a6a1b0..50f3c5ef6 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -16,6 +16,8 @@ import textwrap import six +from letshelp_certbot.magic_typing import List # pylint: disable=unused-import, no-name-in-module + _DESCRIPTION = """ Let's Help is a simple script you can run to help out the Certbot project. Since Certbot will support automatically configuring HTTPS on @@ -87,7 +89,8 @@ def copy_config(server_root, temp_dir): :rtype: `tuple` of `list` of `str` """ - copied_files, copied_dirs = [], [] + copied_files = [] # type: List[str] + copied_dirs = [] # type: List[str] dir_len = len(os.path.dirname(server_root)) for config_path, config_dirs, config_files in os.walk(server_root): diff --git a/letshelp-certbot/letshelp_certbot/apache_test.py b/letshelp-certbot/letshelp_certbot/apache_test.py index e0656ae05..a1115bc06 100644 --- a/letshelp-certbot/letshelp_certbot/apache_test.py +++ b/letshelp-certbot/letshelp_certbot/apache_test.py @@ -203,13 +203,19 @@ class LetsHelpApacheTest(unittest.TestCase): tempdir_path, "config.tar.gz")) tempdir = tar.next() - self.assertTrue(tempdir.isdir()) - self.assertEqual(tempdir.name, ".") + if tempdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(tempdir.isdir()) + self.assertEqual(tempdir.name, ".") testdir = tar.next() - self.assertTrue(testdir.isdir()) - self.assertEqual(os.path.basename(testdir.name), - testdir_basename) + if testdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(testdir.isdir()) + self.assertEqual(os.path.basename(testdir.name), + testdir_basename) self.assertEqual(tar.next(), None) diff --git a/letshelp-certbot/letshelp_certbot/magic_typing.py b/letshelp-certbot/letshelp_certbot/magic_typing.py new file mode 100644 index 000000000..471b8dfa9 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing.py @@ -0,0 +1,16 @@ +"""Shim class to not have to depend on typing module in prod.""" +import sys + +class TypingClass(object): + """Ignore import errors by getting anything""" + def __getattr__(self, name): + return None + +try: + # mypy doesn't respect modifying sys.modules + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import +except ImportError: + sys.modules[__name__] = TypingClass() diff --git a/letshelp-certbot/letshelp_certbot/magic_typing_test.py b/letshelp-certbot/letshelp_certbot/magic_typing_test.py new file mode 100644 index 000000000..200ca03b8 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing_test.py @@ -0,0 +1,41 @@ +"""Tests for letshelp_certbot.magic_typing.""" +import sys +import unittest + +import mock + + +class MagicTypingTest(unittest.TestCase): + """Tests for letshelp_certbot.magic_typing.""" + def test_import_success(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + typing_class_mock = mock.MagicMock() + text_mock = mock.MagicMock() + typing_class_mock.Text = text_mock + sys.modules['typing'] = typing_class_mock + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertEqual(Text, text_mock) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + def test_import_failure(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + sys.modules['typing'] = None + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertTrue(Text is None) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/mypy.ini b/mypy.ini index f0c99e65f..188ed031f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,48 +1,10 @@ [mypy] -python_version = 2.7 -ignore_missing_imports = True - -[mypy-acme.*] check_untyped_defs = True +ignore_missing_imports = True +python_version = 2.7 [mypy-acme.magic_typing_test] ignore_errors = True -[mypy-certbot.*] -check_untyped_defs = True - -[mypy-certbot_apache.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudflare.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudxns.*] -check_untyped_defs = True - -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsimple.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsmadeeasy.*] -check_untyped_defs = True - -[mypy-certbot_dns_google.*] -check_untyped_defs = True - -[mypy-certbot_dns_luadns.*] -check_untyped_defs = True - -[mypy-certbot_dns_nsone.*] -check_untyped_defs = True - -[mypy-certbot_dns_rfc2136.*] -check_untyped_defs = True - -[mypy-certbot_dns_route53.*] -check_untyped_defs = True - -[mypy-certbot_nginx.*] -check_untyped_defs = True +[mypy-letshelp_certbot.magic_typing_test] +ignore_errors = True diff --git a/tests/lock_test.py b/tests/lock_test.py index 4bb2865b4..b01cc5d58 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -198,7 +198,7 @@ def report_failure(err_msg, out, err): :param str err: stderr output """ - logger.fatal(err_msg) + logger.critical(err_msg) log_output(logging.INFO, out, err) sys.exit(err_msg) From cfd4b8f3634df21370f0e21876400e46d8b5b8eb Mon Sep 17 00:00:00 2001 From: Quang Vu Date: Tue, 22 May 2018 15:32:44 -0700 Subject: [PATCH 217/639] #4242 Support multi emails register (#5994) This change will allow registering/updating account with multi emails. Detail is enclosed in #4242 * support multi emails register * add more test cases * update test to unregister before register * update create path to support multi emaill * refactor payload updating * fix typo * move command line doc to another place * revert the change for updating account registration info, added unit test * rearrange text for consistency --- acme/acme/messages.py | 2 +- certbot/cli.py | 2 +- certbot/client.py | 3 ++- certbot/interfaces.py | 4 +++- certbot/main.py | 33 +++++++++++++++++---------------- certbot/tests/main_test.py | 10 +++++++--- tests/boulder-integration.sh | 9 ++++++++- 7 files changed, 39 insertions(+), 24 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 03dbc3255..827a4dd11 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -285,7 +285,7 @@ class Registration(ResourceBody): if phone is not None: details.append(cls.phone_prefix + phone) if email is not None: - details.append(cls.email_prefix + email) + details.extend([cls.email_prefix + mail for mail in email.split(',')]) kwargs['contact'] = tuple(details) return cls(**kwargs) diff --git a/certbot/cli.py b/certbot/cli.py index 8a1ad381a..25319bbd8 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -430,7 +430,7 @@ VERB_HELP = [ }), ("enhance", { "short": "Add security enhancements to your existing configuration", - "opts": ("Helps to harden the TLS configration by adding security enhancements " + "opts": ("Helps to harden the TLS configuration by adding security enhancements " "to already existing configuration."), "usage": "\n\n certbot enhance [options]\n\n" }), diff --git a/certbot/client.py b/certbot/client.py index 1932ab83e..dadc3a0f8 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -185,8 +185,9 @@ def perform_registration(acme, config, tos_cb): Actually register new account, trying repeatedly if there are email problems - :param .IConfig config: Client configuration. :param acme.client.Client client: ACME client object. + :param .IConfig config: Client configuration. + :param Callable tos_cb: a callback to handle Term of Service agreement. :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` diff --git a/certbot/interfaces.py b/certbot/interfaces.py index c96f6bd51..6233e3592 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -201,7 +201,9 @@ class IConfig(zope.interface.Interface): """ server = zope.interface.Attribute("ACME Directory Resource URI.") email = zope.interface.Attribute( - "Email used for registration and recovery contact. (default: Ask)") + "Email used for registration and recovery contact. Use comma to " + "register multiple emails, ex: u1@example.com,u2@example.com. " + "(default: Ask).") rsa_key_size = zope.interface.Attribute("Size of the RSA key.") must_staple = zope.interface.Attribute( "Adds the OCSP Must Staple extension to the certificate. " diff --git a/certbot/main.py b/certbot/main.py index 6c1d82793..dad1b793e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -487,6 +487,21 @@ def _determine_account(config): :raises errors.Error: If unable to register an account with ACME server """ + def _tos_cb(terms_of_service): + if config.tos: + return True + msg = ("Please read the Terms of Service at {0}. You " + "must agree in order to register with the ACME " + "server at {1}".format( + terms_of_service, config.server)) + obj = zope.component.getUtility(interfaces.IDisplay) + result = obj.yesno(msg, "Agree", "Cancel", + cli_flag="--agree-tos", force_interactive=True) + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") + account_storage = account.AccountFileStorage(config) acme = None @@ -501,21 +516,6 @@ def _determine_account(config): else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - - def _tos_cb(terms_of_service): - if config.tos: - return True - msg = ("Please read the Terms of Service at {0}. You " - "must agree in order to register with the ACME " - "server at {1}".format( - terms_of_service, config.server)) - obj = zope.component.getUtility(interfaces.IDisplay) - result = obj.yesno(msg, "Agree", "Cancel", - cli_flag="--agree-tos", force_interactive=True) - if not result: - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) @@ -735,8 +735,9 @@ def register(config, unused_plugins): acc, acme = _determine_account(config) cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. + acc_contacts = ['mailto:' + email for email in config.email.split(',')] acc.regr = cb_client.acme.update_registration(acc.regr.update( - body=acc.regr.body.update(contact=('mailto:' + config.email,)))) + body=acc.regr.body.update(contact=acc_contacts))) account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 14cde27ee..8c9e24354 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1435,7 +1435,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mocked_storage = mock.MagicMock() mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] - mocked_det.return_value = (mock.MagicMock(), "foo") + mock_acc = mock.MagicMock() + mock_regr = mock_acc.regr + mocked_det.return_value = (mock_acc, "foo") cb_client = mock.MagicMock() mocked_client.Client.return_value = cb_client x = self._call_no_clientmock( @@ -1445,8 +1447,10 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(x[0] is None) # and we got supposedly did update the registration from # the server - self.assertTrue( - cb_client.acme.update_registration.called) + reg_arg = cb_client.acme.update_registration.call_args[0][0] + # Test the return value of .update() was used because + # the regr is immutable. + self.assertEqual(reg_arg, mock_regr.update()) # and we saved the updated registration on disk self.assertTrue(mocked_storage.save_regr.called) self.assertTrue( diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 9748befa3..e931e30f3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -191,7 +191,14 @@ for dir in $renewal_hooks_dirs; do exit 1 fi done -common register --update-registration --email example@example.org + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org common plugins --init --prepare | grep webroot From 8440d0814de346fb8beb2ca1497e1cc7803a19fe Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Tue, 22 May 2018 18:35:12 -0400 Subject: [PATCH 218/639] fixed dependency-requirements.txt (#6023) --- letsencrypt-auto-source/letsencrypt-auto | 18 ++++++++++++------ .../pieces/dependency-requirements.txt | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0b83b08a7..28281e20d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index a30a32b48..376e19deb 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -59,9 +59,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -116,7 +118,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -142,7 +145,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -170,9 +174,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ From deb5b072d9bfb10646df5a7de71f05ac6c9b9cd6 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Wed, 23 May 2018 12:59:49 -0400 Subject: [PATCH 219/639] Log cases when standalone fails to bind a port. (#5985) * Log cases when standalone fails to bind a port. * Fix linter + changed log message * Changed multiline string format * Fixed indentation in standalone.py --- acme/acme/standalone.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index a370501ee..ff9159933 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -83,9 +83,22 @@ class BaseDualNetworkedServers(object): new_address = (server_address[0],) + (port,) + server_address[2:] new_args = (new_address,) + remaining_args server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args - except socket.error: - logger.debug("Failed to bind to %s:%s using %s", new_address[0], + logger.debug( + "Successfully bound to %s:%s using %s", new_address[0], new_address[1], "IPv6" if ip_version else "IPv4") + except socket.error: + if self.servers: + # Already bound using IPv6. + logger.debug( + "Certbot wasn't able to bind to %s:%s using %s, this " + + "is often expected due to the dual stack nature of " + + "IPv6 socket implementations.", + new_address[0], new_address[1], + "IPv6" if ip_version else "IPv4") + else: + logger.debug( + "Failed to bind to %s:%s using %s", new_address[0], + new_address[1], "IPv6" if ip_version else "IPv4") else: self.servers.append(server) # If two servers are set up and port 0 was passed in, ensure we always From 4304ff0d623bc3512343c09f545579119887138e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 23 May 2018 11:33:21 -0700 Subject: [PATCH 220/639] Bring up just the boulder container. (#6031) Boulder recently added a "netaccess" container which may conflict. --- tests/boulder-fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index d513ec064..7a677ad12 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,7 +17,7 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -docker-compose up -d +docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder until curl http://localhost:4000/directory 2>/dev/null; do From 0b215366b1c66950195ddbbfb6f16b8657f4b198 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 May 2018 13:57:22 -0700 Subject: [PATCH 221/639] turn off cancel notifications (#5918) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 111ddb3d4..e3d964326 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,6 +107,7 @@ notifications: irc: channels: - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw=" + on_cancel: never on_success: never on_failure: always use_notice: true From a1f5dc27f28e46cc2aa92777f12f17b418fd6e7c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 23 May 2018 14:03:30 -0700 Subject: [PATCH 222/639] Add domain to error message when no matching server block found (#6034) --- certbot-nginx/certbot_nginx/configurator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 118699aa2..293f7378e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -334,7 +334,7 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost(port) + default_vhost = self._get_default_vhost(port, domain) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,7 +350,7 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port): + def _get_default_vhost(self, port, domain): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] @@ -367,7 +367,7 @@ class NginxConfigurator(common.Installer): # TODO: present a list of vhosts for user to choose from raise errors.MisconfigurationError("Could not automatically find a matching server" - " block. Set the `server_name` directive to use the Nginx installer.") + " block for %s. Set the `server_name` directive to use the Nginx installer." % domain) def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. From b1bcccb04bbd4d9d35bc95397f9ba1633b91242a Mon Sep 17 00:00:00 2001 From: Jeremy Gillula Date: Wed, 23 May 2018 20:40:34 -0700 Subject: [PATCH 223/639] Changing opt-in ask for emails (#6035) --- certbot/eff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/eff.py b/certbot/eff.py index b047c0b97..388ae986b 100644 --- a/certbot/eff.py +++ b/certbot/eff.py @@ -41,8 +41,8 @@ def _want_subscription(): 'Would you be willing to share your email address with the ' "Electronic Frontier Foundation, a founding partner of the Let's " 'Encrypt project and the non-profit organization that develops ' - "Certbot? We'd like to send you email about EFF and our work to " - 'encrypt the web, protect its users and defend digital rights.') + "Certbot? We'd like to send you email about our work encrypting " + "the web, EFF news, campaigns, and ways to support digital freedom. ") display = zope.component.getUtility(interfaces.IDisplay) return display.yesno(prompt, default=False) From a03c68fc8334cac84e8ee22f2b7b7fa7fc323ab9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 24 May 2018 10:53:21 -0700 Subject: [PATCH 224/639] Clean up boulder-fetch a bit. (#6032) The value for FAKE_DNS is now always the same because Boulder's docker-compose hardcodes it, so skip some sed. Set a time limit on how long we'll wait for boulder to come up. --- tests/boulder-fetch.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 7a677ad12..31e0f6b30 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -11,16 +11,20 @@ if [ ! -d ${BOULDERPATH} ]; then fi cd ${BOULDERPATH} -FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) -[ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 -sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml +sed -i "s/FAKE_DNS: .*/FAKE_DNS: 10.77.77.1/" docker-compose.yml docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder -until curl http://localhost:4000/directory 2>/dev/null; do - echo waiting for boulder - sleep 1 +for n in `seq 1 150` ; do + if curl http://localhost:4000/directory 2>/dev/null; then + break + else + sleep 1 + fi done + +if ! curl http://localhost:4000/directory 2>/dev/null; then + echo "timed out waiting for boulder to start" + exit 1 +fi From e48c653245bc08b7e517465aea32f678c5b9b64b Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 25 May 2018 21:00:37 +0300 Subject: [PATCH 225/639] Change GenericUpdater parameter to lineage (#6030) In order to give more flexibility for plugins using interfaces.GenericUpdater interface, lineage needs to be passed to the updater method instead of individual domains. All of the (present and potential) installers do not work on per domain basis, while the lineage does contain a list of them for installers which do. This also means that we don't unnecessarily run the updater method multiple times, potentially invoking expensive tooling up to $max_san_amount times. * Make GenericUpdater use lineage as parameter and get invoked only once per lineage --- certbot/interfaces.py | 18 +++++++++--------- certbot/tests/renewupdater_test.py | 11 ++++------- certbot/updater.py | 7 +++---- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 6233e3592..a5fb426e6 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -604,10 +604,10 @@ class IReporter(zope.interface.Interface): # When "certbot renew" is run, Certbot will iterate over each lineage and check # if the selected installer for that lineage is a subclass of each updater # class. If it is and the update of that type is configured to be run for that -# lineage, the relevant update function will be called for each domain in the -# lineage. These functions are never called for other subcommands, so if an -# installer wants to perform an update during the run or install subcommand, it -# should do so when :func:`IInstaller.deploy_cert` is called. +# lineage, the relevant update function will be called for it. These functions +# are never called for other subcommands, so if an installer wants to perform +# an update during the run or install subcommand, it should do so when +# :func:`IInstaller.deploy_cert` is called. @six.add_metaclass(abc.ABCMeta) class GenericUpdater(object): @@ -623,7 +623,7 @@ class GenericUpdater(object): """ @abc.abstractmethod - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): """Perform any update types defined by the installer. If an installer is a subclass of the class containing this method, this @@ -631,9 +631,10 @@ class GenericUpdater(object): update defined by the installer should be run conditionally, the installer needs to handle checking the conditions itself. - This method is called once for each domain. + This method is called once for each lineage. - :param str domain: domain to handle the updates for + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert """ @@ -661,8 +662,7 @@ class RenewDeployer(object): This method is called once for each lineage renewed - :param lineage: Certificate lineage object that is set if certificate - was renewed on this run. + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert """ diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index 9d0f8d515..ade8db390 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -19,7 +19,7 @@ class RenewUpdaterTest(unittest.TestCase): # pylint: disable=unused-argument self.restart = mock.MagicMock() self.callcounter = mock.MagicMock() - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): self.callcounter(*args, **kwargs) class MockInstallerRenewDeployer(interfaces.RenewDeployer): @@ -46,9 +46,7 @@ class RenewUpdaterTest(unittest.TestCase): def test_server_updates(self, _, mock_select, mock_getsave): config = self.get_config({"disable_renew_updates": False}) - lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] - mock_getsave.return_value = lineage + mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater @@ -59,14 +57,13 @@ class RenewUpdaterTest(unittest.TestCase): mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, lineage) - self.assertEqual(mock_generic_updater.callcounter.call_count, 2) + updater.run_generic_updaters(config, None, mock.MagicMock()) + self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] mock_deployer = self.renew_deployer updater.run_renewal_deployer(lineage, mock_deployer, config) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) diff --git a/certbot/updater.py b/certbot/updater.py index f822c55ee..1216f38a6 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -61,7 +61,6 @@ def _run_updaters(lineage, installer, config): :returns: `None` :rtype: None """ - for domain in lineage.names(): - if not config.disable_renew_updates: - if isinstance(installer, interfaces.GenericUpdater): - installer.generic_updates(domain) + if not config.disable_renew_updates: + if isinstance(installer, interfaces.GenericUpdater): + installer.generic_updates(lineage) From 9f6b147d6f06e534351377195b4f28c61270115d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sat, 26 May 2018 18:31:23 +0300 Subject: [PATCH 226/639] Do not call updaters and deployers when run with --dry-run (#6038) When Certbot is run with --dry-run, skip running GenericUpdater and RenewDeployer interface methods. This PR also makes the parameter order of updater.run_generic_updaters and updater.run_renewal_deployer consistent. Fixes #5927 * Do not call updaters and deployers when run with --dry-run * Use ConfigTestCase instead of mocking config objects manually --- certbot/main.py | 2 +- certbot/renewal.py | 4 ++-- certbot/tests/main_test.py | 3 ++- certbot/tests/renewupdater_test.py | 35 ++++++++++++++++++------------ certbot/updater.py | 20 ++++++++++++----- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index dad1b793e..8fa6ebfc6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1163,7 +1163,7 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. - updater.run_renewal_deployer(renewed_lineage, installer, config) + updater.run_renewal_deployer(config, renewed_lineage, installer) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) diff --git a/certbot/renewal.py b/certbot/renewal.py index 0a6568426..236330a85 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -431,8 +431,8 @@ def handle_renewal_request(config): renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, expiry.strftime("%Y-%m-%d"))) # Run updater interface methods - updater.run_generic_updaters(lineage_config, plugins, - renewal_candidate) + updater.run_generic_updaters(lineage_config, renewal_candidate, + plugins) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8c9e24354..36821cf53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1464,7 +1464,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met None, None, None) with mock.patch('certbot.updater.logger.warning') as mock_log: - updater.run_generic_updaters(None, None, None) + self.config.dry_run = False + updater.run_generic_updaters(self.config, None, None) self.assertTrue(mock_log.called) self.assertTrue("Could not choose appropriate plugin for updaters" in mock_log.call_args[0][0]) diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index ade8db390..bd1cd891e 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -9,10 +9,11 @@ from certbot import updater import certbot.tests.util as test_util -class RenewUpdaterTest(unittest.TestCase): +class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): + super(RenewUpdaterTest, self).setUp() class MockInstallerGenericUpdater(interfaces.GenericUpdater): """Mock class that implements GenericUpdater""" def __init__(self, *args, **kwargs): @@ -33,41 +34,47 @@ class RenewUpdaterTest(unittest.TestCase): self.generic_updater = MockInstallerGenericUpdater() self.renew_deployer = MockInstallerRenewDeployer() - def get_config(self, args): - """Get mock config from dict of parameters""" - config = mock.MagicMock() - for key in args.keys(): - config.__dict__[key] = args[key] - return config - @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') @test_util.patch_get_utility() def test_server_updates(self, _, mock_select, mock_getsave): - config = self.get_config({"disable_renew_updates": False}) - mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater mock_select.return_value = (mock_generic_updater, None) with mock.patch('certbot.main._init_le_client'): - main.renew_cert(config, None, mock.MagicMock()) + main.renew_cert(self.config, None, mock.MagicMock()) self.assertTrue(mock_generic_updater.restart.called) mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, mock.MagicMock()) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): - config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() mock_deployer = self.renew_deployer - updater.run_renewal_deployer(lineage, mock_deployer, config) + updater.run_renewal_deployer(self.config, lineage, mock_deployer) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + @mock.patch("certbot.updater.logger.debug") + def test_updater_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_generic_updaters(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping updaters in dry-run mode.") + + @mock.patch("certbot.updater.logger.debug") + def test_deployer_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_renewal_deployer(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping renewal deployer in dry-run mode.") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py index 1216f38a6..112cf06ef 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -8,21 +8,24 @@ from certbot.plugins import selection as plug_sel logger = logging.getLogger(__name__) -def run_generic_updaters(config, plugins, lineage): +def run_generic_updaters(config, lineage, plugins): """Run updaters that the plugin supports :param config: Configuration object :type config: interfaces.IConfig - :param plugins: List of plugins - :type plugins: `list` of `str` - :param lineage: Certificate lineage object :type lineage: storage.RenewableCert + :param plugins: List of plugins + :type plugins: `list` of `str` + :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping updaters in dry-run mode.") + return try: # installers are used in auth mode to determine domain names installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") @@ -31,10 +34,13 @@ def run_generic_updaters(config, plugins, lineage): return _run_updaters(lineage, installer, config) -def run_renewal_deployer(lineage, installer, config): +def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used installer plugin. + :param config: Configuration object + :type config: interfaces.IConfig + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert @@ -44,6 +50,10 @@ def run_renewal_deployer(lineage, installer, config): :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping renewal deployer in dry-run mode.") + return + if not config.disable_renew_updates and isinstance(installer, interfaces.RenewDeployer): installer.renew_deploy(lineage) From d53ef1f7c24c21d297a23349f3d566e24252b1fb Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 31 May 2018 13:57:23 -0700 Subject: [PATCH 227/639] Add mypy info to Certbot docs (#6033) * Add mypy info to Certbot docs * break up lines * link to mypy docs and links to https * Expand on import wording * be consistent about mypy styling --- docs/contributing.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ed986c562..58db251d4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,40 @@ Please: .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +Mypy type annotations +===================== + +Certbot uses the `mypy`_ static type checker. Python 3 natively supports official type annotations, +which can then be tested for consistency using mypy. Python 2 doesn’t, but type annotations can +be `added in comments`_. Mypy does some type checks even without type annotations; we can find +bugs in Certbot even without a fully annotated codebase. + +Certbot supports both Python 2 and 3, so we’re using Python 2-style annotations. + +Zulip wrote a `great guide`_ to using mypy. It’s useful, but you don’t have to read the whole thing +to start contributing to Certbot. + +To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. + +Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from +``acme.magic_typing`` and have to add some comments for pylint like this: + +.. code-block:: python + + from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module + +Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. +Those imports should look like this: + +.. code-block:: python + + from OpenSSL import crypto + from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 + +.. _mypy: https://mypy.readthedocs.io +.. _added in comments: https://mypy.readthedocs.io/en/latest/cheat_sheet.html +.. _great guide: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/ + Submitting a pull request ========================= From fb0d2ec3d60cf29d2c8aff09a3adbf50ba7f938f Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 1 Jun 2018 18:09:02 -0400 Subject: [PATCH 228/639] Include missing space (#6061) --- certbot-apache/certbot_apache/configurator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb82a9d3f..861fe4458 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -132,10 +132,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): default=cls.OS_DEFAULTS["challenge_location"], help="Directory path for challenge configuration.") add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you." + + help="Let installer handle enabling required modules for you. " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you." + + help="Let installer handle enabling sites for you. " + "(Only Ubuntu/Debian currently)") util.add_deprecated_argument(add, argument_name="ctl", nargs=1) util.add_deprecated_argument( From e2d6faa8a95b05e11369f7ae646fa8216b2b883a Mon Sep 17 00:00:00 2001 From: schoen Date: Fri, 1 Jun 2018 15:21:02 -0700 Subject: [PATCH 229/639] Add --reuse-key feature (#5901) * Initial work on new version of --reuse-key * Test for reuse_key * Make lint happier * Also test a non-dry-run reuse_key renewal * Test --reuse-key in boulder integration test * Better reuse-key integration testing * Log fact that key was reused * Test that the certificates themselves are different * Change "oldkeypath" to "old_keypath" * Simply appearance of new-key generation logic * Reorganize new-key logic * Move awk logic into TotalAndDistinctLines function * After refactor, there's now explicit None rather than missing param * Indicate for MyPy that key can be None * Actually import the Optional type * magic_typing is too magical for pylint * Remove --no-reuse-key option * Correct pylint test disable --- certbot/cli.py | 6 ++++++ certbot/client.py | 32 +++++++++++++++++++++++++++----- certbot/constants.py | 1 + certbot/renewal.py | 7 +++++-- certbot/tests/main_test.py | 24 +++++++++++++++++++++--- tests/boulder-integration.sh | 28 ++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 10 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 25319bbd8..05e316133 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1017,6 +1017,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "certificate already exists for the requested certificate name " "but does not match the requested domains, renew it now, " "regardless of whether it is near expiry.") + helpful.add( + "automation", "--reuse-key", dest="reuse_key", + action="store_true", default=flag_default("reuse_key"), + help="When renewing, use the same private key as the existing " + "certificate.") + helpful.add( ["automation", "renew", "certonly"], "--allow-subset-of-names", action="store_true", diff --git a/certbot/client.py b/certbot/client.py index dadc3a0f8..cc2f31d56 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -4,6 +4,7 @@ import logging import os import platform + from cryptography.hazmat.backends import default_backend # https://github.com/python/typeshed/blob/master/third_party/ # 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi @@ -16,6 +17,7 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import,no-name-in-module import certbot @@ -273,7 +275,7 @@ class Client(object): cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) return cert.encode(), chain.encode() - def obtain_certificate(self, domains): + def obtain_certificate(self, domains, old_keypath=None): """Obtains a certificate from the ACME server. `.register` must be called before `.obtain_certificate` @@ -286,16 +288,36 @@ class Client(object): :rtype: tuple """ + + # We need to determine the key path, key PEM data, CSR path, + # and CSR PEM data. For a dry run, the paths are None because + # they aren't permanently saved to disk. For a lineage with + # --reuse-key, the key path and PEM data are derived from an + # existing file. + + if old_keypath is not None: + # We've been asked to reuse a specific existing private key. + # Therefore, we'll read it now and not generate a new one in + # either case below. + with open(old_keypath, "r") as f: + keypath = old_keypath + keypem = f.read() + key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + logger.info("Reusing existing private key from %s.", old_keypath) + else: + # The key is set to None here but will be created below. + key = None + # Create CSR from names if self.config.dry_run: - key = util.Key(file=None, - pem=crypto_util.make_key(self.config.rsa_key_size)) + key = key or util.Key(file=None, + pem=crypto_util.make_key(self.config.rsa_key_size)) csr = util.CSR(file=None, form="pem", data=acme_crypto_util.make_csr( key.pem, domains, self.config.must_staple)) else: - key = crypto_util.init_save_key( - self.config.rsa_key_size, self.config.key_dir) + key = key or crypto_util.init_save_key(self.config.rsa_key_size, + self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) diff --git a/certbot/constants.py b/certbot/constants.py index 40557d287..1dd25e799 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,6 +64,7 @@ CLI_DEFAULTS = dict( pref_challs=[], validate_hooks=True, directory_hooks=True, + reuse_key=False, disable_renew_updates=False, # Subparsers diff --git a/certbot/renewal.py b/certbot/renewal.py index 236330a85..aa8c9722a 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -36,7 +36,7 @@ STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "pre_hook", "post_hook", "tls_sni_01_address", "http01_address"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names"] +BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key"] CONFIG_ITEMS = set(itertools.chain( BOOL_CONFIG_ITEMS, INT_CONFIG_ITEMS, STR_CONFIG_ITEMS, ('pref_challs',))) @@ -298,7 +298,10 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) + # The private key is the existing lineage private key if reuse_key is set. + # Otherwise, generate a fresh private key by passing None. + new_key = lineage.privkey if config.reuse_key else None + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 36821cf53..4b251c421 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1026,8 +1026,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False, expiry_date=datetime.datetime.now()): - # pylint: disable=too-many-locals,too-many-arguments + quiet_mode=False, expiry_date=datetime.datetime.now(), + reuse_key=False): + # pylint: disable=too-many-locals,too-many-arguments,too-many-branches cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path, @@ -1077,7 +1078,13 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met traceback.format_exc()) if should_renew: - mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) + if reuse_key: + # The location of the previous live privkey.pem is passed + # to obtain_certificate + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], + os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem")) + else: + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None) else: self.assertEqual(mock_client.obtain_certificate.call_count, 0) except: @@ -1127,6 +1134,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + def test_reuse_key(self): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + + @mock.patch('certbot.storage.RenewableCert.save_successor') + def test_reuse_key_no_dry_run(self, unused_save_successor): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + @mock.patch('certbot.renewal.should_renew') def test_renew_skips_recent_certs(self, should_renew): should_renew.return_value = False diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e931e30f3..ef611e743 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -166,6 +166,14 @@ CheckRenewHook() { CheckSavedRenewHook $1 } +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + # Cleanup coverage data coverage erase @@ -347,6 +355,26 @@ if common certificates | grep "fail\.dns1\.le\.wtf"; then exit 1 fi +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ From f19ebab44173b3ed70eb2aa81464fe1aa2fc0122 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Jun 2018 21:08:40 +0300 Subject: [PATCH 230/639] Make sure the pluginstorage file gets truncated when writing to it (#6062) --- certbot/plugins/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index 9472a1ebb..ae3ca1889 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -84,7 +84,8 @@ class PluginStorage(object): raise errors.PluginStorageError(errmsg) try: with os.fdopen(os.open(self._storagepath, - os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fh: + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600), 'w') as fh: fh.write(serialized) except IOError as e: errmsg = "Could not write PluginStorage data to file {0} : {1}".format( From 4151737e171b9a73923d02dc746cb593fb1f6d33 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 13:13:23 -0700 Subject: [PATCH 231/639] Read in bytes to fix --reuse-key on Python 3 (#6069) * Read bytes for now for compatibility * add clarifying comment --- certbot/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index cc2f31d56..d97de0571 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -299,7 +299,10 @@ class Client(object): # We've been asked to reuse a specific existing private key. # Therefore, we'll read it now and not generate a new one in # either case below. - with open(old_keypath, "r") as f: + # + # We read in bytes here because the type of `key.pem` + # created below is also bytes. + with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] From 15f1405fff7083bf5d4f599a58c54a43be499740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20D=C4=99bski?= Date: Mon, 4 Jun 2018 23:54:17 +0200 Subject: [PATCH 232/639] Implement TLS-ALPN-01 challenge and standalone TLS-ALPN server (#5894) The new challenge is described in https://github.com/rolandshoemaker/acme-tls-alpn. * TLS-ALPN tests * Implement TLS-ALPN challenge * Skip TLS-ALPN tests on old pyopenssl * make _selection methods private. --- acme/acme/challenges.py | 152 +++++++++++++++++++++++++++- acme/acme/challenges_test.py | 121 ++++++++++++++++++++++ acme/acme/crypto_util.py | 61 ++++++++--- acme/acme/crypto_util_test.py | 16 ++- acme/acme/standalone.py | 48 ++++++++- acme/acme/standalone_test.py | 57 +++++++++++ acme/acme/testdata/README | 6 +- acme/acme/testdata/rsa1024_cert.pem | 13 +++ 8 files changed, 453 insertions(+), 21 deletions(-) create mode 100644 acme/acme/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 674f2c38f..ce788e2cc 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,5 +1,6 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging @@ -7,7 +8,7 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import OpenSSL +from OpenSSL import crypto import requests import six @@ -411,8 +412,8 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ if key is None: - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) return crypto_util.gen_ss_cert(key, [ # z_domain is too big to fit into CN, hence first dummy domain 'dummy', self.z_domain.decode()], force_san=True), key @@ -507,6 +508,151 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) +@ChallengeResponse.register +class TLSALPN01Response(KeyAuthorizationChallengeResponse): + """ACME tls-alpn-01 challenge response.""" + typ = "tls-alpn-01" + + PORT = 443 + """Verification port as defined by the protocol. + + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` iff client's control of the domain has been + verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(cert, domain) + + +@Challenge.register # pylint: disable=too-many-ancestors +class TLSALPN01(KeyAuthorizationChallenge): + """ACME tls-alpn-01 challenge.""" + response_cls = TLSALPN01Response + typ = response_cls.typ + + def validation(self, account_key, **kwargs): + """Generate validation. + + :param JWK account_key: + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) + + @Challenge.register # pylint: disable=too-many-ancestors class DNS(_TokenChallenge): """ACME "dns" challenge.""" diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 834d569aa..b929d4939 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -393,6 +393,127 @@ class TLSSNI01Test(unittest.TestCase): mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) +class TLSALPN01ResponseTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) + self.jmsg = { + 'resource': 'challenge', + 'type': 'tls-alpn-01', + 'keyAuthorization': self.response.key_authorization, + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.response.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01Response + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01Response + hash(TLSALPN01Response.from_json(self.jmsg)) + + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, mock.sentinel.cert, self.domain) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + + +class TLSALPN01Test(unittest.TestCase): + + def setUp(self): + from acme.challenges import TLSALPN01 + self.msg = TLSALPN01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) + self.jmsg = { + 'type': 'tls-alpn-01', + 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.msg.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01 + self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01 + hash(TLSALPN01.from_json(self.jmsg)) + + def test_from_json_invalid_token_length(self): + from acme.challenges import TLSALPN01 + self.jmsg['token'] = jose.encode_b64jose(b'abcd') + self.assertRaises( + jose.DeserializationError, TLSALPN01.from_json, self.jmsg) + + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + + class DNSTest(unittest.TestCase): def setUp(self): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d0e203417..d25c2340b 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) _DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. @@ -38,12 +47,25 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_TLSSNI01_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -60,18 +82,19 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -96,6 +119,8 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -111,8 +136,9 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -124,6 +150,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -160,6 +188,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -251,12 +281,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -269,10 +301,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 36d62b324..b661e4e70 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -19,7 +19,6 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -34,7 +33,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -66,6 +66,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ff9159933..3bcb0b230 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -43,7 +43,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): # pylint: disable=missing-docstring self._wrap_sock() @@ -147,6 +154,45 @@ class TLSSNI01DualNetworkedServers(BaseDualNetworkedServers): BaseDualNetworkedServers.__init__(self, TLSSNI01Server, *args, **kwargs) +class BadALPNProtos(Exception): + """Error raised when cannot negotiate ALPN protocol.""" + pass + + +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + # Raising an exception causes openssl to terminate handshake and + # send fatal tls alert. + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + raise BadALPNProtos("Got: %s" % str(alpn_protos)) + + class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): """BaseRequestHandler with logging.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 1591187e5..aee592187 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -10,6 +10,7 @@ import unittest from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose import mock import requests @@ -119,6 +120,62 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@unittest.skipUnless( + hasattr(SSL.Connection, "set_alpn_protos") and + hasattr(SSL.Context, "set_alpn_select_callback"), + "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + #TODO: This is not implemented yet, see comments in standalone.py + #def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" diff --git a/acme/acme/testdata/README b/acme/acme/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/acme/testdata/README +++ b/acme/acme/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/acme/testdata/rsa1024_cert.pem b/acme/acme/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/acme/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- From 236f9630e074ad45aa2b834483cbf373aae18566 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 15:04:56 -0700 Subject: [PATCH 233/639] Remove unneeded sys import (#5873) * Remove unneeded sys import. Once upon a time we needed this in some of these setup.py files because we were using sys in the file, but we aren't anymore so let's remove the import. * use setuptools instead of distutils --- acme/setup.py | 2 -- certbot-apache/setup.py | 2 -- certbot-dns-cloudflare/setup.py | 2 -- certbot-dns-cloudxns/setup.py | 2 -- certbot-dns-digitalocean/setup.py | 2 -- certbot-dns-dnsimple/setup.py | 2 -- certbot-dns-dnsmadeeasy/setup.py | 2 -- certbot-dns-google/setup.py | 2 -- certbot-dns-luadns/setup.py | 2 -- certbot-dns-nsone/setup.py | 2 -- certbot-dns-rfc2136/setup.py | 2 -- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 2 -- letshelp-certbot/setup.py | 2 -- setup.py | 1 - 15 files changed, 1 insertion(+), 30 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e91c36b3d..f8196e953 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 0e4304300..59ca3ed4a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 8e1f9d28b..d80a1ae20 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 05998ee6a..3bb7457e3 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index cd3b0613e..1d7c34197 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 10ee710cd..9c678ec60 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a7f44b989..66f3edd55 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index c171e5014..64b332b04 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2c0e35308..8df4f1735 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 821a40655..4425ace16 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 21d9dec29..6ced18b5d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 083cd15ae..9ee587c4e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,6 +1,4 @@ -import sys - -from distutils.core import setup +from setuptools import setup from setuptools import find_packages version = '0.25.0.dev0' diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6889d06e4..067f280d4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b5be07a59..28ce0e962 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/setup.py b/setup.py index ee0470d3a..e32925583 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ import codecs import os import re -import sys from setuptools import setup from setuptools import find_packages From 8e4303af9f30f514fce04b88bff70b6bbb9d43c8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 4 Jun 2018 16:04:47 -0700 Subject: [PATCH 234/639] Reuse ACMEv1 accounts for ACMEv2 (#5902) * Reuse ACMEv1 accounts for ACMEv2 * Correct behavior * add unit tests * add _find_all_inner to comply with interface * acme-staging-v01 --> acme-staging * only create symlink to previous account if there is one there * recurse on server path * update tests and change internal use of load to use server_path * fail gracefully on corrupted account file by returning [] when rmdir fails * only reuse accounts in staging for now --- certbot/account.py | 42 ++++++++++++++++++++---- certbot/configuration.py | 6 +++- certbot/constants.py | 6 ++++ certbot/tests/account_test.py | 62 ++++++++++++++++++++++++++++++++++- 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index 70d9a7fc3..c4eeb1388 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -16,6 +16,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import constants from certbot import errors from certbot import interfaces from certbot import util @@ -142,7 +143,11 @@ class AccountFileStorage(interfaces.AccountStorage): self.config.strict_permissions) def _account_dir_path(self, account_id): - return os.path.join(self.config.accounts_dir, account_id) + return self._account_dir_path_for_server_path(account_id, self.config.server_path) + + def _account_dir_path_for_server_path(self, account_id, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + return os.path.join(accounts_dir, account_id) @classmethod def _regr_path(cls, account_dir_path): @@ -156,22 +161,44 @@ class AccountFileStorage(interfaces.AccountStorage): def _metadata_path(cls, account_dir_path): return os.path.join(account_dir_path, "meta.json") - def find_all(self): + def _find_all_for_server_path(self, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) try: - candidates = os.listdir(self.config.accounts_dir) + candidates = os.listdir(accounts_dir) except OSError: return [] accounts = [] for account_id in candidates: try: - accounts.append(self.load(account_id)) + accounts.append(self._load_for_server_path(account_id, server_path)) except errors.AccountStorageError: logger.debug("Account loading problem", exc_info=True) + + + if not accounts and server_path in constants.LE_REUSE_SERVERS: + # find all for the next link down + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_accounts = self._find_all_for_server_path(prev_server_path) + # if we found something, link to that + if prev_accounts: + if os.path.islink(accounts_dir): + os.unlink(accounts_dir) + else: + try: + os.rmdir(accounts_dir) + except OSError: + return [] + prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) + os.symlink(prev_account_dir, accounts_dir) + accounts = prev_accounts return accounts - def load(self, account_id): - account_dir_path = self._account_dir_path(account_id) + def find_all(self): + return self._find_all_for_server_path(self.config.server_path) + + def _load_for_server_path(self, account_id, server_path): + account_dir_path = self._account_dir_path_for_server_path(account_id, server_path) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) @@ -193,6 +220,9 @@ class AccountFileStorage(interfaces.AccountStorage): account_id, acc.id)) return acc + def load(self, account_id): + return self._load_for_server_path(account_id, self.config.server_path) + def save(self, account, acme): self._save(account, acme, regr_only=False) diff --git a/certbot/configuration.py b/certbot/configuration.py index 297795609..daf514be8 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -65,8 +65,12 @@ class NamespaceConfig(object): @property def accounts_dir(self): # pylint: disable=missing-docstring + return self.accounts_dir_for_server_path(self.server_path) + + def accounts_dir_for_server_path(self, server_path): + """Path to accounts directory based on server_path""" return os.path.join( - self.namespace.config_dir, constants.ACCOUNTS_DIR, self.server_path) + self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) @property def backup_dir(self): # pylint: disable=missing-docstring diff --git a/certbot/constants.py b/certbot/constants.py index 1dd25e799..c6746e888 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -158,6 +158,12 @@ CONFIG_DIRS_MODE = 0o755 ACCOUNTS_DIR = "accounts" """Directory where all accounts are saved.""" +LE_REUSE_SERVERS = { + 'acme-staging-v02.api.letsencrypt.org/directory': + 'acme-staging.api.letsencrypt.org/directory' +} +"""Servers that can reuse accounts from other servers.""" + BACKUP_DIR = "backups" """Directory (relative to `IConfig.work_dir`) where backups are kept.""" diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ebda56af..a8059fbcf 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -95,6 +95,7 @@ class AccountMemoryStorageTest(unittest.TestCase): class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot.account.AccountFileStorage.""" + #pylint: disable=too-many-public-methods def setUp(self): super(AccountFileStorageTest, self).setUp() @@ -159,7 +160,8 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertEqual([], self.storage.find_all()) def test_find_all_load_skips(self): - self.storage.load = mock.MagicMock( + # pylint: disable=protected-access + self.storage._load_for_server_path = mock.MagicMock( side_effect=["x", errors.AccountStorageError, "z"]) with mock.patch("certbot.account.os.listdir") as mock_listdir: mock_listdir.return_value = ["x", "y", "z"] @@ -175,6 +177,64 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.AccountStorageError, self.storage.load, "x" + self.acc.id) + def _set_server(self, server): + self.config.server = server + from certbot.account import AccountFileStorage + self.storage = AccountFileStorage(self.config) + + def test_find_all_neither_exists(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.assertEqual([], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + + def test_find_all_find_before_save(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + # we shouldn't have created a v1 account + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_save_before_find(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + self.assertTrue(os.path.isdir(self.config.accounts_dir)) + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_server_downgrade(self): + # don't use v2 accounts with a v1 url + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + + def test_upgrade_version(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([self.acc], self.storage.find_all()) + + @mock.patch('os.rmdir') + def test_corrupted_account(self, mock_rmdir): + # pylint: disable=protected-access + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + mock_rmdir.side_effect = OSError + self.storage._load_for_server_path = mock.MagicMock( + side_effect=errors.AccountStorageError) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() From 09a28c7a27fd01d1517e6381ffb37501bffca292 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 4 Jun 2018 17:44:51 -0700 Subject: [PATCH 235/639] Allow multiple add_headers directives (#6068) * fix(nginx-hsts): allow multiple add_headers * test(nginx): fix nginx tests --- certbot-nginx/certbot_nginx/parser.py | 2 +- .../certbot_nginx/tests/configurator_test.py | 15 ++++++++++++--- certbot-nginx/certbot_nginx/tests/parser_test.py | 5 +++-- .../testdata/etc_nginx/sites-enabled/headers.com | 4 ++++ 4 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 5bc7946dc..7d1da2e73 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -566,7 +566,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e88dcb8e0..0668a38c9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -47,7 +47,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(10, len(self.config.parser.parsed)) + self.assertEqual(11, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -91,7 +91,8 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) + "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", + "headers.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -548,6 +549,14 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[example_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_multiple_headers_hsts(self): + headers_conf = self.config.parser.abs_path('sites-enabled/headers.com') + self.config.enhance("headers.com", "ensure-http-header", + "Strict-Transport-Security") + expected = ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always'] + generated_conf = self.config.parser.parsed[headers_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_http_header_hsts_twice(self): self.config.enhance("www.example.com", "ensure-http-header", "Strict-Transport-Security") @@ -852,7 +861,7 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 5) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 5a37c9565..f6f28e42b 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -49,6 +49,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', 'sites-enabled/example.com', + 'sites-enabled/headers.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', 'sites-enabled/globalssl.com', @@ -77,7 +78,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(7, len( + self.assertEqual(8, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -160,7 +161,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(12, len(vhosts)) + self.assertEqual(13, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com new file mode 100644 index 000000000..6c032928c --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com @@ -0,0 +1,4 @@ +server { + server_name headers.com; + add_header X-Content-Type-Options nosniff; +} From d905886f4c63d1546ff723a909a088ab9334b91f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 5 Jun 2018 13:40:48 -0700 Subject: [PATCH 236/639] Automatically select among default vhosts if we have a port preference in nginx (#5944) * automatically select among default vhosts if we have a port preference * ports should be strings in the nginx plugin * clarify port vs preferred_port behavior by adding allow_port_mismatch flag * update all instances of default_vhosts to all_default_vhosts * require port * port should never be None in _get_default_vhost --- certbot-nginx/certbot_nginx/configurator.py | 31 ++++++++++++------- .../certbot_nginx/tests/configurator_test.py | 7 +++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 293f7378e..41b5124b8 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -289,7 +289,8 @@ class NginxConfigurator(common.Installer): if not vhosts: if create_if_no_match: # result will not be [None] because it errors on failure - vhosts = [self._vhost_from_duplicated_default(target_name)] + vhosts = [self._vhost_from_duplicated_default(target_name, True, + str(self.config.tls_sni_01_port))] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -332,9 +333,12 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, port=None): + def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + """if allow_port_mismatch is False, only server blocks with matching ports will be + used as a default server block template. + """ if self.new_vhost is None: - default_vhost = self._get_default_vhost(port, domain) + default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,19 +354,24 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port, domain): + def _get_default_vhost(self, domain, allow_port_mismatch, port): + """Helper method for _vhost_from_duplicated_default; see argument documentation there""" vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one - default_vhosts = [] + all_default_vhosts = [] + port_matching_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - if port is None or self._port_matches(port, addr.get_port()): - default_vhosts.append(vhost) - break + all_default_vhosts.append(vhost) + if self._port_matches(port, addr.get_port()): + port_matching_vhosts.append(vhost) + break - if len(default_vhosts) == 1: - return default_vhosts[0] + if len(port_matching_vhosts) == 1: + return port_matching_vhosts[0] + elif len(all_default_vhosts) == 1 and allow_port_mismatch: + return all_default_vhosts[0] # TODO: present a list of vhosts for user to choose from @@ -471,7 +480,7 @@ class NginxConfigurator(common.Installer): matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts def _port_matches(self, test_port, matching_port): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 0668a38c9..9386c3cd9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -731,6 +731,13 @@ class NginxConfiguratorTest(util.NginxTest): "www.nomatch.com", "example/cert.pem", "example/key.pem", "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_multiple_defaults_ok(self): + foo_conf = self.config.parser.abs_path('foo.conf') + self.config.parser.parsed[foo_conf][2][1][0][1][0][1] = '*:5001' + self.config.version = (1, 3, 1) + self.config.deploy_cert("www.nomatch.com", "example/cert.pem", "example/key.pem", + "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_add_redirect(self): default_conf = self.config.parser.abs_path('sites-enabled/default') foo_conf = self.config.parser.abs_path('foo.conf') From 868e5b831b27ac07cb4a73e04ed9e6fac5987fc7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Jun 2018 17:59:11 -0700 Subject: [PATCH 237/639] Make python setup.py test use pytest for acme (#6072) --- acme/setup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index f8196e953..0fb4d8fff 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,6 +1,7 @@ from setuptools import setup from setuptools import find_packages - +from setuptools.command.test import test as TestCommand +import sys version = '0.25.0.dev0' @@ -33,6 +34,19 @@ docs_extras = [ 'sphinx_rtd_theme', ] +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) setup( name='acme', @@ -65,5 +79,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, + tests_require=["pytest"], test_suite='acme', + cmdclass={"test": PyTest}, ) From 3cffe1449c4e9166b65eaed75022d73b7ad79328 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 07:58:50 -0700 Subject: [PATCH 238/639] Revert "switch signature verification to use pure cryptography (#6000)" (#6074) This reverts commit 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2. --- certbot/crypto_util.py | 36 +++++-------------- certbot/tests/crypto_util_test.py | 10 ------ .../tests/testdata/cert-nosans_nistp256.pem | 11 ------ .../tests/testdata/csr-nosans_nistp256.pem | 8 ----- certbot/tests/testdata/nistp256_key.pem | 5 --- 5 files changed, 9 insertions(+), 61 deletions(-) delete mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 71f6c990c..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,16 +12,11 @@ import os import pyrfc3339 import six import zope.component -from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA -from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey -from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 -from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore +from cryptography.hazmat.backends import default_backend +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -233,26 +228,13 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) + chain, _ = pyopenssl_load_certificate(chain_file.read()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) - pk = chain.public_key() - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") - except (IOError, ValueError, InvalidSignature) as e: + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) + hash_name = cert.signature_hash_algorithm.name + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index baf14b2ef..2fe0e3d30 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,9 +21,6 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') -P256_KEY = test_util.load_vector('nistp256_key.pem') -P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') -P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -220,13 +217,6 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) - def test_cert_sig_match_ec(self): - renewable_cert = mock.MagicMock() - renewable_cert.cert = P256_CERT_PATH - renewable_cert.chain = P256_CERT_PATH - renewable_cert.privkey = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) - def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem deleted file mode 100644 index 4ec3f24ce..000000000 --- a/certbot/tests/testdata/cert-nosans_nistp256.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER -MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD -RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 -MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG -A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY -QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK -BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O -Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM ------END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem deleted file mode 100644 index 2f0a671ed..000000000 --- a/certbot/tests/testdata/csr-nosans_nistp256.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ -BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl -LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw -GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg -ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ -A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== ------END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem deleted file mode 100644 index 4be37e49b..000000000 --- a/certbot/tests/testdata/nistp256_key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 -AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ -pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== ------END EC PRIVATE KEY----- From 4ae2390c441f5e2e276ffb0460f097af1c325846 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 13:50:30 -0700 Subject: [PATCH 239/639] Release 0.25.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 47 +++++++++++------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 29 +++++++---- letsencrypt-auto | 47 +++++++++++------- letsencrypt-auto-source/certbot-auto.asc | 16 +++--- letsencrypt-auto-source/letsencrypt-auto | 26 +++++----- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++----- 22 files changed, 124 insertions(+), 95 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0fb4d8fff..e24f4297b 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.25.0.dev0' +version = '0.25.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 59ca3ed4a..18e10223d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 0848080b3..c15a851db 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ @@ -1187,6 +1193,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # @@ -1199,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 50df2a56e..e739c4924 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d80a1ae20..861ad12d7 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3bb7457e3..52aebb374 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1d7c34197..c32a2d003 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9c678ec60..026eb3ed2 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 66f3edd55..799163d1d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 64b332b04..ef4ae0f8c 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 8df4f1735..838d323af 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4425ace16..75d3c25e4 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 6ced18b5d..6042365e7 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 9ee587c4e..b6e8293b0 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 067f280d4..be057a928 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.25.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 27c63e266..85b4d9fd8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.25.0.dev0' +__version__ = '0.25.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 259142e62..f40a9aa4c 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,9 +108,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.24.0 (certbot; - darwin 10.13.4) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags + "". (default: CertbotACMEClient/0.25.0 (certbot; + darwin 10.13.5) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -143,6 +143,8 @@ automation: certificate name but does not match the requested domains, renew it now, regardless of whether it is near expiry. (default: False) + --reuse-key When renewing, use the same private key as the + existing certificate. (default: False) --allow-subset-of-names When performing domain validation, do not consider it a failure if authorizations can not be obtained for a @@ -319,6 +321,13 @@ renew: disable it. (default: False) --no-directory-hooks Disable running executables found in Certbot's hook directories during renewal. (default: False) + --disable-renew-updates + Disable automatic updates to your server configuration + that would otherwise be done by the selected installer + plugin, and triggered when the user executes "certbot + renew", regardless of if the certificate is renewed. + This setting does not apply to important TLS + configuration updates. (default: False) certificates: List certificates managed by Certbot @@ -360,8 +369,9 @@ register: e-mail address, should be updated, rather than registering a new account. (default: False) -m EMAIL, --email EMAIL - Email used for registration and recovery contact. - (default: Ask) + Email used for registration and recovery contact. Use + comma to register multiple emails, ex: + u1@example.com,u2@example.com. (default: Ask). --eff-email Share your e-mail address with EFF (default: None) --no-eff-email Don't share your e-mail address with EFF (default: None) @@ -399,7 +409,7 @@ update_symlinks: changed them by hand or edited a renewal configuration file enhance: - Helps to harden the TLS configration by adding security enhancements to + Helps to harden the TLS configuration by adding security enhancements to already existing configuration. plugins: @@ -472,9 +482,9 @@ apache: /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: False) + you. (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES - Let installer handle enabling sites for you.(Only + Let installer handle enabling sites for you. (Only Ubuntu/Debian currently) (default: False) certbot-route53:auth: @@ -628,7 +638,8 @@ nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT - Nginx server root directory. (default: /etc/nginx) + Nginx server root directory. (default: + /usr/local/etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and retrieving nginx version number. (default: nginx) diff --git a/letsencrypt-auto b/letsencrypt-auto index 0848080b3..c15a851db 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ @@ -1187,6 +1193,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # @@ -1199,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 641ebaef8..11e9750b5 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlro/1AACgkQTRfJlc2X -dfLm5ggAxCrWU9dmYZKllcFzp7TFOdRap0pmarfL4gwSYj7B/bSceD7ysOyoQ8Ra -7UHuZKAQyurZn1seN49d88Kgor9KWZQ1jZiGkfiEpp8qAkdWzFR8UqYa2/CZtk2l -bExm8YQDwhuKvCObGLDGi3ydcIQpfg/rsBkSTphKYXN/Zebx9mAelZN4CgGRy03Y -3z2UqqnyqFPAg4wUGcNfCgUEbJ5bUPr733vQzjBS2IVUbDbu06/1Y8oYzurezXNS -6lEyvTfC5G8RGlSWupNu7yWviD14M4LnAo6WXWEVH+C+ssJaPrZVhZ6KfEt/Erg3 -k06WZSPDCtOm5EfhDm0Rumqm1owA2g== -=Bc4G +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsYSPUACgkQTRfJlc2X +dfKOzQgAhyWglFSdc2VTdqL5FDkg9yv5HzlpwUKp+Q3Q0Tq7/fecZeMybUnj8aJZ +6kiIQ0TE7vlQiKknxCg883hjoW/g0ZMemHsyVIAB6yi69Xltf/maxwwVdDPd+ens +db3/mRiefW+WE2tf5xPKc5xcch1Ej0H9bTIOyj7sgod/bFMuLEtyT/Y58Sb5gWIK +hIrfpWl0L3IP1EAFGSTbRhhxDPsMI6jveHMAQdh5uYgvDneUMVDwcuF4HW+qpvxG +lVQJqDTCSYpIg2bpzI8KHqsiHoe37Au5KbxewPb7Mmx91QE9u3deuPLM/jutHqQ7 +kGs5isnImrtQODSfAnWQbkq9BQocdA== +=EBXk -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 28281e20d..c15a851db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0.dev0" +LE_AUTO_VERSION="0.25.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.24.0 \ - --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ - --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 -acme==0.24.0 \ - --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ - --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb -certbot-apache==0.24.0 \ - --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ - --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 -certbot-nginx==0.24.0 \ - --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ - --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 +certbot==0.25.0 \ + --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ + --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 +acme==0.25.0 \ + --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ + --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 +certbot-apache==0.25.0 \ + --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ + --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 +certbot-nginx==0.25.0 \ + --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ + --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4a937e7e0c442056410a4d75bc6fc64fa7fc5ed8..12330ad162164927502aadb9e73375d8a40b3d1a 100644 GIT binary patch literal 256 zcmV+b0ssC$@8wI45>gS1>Vf-0?P)E}7cN~xWB;XyfZbYHf67R;8??@TB|A<6&Ymtv za~(Bb7VdlqKAz=5dK^kLhO%Pu%mcZ{cVC7weDiho<&%Nz(;*Xb*c(;*W-^&XC=|OD_~8fI^IFAUF~p|$&jhTt;E~D3eY}R9 z()EQT9^Wu3iJV-m%DTGS!1TdJ#YvLh-mJo<+p^7z6cnVGbv2Zai!|_#2965E&iQo& zOyxwu*TNkoH+8^qQvf_x;y1Sb&ZX_LL&2M~e^bphBFP!(!0Q2!3tl{Y_@sRtrU<`!&JJ7Yet?gj zP<-mLp*eOt8&aten_HT4Xaq=Kx#zQxgGH^wqLqmDbiNH^#Z>!uQ#lxbz9TmzQ``gA zepS_2=ZD+oI|Kl|@HhjznV9>uJqAGPzsGZ`_Tr$cOh5JZM+hOCqkF1gQK5rq?01Ha z$#L4=i}~HfWo>cf%K<$CERWB0i$y+n$+{zxu0TOGArrjm@c)#iwau&wz!kfWn^wMB GGm3kHFn Date: Wed, 6 Jun 2018 13:50:46 -0700 Subject: [PATCH 240/639] Bump version to 0.26.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e24f4297b..ecafac61a 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.25.0' +version = '0.26.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 18e10223d..7a0dc43bf 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e739c4924..bf86df5da 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 861ad12d7..055a8cffc 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 52aebb374..2c0c074be 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c32a2d003..bd5f2ff26 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 026eb3ed2..581c478b8 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 799163d1d..c5d310d71 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ef4ae0f8c..01d979c2b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 838d323af..44f67c0a7 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 75d3c25e4..87a7da4cc 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 6042365e7..c1fffc4a2 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index b6e8293b0..8e2821332 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index be057a928..3de257a5f 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0' +version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 85b4d9fd8..3ae0e315b 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.25.0' +__version__ = '0.26.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c15a851db..cac73dcbd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.26.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From eec37f65a83ccf185a67378b6b8010b37f4fba1f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 19:01:55 -0700 Subject: [PATCH 241/639] Update changelog for 0.25.0 (#6076) --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 044e55250..5facc5380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,70 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.25.0 - 2018-06-06 + +### Added + +* Support for the ready status type was added to acme. Without this change, + Certbot and acme users will begin encountering errors when using Let's + Encrypt's ACMEv2 API starting on June 19th for the staging environment and + July 5th for production. See + https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more + information. +* Certbot now accepts the flag --reuse-key which will cause the same key to be + used in the certificate when the lineage is renewed rather than generating a + new key. +* You can now add multiple email addresses to your ACME account with Certbot by + providing a comma separated list of emails to the --email flag. +* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme. + For more information, see + https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1. +* acme now supports specifying the source address to bind to when sending + outgoing connections. You still cannot specify this address using Certbot. +* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't + already have an account registered at that server URL, Certbot will + automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint + if it exists. +* Interfaces were added to Certbot allowing plugins to be called at additional + points. The `GenericUpdater` interface allows plugins to perform actions + every time `certbot renew` is run, regardless of whether any certificates are + due for renewal, and the `RenewDeployer` interface allows plugins to perform + actions when a certificate is renewed. See `certbot.interfaces` for more + information. + +### Changed + +* When running Certbot with --dry-run and you don't already have a staging + account, the created account does not contain an email address even if one + was provided to avoid expiration emails from Let's Encrypt's staging server. +* certbot-nginx does a better job of automatically detecting the location of + Nginx's configuration files when run on BSD based systems. +* acme now requires and uses pytest when running tests with setuptools with + `python setup.py test`. +* `certbot config_changes` no longer waits for user input before exiting. + +### Fixed + +* Misleading log output that caused users to think that Certbot's standalone + plugin failed to bind to a port when performing a challenge has been + corrected. +* An issue where certbot-nginx would fail to enable HSTS if the server block + already had an `add_header` directive has been resolved. +* certbot-nginx now does a better job detecting the server block to base the + configuration for TLS-SNI challenges on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with functional changes were: + +* acme +* certbot +* certbot-apache +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/54?closed=1 + ## 0.24.0 - 2018-05-02 ### Added From da6320f4d1b816592991f144caa4d6197faafd14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:11:06 -0700 Subject: [PATCH 242/639] Stop testing against Debian 7. (#6077) Debian Wheezy is no longer supported (see https://wiki.debian.org/LTS) and Amazon shut down their Debian 7 mirrors so let's stop trying to use Debian 7 during testing. --- tests/letstest/targets.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 9c1aca24e..766b4ea09 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -22,24 +22,6 @@ targets: # #cloud-init # runcmd: # - [ apt-get, install, -y, curl ] - - ami: ami-e0efab88 - name: debian7.8.aws.1 - type: ubuntu - virt: hvm - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] - - ami: ami-e6eeaa8e - name: debian7.8.aws.1_32bit - type: ubuntu - virt: pv - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-60b6c60a From 780a1b3a26192f06c58fce157db5c2a32ecbdc23 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:43:45 -0700 Subject: [PATCH 243/639] Don't require festival during signing. (#6079) Festival isn't available via Homebrew and is only needed to read the hash aloud, so let's not make it a strict requirement that it's installed. You can simply read the hash from the terminal instead. --- tools/offline-sigrequest.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh index 7706796ef..6443ae8af 100755 --- a/tools/offline-sigrequest.sh +++ b/tools/offline-sigrequest.sh @@ -2,17 +2,17 @@ set -o errexit -if ! `which festival > /dev/null` ; then - echo Please install \'festival\'! - exit 1 -fi - function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do - cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ - echo -n '(SayText "'; \ - sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ - echo '")' ) | festival + if ! `which festival > /dev/null` ; then + echo \`festival\` is not installed! + echo Please install it to read the hash aloud + else + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ + echo -n '(SayText "'; \ + sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + echo '")' ) | festival + fi done echo 'Paste in the data from the QR code, then type Ctrl-D:' From 3a8de6d172d3d91348cb4b817586869a756cc2a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 07:50:36 -0700 Subject: [PATCH 244/639] Upgrade pinned twine version. (#6078) For the past couple of releases, twine has errored while trying to upload packages and this is fixed by upgrading to a newer version of twine. This commit updates our pinned version installed when using tools/venv.sh to the latest available version. pkginfo had to be upgraded as well to support the latest version of twine. --- tools/dev_constraints.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d965d4470..777222ffb 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -36,7 +36,7 @@ oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkginfo==1.4.1 +pkginfo==1.4.2 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 @@ -66,7 +66,7 @@ tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 -twine==1.9.1 +twine==1.11.0 typed-ast==1.1.0 typing==3.6.4 uritemplate==0.6 From afa7e3fb8266fb6af7c36d46eb8ee20e844d8107 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 14:45:23 -0700 Subject: [PATCH 245/639] Unrevert #6000 and silence deprecation warnings (#6082) * Revert "Revert "switch signature verification to use pure cryptography (#6000)" (#6074)" This reverts commit 3cffe1449c4e9166b65eaed75022d73b7ad79328. * Fixes #6073. This silences the deprecation warnings from cryptography. I looked into only silencing the cryptography warning specifically in the function, however, CryptographyDeprecationWarning doesn't seem to be publicly documented, so we probably shouldn't depend on it. --- certbot/crypto_util.py | 40 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 +++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 +++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 ++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..943e2c87f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -7,16 +7,21 @@ import hashlib import logging import os - +import warnings import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,28 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY----- From da028ca9c27acf31a6266ae5c0abe53922da86b4 Mon Sep 17 00:00:00 2001 From: Roland Bracewell Shoemaker Date: Mon, 11 Jun 2018 11:59:57 -0700 Subject: [PATCH 246/639] Wrap TLS-ALPN extension with ASN.1 (#6089) * Wrap TLS-ALPN extension with ASN.1 * Fix test --- acme/acme/challenges.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index ce788e2cc..30983e28f 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -546,7 +546,9 @@ class TLSALPN01Response(KeyAuthorizationChallengeResponse): key.generate_key(crypto.TYPE_RSA, bits) - der_value = b"DER:" + codecs.encode(self.h, 'hex') + # Instead of using a ASN.1 encoding library just append the OCTET STRING tag (0x04) + # and the length of the SHA256 hash (0x20) since both of these should never change + der_value = b"DER:0420" + codecs.encode(self.h, 'hex') acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, critical=True, value=der_value) @@ -592,7 +594,8 @@ class TLSALPN01Response(KeyAuthorizationChallengeResponse): # way to get full OID of an unknown extension from pyopenssl. if ext.get_short_name() == b'UNDEF': data = ext.get_data() - return data == self.h + # Add the ASN.1 tag/length prefix to the hash before comparison + return data == b'\x04\x20' + self.h return False From 95892cd4ab57636f39dfd5b31b69b810f42d7481 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jun 2018 17:24:19 -0700 Subject: [PATCH 247/639] Require acme>=0.25.0 for nginx (#6099) --- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 65f5a758e..e70ac0c7f 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 -e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3de257a5f..d9cb4a9c2 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -7,10 +7,7 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - # This plugin works with an older version of acme, but Certbot does not. - # 0.22.0 is specified here to work around - # https://github.com/pypa/pip/issues/988. - 'acme>0.21.1', + 'acme>=0.25.0', 'certbot>0.21.1', 'mock', 'PyOpenSSL', From 9f20fa0ef969811803042be4a60f8127f6883e34 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jun 2018 17:31:22 -0700 Subject: [PATCH 248/639] Fixes #6085. (#6091) The value of norecusedirs is the default in newer versions of pytest which is listed at https://docs.pytest.org/en/3.0.0/customize.html#confval-norecursedirs. --- acme/MANIFEST.in | 1 + acme/pytest.ini | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 acme/pytest.ini diff --git a/acme/MANIFEST.in b/acme/MANIFEST.in index 5367e484a..1619bef69 100644 --- a/acme/MANIFEST.in +++ b/acme/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE.txt include README.rst +include pytest.ini recursive-include docs * recursive-include examples * recursive-include acme/testdata * diff --git a/acme/pytest.ini b/acme/pytest.ini new file mode 100644 index 000000000..0c07ceac7 --- /dev/null +++ b/acme/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = .* build dist CVS _darcs {arch} *.egg From c9ae365f6678ae64134a9408185601e124cdf296 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jun 2018 14:20:15 -0700 Subject: [PATCH 249/639] 0.25.1 update for master (#6110) * Release 0.25.1 (cherry picked from commit 21b5e4eadb445d0a3dcd8cebb709a9fd15e18278) * Bump version to 0.26.0 --- certbot-auto | 26 +++++++++--------- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/certbot-auto b/certbot-auto index c15a851db..d2cfa672d 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.25.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f40a9aa4c..49821a7b5 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.0 (certbot; + "". (default: CertbotACMEClient/0.25.1 (certbot; darwin 10.13.5) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags encoded in the user agent are: --duplicate, --force- diff --git a/letsencrypt-auto b/letsencrypt-auto index c15a851db..d2cfa672d 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.0" +LE_AUTO_VERSION="0.25.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 11e9750b5..67bab66d4 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsYSPUACgkQTRfJlc2X -dfKOzQgAhyWglFSdc2VTdqL5FDkg9yv5HzlpwUKp+Q3Q0Tq7/fecZeMybUnj8aJZ -6kiIQ0TE7vlQiKknxCg883hjoW/g0ZMemHsyVIAB6yi69Xltf/maxwwVdDPd+ens -db3/mRiefW+WE2tf5xPKc5xcch1Ej0H9bTIOyj7sgod/bFMuLEtyT/Y58Sb5gWIK -hIrfpWl0L3IP1EAFGSTbRhhxDPsMI6jveHMAQdh5uYgvDneUMVDwcuF4HW+qpvxG -lVQJqDTCSYpIg2bpzI8KHqsiHoe37Au5KbxewPb7Mmx91QE9u3deuPLM/jutHqQ7 -kGs5isnImrtQODSfAnWQbkq9BQocdA== -=EBXk +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsgc/cACgkQTRfJlc2X +dfLjBgf/bHZn/q+Dqn34uBXHymRSce7UxQn17izcKAt7hZBl4j4sebQ9+0jjuNur +zrW8b0XJ0PsI10GG9qHR3ajC+04pWfRritnK1g4Ycb/pDcUkWo+8uRwr7skAVcvC +oa8ToBS3iUbd3csFl1mu1BGACUHLvVs2cYdDtMuJj8wjsVZ7KnWBGKULAskwmU4Z +VVUxeUrG9f+2kT35meEJUk91FS+4tmqNIVsVlBzf0Q0ZU1iQnV56dMwTqFRzdDJ2 +DBecE0GwuYnKXo2I7kIYaqACQmk9YFh55Sh0K9PbQxyv7YEZXZtkcdqFqyhxy3Nh +EJ2kurFaM3/VmLljc/rW8QW8B3QNbw== +=pkDz -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index cac73dcbd..dc9190630 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1208,18 +1208,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.0 \ - --hash=sha256:6acd1e241785d73547803ca74bd1477eab6576e83eb035e0c343f1c8fc97b884 \ - --hash=sha256:bfdf0e2fe67f48034fa9a9bc16b12dd23ef3ac8bbac4e15ece876cd764eb40f8 -acme==0.25.0 \ - --hash=sha256:b20d27d6fd5b9d0e6fa4bf0528d7c6c2b9b301b49bdba4aad41fb9758fda1b3d \ - --hash=sha256:78c72b37d9ebc16ceb21df7f6b1037e80297abd61d0555c9d11f219a7118cef2 -certbot-apache==0.25.0 \ - --hash=sha256:e95eb8f24bd93d0c3e4e62a15ebe3042d411aaa1b107da5d869301472185924e \ - --hash=sha256:ca660d10e1945a78e0a00fd2be330be5acef97f215d3b03cb72cb0a996d63a64 -certbot-nginx==0.25.0 \ - --hash=sha256:8081edfe29943de54780e24c2a4ba7488e375177455f2cfad8bfe1b578bdd235 \ - --hash=sha256:0848642c28f3fad9759309f3e78652d8dd68062e068844a74f828155d2fda416 +certbot==0.25.1 \ + --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ + --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e +acme==0.25.1 \ + --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ + --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 +certbot-apache==0.25.1 \ + --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ + --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a +certbot-nginx==0.25.1 \ + --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ + --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 12330ad162164927502aadb9e73375d8a40b3d1a..f266c93e99a6c6b451a0dd066cf33fc069cac53c 100644 GIT binary patch literal 256 zcmV+b0ssDtYs3=1`HBT)24SsR1u@$W7ZPSU$S&OOtBrFsR+98n78ySvPt!!G z;8VuT@C4ln966cf7@p1|^2x*#OsORv*dICX|C;T?NkAcYvB#;S?b|v(@U=jc^F-3_=^g!<2^ko94v~{-OR8 zs(jE6oLCW6yS;4jMUbxlVGl~J&StH_A-KuM!mMbFXeL^Jnpkr41{l6#rWYsfUp^(i zzO<}YgQBPwyO%n@J11S;c~Voa9yf?}P0YBYWV5x#$dd)Xb67iJaEyXJESY09t?}#b GsB~g1X@A53 literal 256 zcmV+b0ssC$@8wI45>gS1>Vf-0?P)E}7cN~xWB;XyfZbYHf67R;8??@TB|A<6&Ymtv za~(Bb7VdlqKAz=5dK^kLhO%Pu%mcZ{cVC7weDiho<&%Nz(;*Xb*c(;*W-^&XC=|OD_~8fI^IFAUF~p|$&jhTt;E~D3eY}R9 z()EQT9^Wu3iJV-m%DTGS!1TdJ#YvLh-mJo<+p^7z6cnVGbv2Zai!|_#2965E&iQo& zOyxwu*TNkoH+8^qQvf_x;y1Sb&ZX_LL&2M~e^bphBFP!(!0Q2!3tl{Y_@sR Date: Wed, 13 Jun 2018 14:20:43 -0700 Subject: [PATCH 250/639] add 0.25.1 changelog (#6111) --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5facc5380..88251e48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.25.1 - 2018-06-13 + +### Fixed + +* TLS-ALPN-01 support has been removed from our acme library. Using our current + dependencies, we are unable to provide a correct implementation of this + challenge so we decided to remove it from the library until we can provide + proper support. +* Issues causing test failures when running the tests in the acme package with + pytest<3.0 has been resolved. +* certbot-nginx now correctly depends on acme>=0.25.0. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* acme +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/56?closed=1 + ## 0.25.0 - 2018-06-06 ### Added From c4ae376279e90ef12b4461f958403d4e517e6348 Mon Sep 17 00:00:00 2001 From: r5d Date: Wed, 13 Jun 2018 18:24:51 -0400 Subject: [PATCH 251/639] Add autorenew option to `renew` subcommand (#5911) * Add autorenew option to `renew` subcommand. * Change default value for 'autorenew' cli option. * Update certbot.cli.prepare_and_parse_args (autorenew) Set `default` for --autorenew and --no-autorenew. * Update certbot.storage.RenewableCert.should_autorenew. - Remove `interactive` argument in RenewableCert.should_autorenew. - Update certbot.renewal.should_renew. * Move autorenew enable/disable check to certbot.storage. - Remove autorenew enable/disable check in `certbot.renewal.handle_renewal_request`. - Fix RenewableCert.autorenewal_is_enabled; autorenew is stored in 'renewalparams'. - Add autorenew enable/disable check in `RenewableCert.should_autorenew`. - Update tests test_time_interval_judgments, test_autorenewal_is_enabled, test_should_autorenew tests in storage_test.py * certbot: Update RenewableCert.should_autorenew Remove block that sets autorenew option in the renewal configuration file. * certbot: Update prepare_and_parse_args. Remove --autorenew option. * certbot: Update CLI_DEFAULTS. Set default of `autorenew` to True. * Remove unused imports in certbot.storage. --- certbot/cli.py | 4 ++++ certbot/constants.py | 1 + certbot/renewal.py | 5 +++-- certbot/storage.py | 12 ++++-------- certbot/tests/storage_test.py | 18 ++++++++++++------ 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 05e316133..24e0bddac 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,6 +1220,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis " when the user executes \"certbot renew\", regardless of if the certificate" " is renewed. This setting does not apply to important TLS configuration" " updates.") + helpful.add( + "renew", "--no-autorenew", action="store_false", + default=flag_default("autorenew"), dest="autorenew", + help="Disable auto renewal of certificates.") helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) diff --git a/certbot/constants.py b/certbot/constants.py index c6746e888..93bc269af 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -37,6 +37,7 @@ CLI_DEFAULTS = dict( expand=False, renew_by_default=False, renew_with_new_domains=False, + autorenew=True, allow_subset_of_names=False, tos=False, account=None, diff --git a/certbot/renewal.py b/certbot/renewal.py index aa8c9722a..f50131028 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -36,7 +36,8 @@ STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "pre_hook", "post_hook", "tls_sni_01_address", "http01_address"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key"] +BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key", + "autorenew"] CONFIG_ITEMS = set(itertools.chain( BOOL_CONFIG_ITEMS, INT_CONFIG_ITEMS, STR_CONFIG_ITEMS, ('pref_challs',))) @@ -261,7 +262,7 @@ def should_renew(config, lineage): if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True - if lineage.should_autorenew(interactive=True): + if lineage.should_autorenew(): logger.info("Cert is due for renewal, auto-renewing...") return True if config.dry_run: diff --git a/certbot/storage.py b/certbot/storage.py index c453e55b0..5b2293bd1 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -920,10 +920,10 @@ class RenewableCert(object): :rtype: bool """ - return ("autorenew" not in self.configuration or - self.configuration.as_bool("autorenew")) + return ("autorenew" not in self.configuration["renewalparams"] or + self.configuration["renewalparams"].as_bool("autorenew")) - def should_autorenew(self, interactive=False): + def should_autorenew(self): """Should we now try to autorenew the most recent cert version? This is a policy question and does not only depend on whether @@ -934,16 +934,12 @@ class RenewableCert(object): Note that this examines the numerically most recent cert version, not the currently deployed version. - :param bool interactive: set to True to examine the question - regardless of whether the renewal configuration allows - automated renewal (for interactive use). Default False. - :returns: whether an attempt should now be made to autorenew the most current cert version in this lineage :rtype: bool """ - if interactive or self.autorenewal_is_enabled(): + if self.autorenewal_is_enabled(): # Consider whether to attempt to autorenew this cert now # Renewals on the basis of revocation diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 09c752ebe..aa6c52ad4 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -383,8 +383,9 @@ class RenewableCertTests(BaseRenewableCertTest): os.unlink(self.test_rc.cert) self.assertRaises(errors.CertStorageError, self.test_rc.names) + @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.datetime") - def test_time_interval_judgments(self, mock_datetime): + def test_time_interval_judgments(self, mock_datetime, mock_cli): """Test should_autodeploy() and should_autorenew() on the basis of expiry time windows.""" test_cert = test_util.load_vector("cert_512.pem") @@ -399,6 +400,8 @@ class RenewableCertTests(BaseRenewableCertTest): f.write(test_cert) mock_datetime.timedelta = datetime.timedelta + mock_cli.set_by_cli.return_value = False + self.test_rc.configuration["renewalparams"] = {} for (current_time, interval, result) in [ # 2014-12-13 12:00:00+00:00 (about 5 days prior to expiry) @@ -451,22 +454,25 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(self.test_rc.should_autodeploy()) def test_autorenewal_is_enabled(self): + self.test_rc.configuration["renewalparams"] = {} self.assertTrue(self.test_rc.autorenewal_is_enabled()) - self.test_rc.configuration["autorenew"] = "1" + self.test_rc.configuration["renewalparams"]["autorenew"] = "True" self.assertTrue(self.test_rc.autorenewal_is_enabled()) - self.test_rc.configuration["autorenew"] = "0" + self.test_rc.configuration["renewalparams"]["autorenew"] = "False" self.assertFalse(self.test_rc.autorenewal_is_enabled()) + @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.RenewableCert.ocsp_revoked") - def test_should_autorenew(self, mock_ocsp): + def test_should_autorenew(self, mock_ocsp, mock_cli): """Test should_autorenew on the basis of reasons other than expiry time window.""" # pylint: disable=too-many-statements + mock_cli.set_by_cli.return_value = False # Autorenewal turned off - self.test_rc.configuration["autorenew"] = "0" + self.test_rc.configuration["renewalparams"] = {"autorenew": "False"} self.assertFalse(self.test_rc.should_autorenew()) - self.test_rc.configuration["autorenew"] = "1" + self.test_rc.configuration["renewalparams"]["autorenew"] = "True" for kind in ALL_FOUR: self._write_out_kind(kind, 12) # Mandatory renewal on the basis of OCSP revocation From 453eafb11e5a571e9c63f5e95f2f1bd81c39e89c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 01:41:38 -0700 Subject: [PATCH 252/639] Used packaged acme in oldest tests. (#6112) --- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-dns-route53/local-oldest-requirements.txt | 2 +- local-oldest-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 724b61d3f..4e4aadbd8 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt index 724b61d3f..4e4aadbd8 100644 --- a/certbot-dns-route53/local-oldest-requirements.txt +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] +acme[dev]==0.25.0 certbot[dev]==0.21.1 diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 2346300a3..1f449acae 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ --e acme[dev] +acme[dev]==0.25.0 From 8b16a56de830a6aca19846fa9d4f234aea87b2d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 01:43:48 -0700 Subject: [PATCH 253/639] remove comment about renewer (#6115) --- certbot/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 8fa6ebfc6..089eab0c3 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -473,8 +473,7 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None): def _determine_account(config): """Determine which account to use. - In order to make the renewer (configuration de/serialization) happy, - if ``config.account`` is ``None``, it will be updated based on the + If ``config.account`` is ``None``, it will be updated based on the user input. Same for ``config.email``. :param config: Configuration object From 3316eac178bd5a41ed75776159f79695a929aad3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 15 Jun 2018 09:55:16 -0700 Subject: [PATCH 254/639] Separate integration coverage (#6113) * check coverage separately * Add coverage minimums for integration tests. --- certbot-nginx/tests/boulder-integration.sh | 2 ++ tests/boulder-integration.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index d6bd767ce..980b5d45a 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -62,3 +62,5 @@ test_deployment_and_rollback nginx6.wtf # note: not reached if anything above fails, hence "killall" at the # top nginx -c $nginx_root/nginx.conf -s stop + +coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ef611e743..2f9130489 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -486,11 +486,11 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi +coverage report --fail-under 65 --include 'certbot/*' --show-missing + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; then . ./certbot-nginx/tests/boulder-integration.sh fi - -coverage report --fail-under 67 -m From adc07ef933dea4526314f17a1b07b2f5c13715f3 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 15 Jun 2018 15:03:58 -0700 Subject: [PATCH 255/639] fix(display): alternate spaces and dashes (#6119) * fix(display): alternate spaces and dashes * add comment --- certbot/display/util.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/certbot/display/util.py b/certbot/display/util.py index 5e97bca4e..e157a1123 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -29,6 +29,10 @@ HELP = "help" ESC = "esc" """Display exit code when the user hits Escape (UNUSED)""" +# Display constants +SIDE_FRAME = ("- " * 39) + "-" +"""Display boundary (alternates spaces, so when copy-pasted, markdown doesn't interpret +it as a heading)""" def _wrap_lines(msg): """Format lines nicely to 80 chars. @@ -111,12 +115,11 @@ class FileDisplay(object): because it won't cause any workflow regressions """ - side_frame = "-" * 79 if wrap: message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( - line=os.linesep, frame=side_frame, msg=message)) + line=os.linesep, frame=SIDE_FRAME, msg=message)) self.outfile.flush() if pause: if self._can_interact(force_interactive): @@ -208,12 +211,10 @@ class FileDisplay(object): if self._return_default(message, default, cli_flag, force_interactive): return default - side_frame = ("-" * 79) + os.linesep - message = _wrap_lines(message) self.outfile.write("{0}{frame}{msg}{0}{frame}".format( - os.linesep, frame=side_frame, msg=message)) + os.linesep, frame=SIDE_FRAME + os.linesep, msg=message)) self.outfile.flush() while True: @@ -386,8 +387,7 @@ class FileDisplay(object): # Write out the message to the user self.outfile.write( "{new}{msg}{new}".format(new=os.linesep, msg=message)) - side_frame = ("-" * 79) + os.linesep - self.outfile.write(side_frame) + self.outfile.write(SIDE_FRAME + os.linesep) # Write out the menu choices for i, desc in enumerate(choices, 1): @@ -397,7 +397,7 @@ class FileDisplay(object): # Keep this outside of the textwrap self.outfile.write(os.linesep) - self.outfile.write(side_frame) + self.outfile.write(SIDE_FRAME + os.linesep) self.outfile.flush() def _get_valid_int_ans(self, max_): @@ -482,12 +482,11 @@ class NoninteractiveDisplay(object): :param bool wrap: Whether or not the application should wrap text """ - side_frame = "-" * 79 if wrap: message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( - line=os.linesep, frame=side_frame, msg=message)) + line=os.linesep, frame=SIDE_FRAME, msg=message)) self.outfile.flush() def menu(self, message, choices, ok_label=None, cancel_label=None, From 5025b4ea9614f87b8e3f8d01c90066fdde664df3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Aug 2017 16:08:47 -0700 Subject: [PATCH 256/639] Add certbot-postfix to tools pep8ify Delint cover++ test more_info() Refactor get_config_var Don't duplicate changes to Postfix config document instance variables Always clear save_notes on save Test deploy_cert and save and add MockPostfix. Move mock and call to InstallerTest Add getters and setters Use postfix getters and setters protect get_config_var bump cover to 100% bump required coverage to 100 s/config_dir/config_utility Decrease minimum version to Postfix 2.6. This is the minimum version that allows us to set ciphers to be used with opportunistic TLS and is the oldest version packaged in any major distro. Use tls_security_level instead of use_tls. smtpd_tls_security_level should be used instead according to Postfix documentation. Test smtpd_tls_security_level conditional make dunder method an under method refactor postconf usage add check_all_output test check_all_output Add and test verify_exe_exists Add PostfixUtilBase Add ReadOnlyMainMap Use _get_output instead of _call Fix split strip typo --- certbot-postfix/certbot_postfix/installer.py | 211 ++++++++---- .../certbot_postfix/installer_test.py | 313 +++++++++++++----- certbot-postfix/certbot_postfix/postconf.py | 62 ++++ certbot-postfix/certbot_postfix/util.py | 122 ++++++- certbot-postfix/certbot_postfix/util_test.py | 92 +++++ tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 9 files changed, 645 insertions(+), 163 deletions(-) create mode 100644 certbot-postfix/certbot_postfix/postconf.py diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 09e1cc18b..387f01051 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,9 +1,7 @@ """Certbot installer plugin for Postfix.""" import logging import os -import string import subprocess -import sys import zope.interface @@ -22,7 +20,15 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Installer): - """Certbot installer plugin for Postfix.""" + """Certbot installer plugin for Postfix. + + :ivar str config_dir: Postfix configuration directory to modify + :ivar dict proposed_changes: configuration parameters and values to + be written to the Postfix config when save() is called + :ivar list save_notes: documentation for proposed changes. This is + cleared and stored in Certbot checkpoints when save() is called + + """ description = "Configure TLS with the Postfix MTA" @@ -53,7 +59,7 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: when version is not supported """ - for param in ("ctl", "config_dir",): + for param in ("ctl", "config_utility",): self._verify_executable_is_available(param) self._set_config_dir() self._check_version() @@ -85,7 +91,7 @@ class Installer(plugins_common.Installer): """ if self.conf("config-dir") is None: - self.config_dir = self.get_config_var("config_directory") + self.config_dir = self._get_config_var("config_directory") else: self.config_dir = self.conf("config-dir") @@ -95,7 +101,7 @@ class Installer(plugins_common.Installer): :raises errors.NotSupportedError: if the version is unsupported """ - if self._get_version() < (2, 11, 0): + if self._get_version() < (2, 6,): raise errors.NotSupportedError('Postfix version is too old') def _lock_config_dir(self): @@ -138,7 +144,7 @@ class Installer(plugins_common.Installer): :raises .PluginError: Unable to find Postfix version. """ - mail_version = self.get_config_var("mail_version", default=True) + mail_version = self._get_config_default("mail_version") return tuple(int(i) for i in mail_version.split('.')) def get_all_names(self): @@ -147,7 +153,7 @@ class Installer(plugins_common.Installer): :rtype: `set` of `str` """ - return set(self.get_config_var(var) + return set(self._get_config_var(var) for var in ('mydomain', 'myhostname', 'myorigin',)) def deploy_cert(self, domain, cert_path, @@ -164,12 +170,16 @@ class Installer(plugins_common.Installer): :raises .PluginError: when cert cannot be deployed """ + # pylint: disable=unused-argument self.save_notes.append("Configuring TLS for {0}".format(domain)) self._set_config_var("smtpd_tls_cert_file", fullchain_path) self._set_config_var("smtpd_tls_key_file", key_path) self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - self._set_config_var("smtpd_use_tls", "yes") + + # Don't configure opportunistic TLS if it's currently mandatory + if self._get_config_var("smtpd_tls_security_level") != "encrypt": + self._set_config_var("smtpd_tls_security_level", "may") def enhance(self, domain, enhancement, options=None): """Raises an exception for request for unsupported enhancement. @@ -178,6 +188,7 @@ class Installer(plugins_common.Installer): are currently supported """ + # pylint: disable=unused-argument raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) @@ -202,14 +213,13 @@ class Installer(plugins_common.Installer): :raises errors.PluginError: when save is unsuccessful """ - if not self.proposed_changes: - return + if self.proposed_changes: + save_files = set((os.path.join(self.config_dir, "main.cf"),)) + self.add_to_checkpoint(save_files, + "\n".join(self.save_notes), temporary) + self._write_config_changes() + self.proposed_changes.clear() - self.add_to_checkpoint(os.path.join(self.config_dir, "main.cf"), - "\n".join(self.save_notes), temporary) - self._write_config_changes() - - self.proposed_changes.clear() del self.save_notes[:] if title and not temporary: @@ -298,70 +308,67 @@ class Installer(plugins_common.Installer): util.check_call(cmd) - def get_config_var(self, name, default=False): + def _get_config_default(self, name): + """Return the default value of the specified config parameter. + + :param str name: name of the Postfix config default to return + + :returns: default for the specified configuration parameter if it + exists, otherwise, None + :rtype: str or types.NoneType + + :raises errors.PluginError: if an error occurs while running postconf + or parsing its output + + """ + try: + return self._get_value_from_postconf(("-d", name,)) + except (subprocess.CalledProcessError, errors.PluginError): + raise errors.PluginError("Unable to determine the default value of" + " the Postfix parameter {0}".format(name)) + + def _get_config_var(self, name): """Return the value of the specified Postfix config parameter. - :param str name: name of the Postfix config parameter to return - :param bool default: whether or not to return the default value - instead of the actual value + If there is an unsaved change modifying the value of the + specified config parameter, the value after this proposed change + is returned rather than the current value. If the value is + unset, `None` is returned. - :returns: value of the specified configuration parameter - :rtype: str + :param str name: name of the Postfix config parameter to return + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if an error occurs while running postconf + or parsing its output """ - cmd = self._build_cmd_for_config_var(name, default) + if name in self.proposed_changes: + return self.proposed_changes[name] try: - output = util.check_output(cmd) - except subprocess.CalledProcessError: - logger.debug("Encountered an error when running 'postconf'", - exc_info=True) - raise errors.PluginError( - "Unable to determine the value " - "of Postfix parameter {0}".format(name)) - - expected_prefix = name + " =" - if not output.startswith(expected_prefix): - raise errors.PluginError( - "Unexpected output '{0}' from '{1}'".format(output, - ' '.join(cmd))) - - return output[len(expected_prefix):].strip() - - def _build_cmd_for_config_var(self, name, default): - """Return a command to run to get a Postfix config parameter. - - :param str name: name of the Postfix config parameter to return - :param bool default: whether or not to return the default value - instead of the actual value - - :returns: command to run - :rtype: list - - """ - cmd = self._postconf_command_base() - - if default: - cmd.append("-d") - - cmd.append(name) - - return cmd + return self._get_value_from_postconf((name,)) + except (subprocess.CalledProcessError, errors.PluginError): + raise errors.PluginError("Unable to determine the value of" + " the Postfix parameter {0}".format(name)) def _set_config_var(self, name, value): """Set the Postfix config parameter name to value. This method only stores the requested change in memory. The Postfix configuration is not modified until save() is called. + If there's already an identical in progress change or the + Postfix configuration parameter already has the specified value, + no changes are made. :param str name: name of the Postfix config parameter :param str value: value to set the Postfix config parameter to """ - assert isinstance(name, str), "Invalid name value" - assert isinstance(value, str), "Invalid key value" - self.proposed_changes[name] = value - self.save_notes.append("\t* Set {0} to {1}".format(name, value)) + if self._get_config_var(name) != value: + self.proposed_changes[name] = value + self.save_notes.append("\t* Set {0} to {1}".format(name, value)) def _write_config_changes(self): """Write proposed changes to the Postfix config. @@ -369,21 +376,83 @@ class Installer(plugins_common.Installer): :raises errors.PluginError: if an error occurs """ - cmd = self._postconf_command_base() - cmd.extend("{0}={1}".format(name, value) - for name, value in self.proposed_changes.items()) - try: - util.check_call(cmd) + self._run_postconf_command( + "{0}={1}".format(name, value) + for name, value in self.proposed_changes.items()) except subprocess.CalledProcessError: raise errors.PluginError( "An error occurred while updating your Postfix config.") - def _postconf_command_base(self): - """Builds start of a postconf command using the selected config.""" - cmd = [self.conf("config-utility")] + def _get_value_from_postconf(self, postconf_args): + """Runs postconf and extracts the specified config value. + It is assumed that the name of the Postfix config parameter to + parse from the output is the last value in postconf_args. If the + value is unset, `None` is returned. If an error occurs, the + relevant information is logged before an exception is raised. + + :param collections.Iterable args: arguments to postconf + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if unable to parse postconf output + :raises subprocess.CalledProcessError: if postconf fails + + """ + name = postconf_args[-1] + output = self._run_postconf_command(postconf_args) + + try: + return self._parse_postconf_output(output, name) + except errors.PluginError: + logger.debug("An error occurred while parsing postconf output", + exc_info=True) + raise + + def _run_postconf_command(self, args): + """Runs a postconf command using the selected config. + + If postconf exits with a nonzero status, the error is logged + before an exception is raised. + + :param collections.Iterable args: additional arguments to postconf + + :returns: stdout output of postconf + :rtype: str + + :raises subprocess.CalledProcessError: if the command fails + + """ + + cmd = [self.conf("config-utility")] if self.conf("config-dir") is not None: cmd.extend(("-c", self.conf("config-dir"),)) + cmd.extend(args) - return cmd + return util.check_output(cmd) + + def _parse_postconf_output(self, output, name): + """Parses postconf output and returns the specified value. + + If the specified Postfix parameter is unset, `None` is returned. + It is assumed that most one configuration parameter will be + included in the given output. + + :param str output: output from postconf + :param str name: name of the Postfix config parameter to obtain + + :returns: value of the parameter included in postconf_args + :rtype: str or types.NoneType + + :raises errors.PluginError: if unable to parse postconf ouput + + """ + expected_prefix = name + " =" + if output.count("\n") != 1 or not output.startswith(expected_prefix): + raise errors.PluginError( + "Unexpected output '{0}' from postconf".format(output)) + + value = output[len(expected_prefix):].strip() + return value if value else None diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py index 69d999b31..c57eda746 100644 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ b/certbot-postfix/certbot_postfix/installer_test.py @@ -1,36 +1,25 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - +"""Tests for certbot_postfix.installer.""" import functools -import logging import os import subprocess import unittest import mock -import six from certbot import errors from certbot.tests import util as certbot_test_util -# Fake Postfix Configs -names_only_config = """mydomain = fubard.org -myhostname = mail.fubard.org -myorigin = fubard.org""" - class InstallerTest(certbot_test_util.ConfigTestCase): + # pylint: disable=too-many-public-methods def setUp(self): super(InstallerTest, self).setUp() self.config.postfix_ctl = "postfix" self.config.postfix_config_dir = self.tempdir self.config.postfix_config_utility = "postconf" + self.mock_postfix = MockPostfix(self.tempdir, + {"mail_version": "3.1.4"}) def test_add_parser_arguments(self): options = set(('ctl', 'config-dir', 'config-utility',)) @@ -51,7 +40,29 @@ class InstallerTest(certbot_test_util.ConfigTestCase): with mock.patch(path_surgery_path, return_value=False): with mock.patch(exe_exists_path, return_value=False): - self.assertRaises(errors.NoInstallationError, installer.prepare) + self.assertRaises(errors.NoInstallationError, + installer.prepare) + + def test_postconf_error(self): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = subprocess.CalledProcessError(42, + "a") + with mock.patch(exe_exists_path, return_value=True): + self.assertRaises(errors.PluginError, installer.prepare) + + def test_unexpected_postconf(self): + installer = self._create_installer() + + check_output_path = "certbot_postfix.installer.util.check_output" + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.return_value = "foobar" + with mock.patch(exe_exists_path, return_value=True): + self.assertRaises(errors.PluginError, installer.prepare) def test_set_config_dir(self): self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") @@ -61,32 +72,100 @@ class InstallerTest(certbot_test_util.ConfigTestCase): expected = self.config.postfix_config_dir self.config.postfix_config_dir = None - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" + self.mock_postfix.set_value("config_directory", expected) exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = [ - "config_directory = " + expected, "mail_version = 3.1.4" - ] - with mock.patch(exe_exists_path, return_value=True): - with mock.patch(check_call_path): - installer.prepare() + with mock.patch(exe_exists_path, return_value=True): + self._mock_postfix_and_call(installer.prepare) self.assertEqual(installer.config_dir, expected) + @mock.patch("certbot_postfix.installer.certbot_util.exe_exists") + def test_old_version(self, mock_exe_exists): + installer = self._create_installer() + mock_exe_exists.return_value = True + self.mock_postfix.set_value("mail_version", "0.0.1") + self._mock_postfix_and_call( + self.assertRaises, errors.NotSupportedError, installer.prepare) + def test_lock_error(self): assert_raises = functools.partial(self.assertRaises, errors.PluginError, self._create_prepared_installer) certbot_test_util.lock_and_call(assert_raises, self.tempdir) - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_all_names(self, mock_check_output): + def test_more_info(self): installer = self._create_prepared_installer() - mock_check_output.side_effect = names_only_config.splitlines() + version = "3.1.2" + self.mock_postfix.set_value("mail_version", version) - result = installer.get_all_names() - self.assertTrue("fubard.org" in result) - self.assertTrue("mail.fubard.org" in result) + output = self._mock_postfix_and_call(installer.more_info) + self.assertTrue("Postfix" in output) + self.assertTrue(self.tempdir in output) + self.assertTrue(version in output) + + def test_get_all_names(self): + config = {"mydomain": "example.org", + "myhostname": "mail.example.org", + "myorigin": "example.org"} + for name, value in config.items(): + self.mock_postfix.set_value(name, value) + + installer = self._create_prepared_installer() + result = self._mock_postfix_and_call(installer.get_all_names) + self.assertEqual(result, set(config.values())) + + def test_deploy(self): + installer = self._create_prepared_installer() + + def deploy_cert(domain): + """Calls deploy_cert for the given domain. + + :param str domain: domain to deploy cert for + + """ + installer.deploy_cert(domain, "foo", "bar", "baz", "qux") + + self._mock_postfix_and_call(deploy_cert, "example.org") + # No calls to postconf are expected so mock isn't needed + deploy_cert("mail.example.org") + + def test_deploy_and_save(self): + self._test_deploy_and_save_common({"smtpd_tls_security_level": "may"}) + + def test_deploy_and_save2(self): + self.mock_postfix.set_value("smtpd_tls_security_level", "encrypt") + self._test_deploy_and_save_common({"smtpd_tls_security_level": + "encrypt"}) + + def _test_deploy_and_save_common(self, expected_config): + key_path = "key_path" + fullchain_path = "fullchain_path" + installer = self._create_prepared_installer() + + for i, domain in enumerate(("example.org", "mail.example.org",)): + self._mock_postfix_and_call( + installer.deploy_cert, domain, "unused", + key_path, "unused", fullchain_path) + if i: + # No mock because Postfix utilities aren't expected to be used + installer.save("noop") + else: + self._mock_postfix_and_call(installer.save, "real save") + + expected_config.setdefault("smtpd_tls_cert_file", fullchain_path) + expected_config.setdefault("smtpd_tls_key_file", key_path) + for key, value in expected_config.items(): + self.assertEqual(self.mock_postfix.get_value(key), value) + + def test_save_error(self): + installer = self._create_prepared_installer() + self._mock_postfix_and_call( + installer.deploy_cert, "example.org", "foo", "bar", "baz", "qux") + + check_call_path = "certbot_postfix.installer.util.check_output" + with mock.patch(check_call_path) as mock_check_call: + mock_check_call.side_effect = subprocess.CalledProcessError(42, + "foo") + self.assertRaises(errors.PluginError, installer.save) def test_enhance(self): self.assertRaises(errors.PluginError, @@ -111,10 +190,10 @@ class InstallerTest(certbot_test_util.ConfigTestCase): ] self.assertRaises(errors.PluginError, installer.restart) - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_reload_success(self, mock_check_call): - installer = self._create_prepared_installer() - installer.restart() + def test_postfix_reload_success(self): + with mock.patch("certbot_postfix.installer.subprocess.check_call"): + installer = self._create_prepared_installer() + installer.restart() @mock.patch("certbot_postfix.installer.subprocess.check_call") def test_postfix_start_failure(self, mock_check_call): @@ -130,52 +209,6 @@ class InstallerTest(certbot_test_util.ConfigTestCase): ] installer.restart() - def test_get_config_var_success(self): - self.config.postfix_config_dir = None - - command = self._test_get_config_var_success_common('foo', False) - self.assertFalse("-c" in command) - self.assertFalse("-d" in command) - - def test_get_config_var_success_with_config(self): - command = self._test_get_config_var_success_common('foo', False) - self.assertTrue("-c" in command) - self.assertFalse("-d" in command) - - def test_get_config_var_success_with_default(self): - self.config.postfix_config_dir = None - - command = self._test_get_config_var_success_common('foo', True) - self.assertFalse("-c" in command) - self.assertTrue("-d" in command) - - @mock.patch("certbot_postfix.installer.logger") - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_config_var_failure(self, mock_check_output, mock_logger): - mock_check_output.side_effect = subprocess.CalledProcessError(42, "foo") - installer = self._create_installer() - self.assertRaises(errors.PluginError, installer.get_config_var, "foo") - self.assertTrue(mock_logger.debug.call_args[1]["exc_info"]) - - @mock.patch("certbot_postfix.installer.util.check_output") - def test_get_config_var_unexpected_output(self, mock_check_output): - self.config.postfix_config_dir = None - mock_check_output.return_value = "foo" - - installer = self._create_installer() - self.assertRaises(errors.PluginError, installer.get_config_var, "foo") - - def _test_get_config_var_success_common(self, name, default): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - with mock.patch(check_output_path) as mock_check_output: - value = "bar" - mock_check_output.return_value = name + " = " + value - self.assertEqual(installer.get_config_var(name, default), value) - - return mock_check_output.call_args[0][0] - def _create_prepared_installer(self): """Creates and returns a new prepared Postfix Installer. @@ -188,14 +221,9 @@ class InstallerTest(certbot_test_util.ConfigTestCase): """ installer = self._create_installer() - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - with mock.patch(exe_exists_path, return_value=True): - with mock.patch(check_call_path): - mock_check_output.return_value = "mail_version = 3.1.4" - installer.prepare() + with mock.patch(exe_exists_path, return_value=True): + self._mock_postfix_and_call(installer.prepare) return installer @@ -211,6 +239,113 @@ class InstallerTest(certbot_test_util.ConfigTestCase): from certbot_postfix import installer return installer.Installer(self.config, name) + def _mock_postfix_and_call(self, func, *args, **kwargs): + """Calls func with mocked responses from Postfix utilities. + + :param callable func: function to call with mocked args + :param tuple args: positional arguments to func + :param dict kwargs: keyword arguments to func + + :returns: the return value of func + + """ + check_call_path = "certbot_postfix.installer.subprocess.check_call" + check_output_path = "certbot_postfix.installer.util.check_output" + + with mock.patch(check_call_path) as mock_check_call: + mock_check_call.side_effect = self.mock_postfix + with mock.patch(check_output_path) as mock_check_output: + mock_check_output.side_effect = self.mock_postfix + return func(*args, **kwargs) + + +class MockPostfix(object): + """A callable to mimic Postfix command line utilities. + + This is best used a side effect to a mock object. All calls to + 'postfix' are noops. For calls to 'postconf', values that are set in + the constructor or through mocked out runs of postconf are + remembered and properly returned if the installer attempts to fetch + the value. If the Postfix installer attempts to obtain a value that + hasn't yet been set, a dummy value is returned. + + :ivar str config_path: path to Postfix main.cf file + + """ + def __init__(self, config_dir, initial_values): + """Create Postfix configuration. + + :param str config_dir: path for Postfix config dir + :param dict initial_values: initial Postfix config values + + """ + initial_values["config_directory"] = config_dir + + self.config_path = os.path.join(config_dir, "main.cf") + self._write_config(initial_values) + + def __call__(self, args, *unused_args, **unused_kwargs): + cmd = os.path.basename(args[0]) + if cmd == "postfix": + return + elif cmd != "postconf": # pragma: no cover + assert False, "Unexpected command '{0}'".format(''.join(args)) + + output = [] + + skip = False + for arg in args[1:]: + if skip: + skip = False + elif arg[0] == "-": + if arg == "-c": + skip = True + elif "=" in arg: + name, _, value = arg.partition("=") + self.set_value(name, value) + else: + output.append("{0} = {1}\n".format(arg, self.get_value(arg))) + + return "\n".join(output) + + def get_value(self, name): + """Returns the value for the Postfix config parameter name. + + If the value isn't set, an empty string is returned. + + :param str name: name of the Postfix config parameter + + :returns: value of the named parameter + :rtype: str + + """ + return self._read_config().get(name, "") + + def set_value(self, name, value): + """Sets the value for a Postfix config parameter. + + :param str name: name of the Postfix config parameter + :param str value: value ot set the parameter to + + """ + config = self._read_config() + config[name] = value + self._write_config(config) + + def _read_config(self): + config = {} + with open(self.config_path) as f: + for line in f: + key, _, value = line.strip().partition(" = ") + config[key] = value + + return config + + def _write_config(self, config): + with open(self.config_path, "w") as f: + f.writelines("{0} = {1}\n".format(key, value) + for key, value in config.items()) + if __name__ == '__main__': - unittest.main() + unittest.main() # pragma: no cover diff --git a/certbot-postfix/certbot_postfix/postconf.py b/certbot-postfix/certbot_postfix/postconf.py new file mode 100644 index 000000000..7738562dd --- /dev/null +++ b/certbot-postfix/certbot_postfix/postconf.py @@ -0,0 +1,62 @@ +"""Classes that wrap the postconf command line utility. + +These classes allow you to interact with a Postfix config like it is a +dictionary, with the getting and setting of values in the config being +handled automatically by the class. + +""" +import collections + +from certbot_postfix import util + + +class ReadOnlyMainMap(util.PostfixUtilBase, collections.Mapping): + """A read-only view of a Postfix main.cf file.""" + + _modifiers = None + """An iterable containing additional CLI flags for postconf.""" + + def __getitem__(self, name): + return next(_parse_main_output(self._get_output([name])))[1] + + def __iter__(self): + for name, _ in _parse_main_output(self._get_output()): + yield name + + def __len__(self): + return sum(1 for _ in _parse_main_output(self._get_output())) + + def _call(self, extra_args=None): + """Runs Postconf and returns the result. + + If self._modifiers is set, it is provided on the command line to + postconf before any values in extra_args. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises subprocess.CalledProcessError: if the command fails + + """ + all_extra_args = [] + for args_list in (self._modifiers, extra_args,): + if args_list is not None: + all_extra_args.extend(args_list) + + return super(ReadOnlyMainMap, self)._call(all_extra_args) + + +def _parse_main_output(output): + """Parses the raw output from Postconf about main.cf. + + :param str output: data postconf wrote to stdout about main.cf + + :returns: generator providing key-value pairs from main.cf + :rtype: generator + + """ + for line in output.splitlines(): + name, _, value = line.partition(" =") + yield name, value.strip() diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index b65e4231e..57196017f 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -1,12 +1,129 @@ """Utility functions for use in the Postfix installer.""" - import logging import subprocess +from certbot import errors +from certbot import util as certbot_util +from certbot.plugins import util as plugins_util + logger = logging.getLogger(__name__) +class PostfixUtilBase(object): + """A base class for wrapping Postfix command line utilities.""" + + def __init__(self, executable, config_dir=None): + """Sets up the Postfix utility class. + + :param str executable: name or path of the Postfix utility + :param str config_dir: path to an alternative Postfix config + + :raises .NoInstallationError: when the executable isn't found + + """ + verify_exe_exists(executable) + + self._base_command = [executable] + if config_dir is not None: + self._base_command.extend(('-c', config_dir,)) + + def _call(self, extra_args=None): + """Runs the Postfix utility and returns the result. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises subprocess.CalledProcessError: if the command fails + + """ + args = list(self._base_command) + if extra_args is not None: + args.extend(extra_args) + return check_all_output(args) + + def _get_output(self, extra_args=None): + """Runs the Postfix utility and returns only stdout output. + + This function relies on self._call for running the utility. + + :param list extra_args: additional arguments for the command + + :returns: data written to stdout + :rtype: str + + :raises subprocess.CalledProcessError: if the command fails + + """ + return self._call(extra_args)[0] + + +def check_all_output(*args, **kwargs): + """A version of subprocess.check_output that also captures stderr. + + This is the same as :func:`subprocess.check_output` except output + written to stderr is also captured and returned to the caller. The + return value is a tuple of two strings (rather than byte strings). + To accomplish this, the caller cannot set the stdout, stderr, or + universal_newlines parameters to :class:`subprocess.Popen`. + + Additionally, if the command exits with a nonzero status, output is + not included in the raised :class:`subprocess.CalledProcessError` + because Python 2.6 does not support this. Instead, the failure + including the output is logged. + + :param tuple args: positional arguments for Popen + :param dict kwargs: keyword arguments for Popen + + :returns: data written to stdout and stderr + :rtype: `tuple` of `str` + + :raises ValueError: if arguments are invalid + :raises subprocess.CalledProcessError: if the command fails + + """ + for keyword in ('stdout', 'stderr', 'universal_newlines',): + if keyword in kwargs: + raise ValueError( + keyword + ' argument not allowed, it will be overridden.') + + kwargs['stdout'] = subprocess.PIPE + kwargs['stderr'] = subprocess.PIPE + kwargs['universal_newlines'] = True + + process = subprocess.Popen(*args, **kwargs) + output, err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get('args') + if cmd is None: + cmd = args[0] + logger.debug( + "'%s' exited with %d. stdout output was:\n%s\nstderr output was:\n%s", + cmd, retcode, output, err) + raise subprocess.CalledProcessError(retcode, cmd) + return (output, err) + + +def verify_exe_exists(exe): + """Ensures an executable with the given name is available. + + If an executable isn't found for the given path or name, extra + directories are added to the user's PATH to help find system + utilities that may not be available in the default cron PATH. + + :param str exe: executable path or name + + :raises .NoInstallationError: when the executable isn't found + + """ + if not (certbot_util.exe_exists(exe) or plugins_util.path_surgery(exe)): + raise errors.NoInstallationError( + "Cannot find executable '{0}'.".format(exe)) + + def check_call(*args, **kwargs): """A simple wrapper of subprocess.check_call that logs errors. @@ -63,7 +180,8 @@ def check_output(*args, **kwargs): if retcode: cmd = _get_cmd(*args, **kwargs) logger.debug( - "'%s' exited with %d. Output was:\n%s", cmd, retcode, output) + "'%s' exited with %d. Output was:\n%s", + cmd, retcode, output, exc_info=True) raise subprocess.CalledProcessError(retcode, cmd) return output diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py index 95253e1fd..4a014ca9b 100644 --- a/certbot-postfix/certbot_postfix/util_test.py +++ b/certbot-postfix/certbot_postfix/util_test.py @@ -5,6 +5,98 @@ import unittest import mock +from certbot import errors + + +class PostfixUtilBaseTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostfixUtilBase.""" + + @classmethod + def _create_object(cls, *args, **kwargs): + from certbot_postfix.util import PostfixUtilBase + return PostfixUtilBase(*args, **kwargs) + + @mock.patch('certbot_postfix.util.verify_exe_exists') + def test_no_exe(self, mock_verify): + expected_error = errors.NoInstallationError + mock_verify.side_effect = expected_error + self.assertRaises(expected_error, self._create_object, 'nonexistent') + + def test_object_creation(self): + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self._create_object('existent') + + +class CheckAllOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_all_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_all_output + return check_all_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + err = 'baz' + + mock_popen().communicate.return_value = (output, err) + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + log_args = mock_logger.debug.call_args[0] + for value in (command, retcode, output, err,): + self.assertTrue(value in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + expected = ('bar', '') + mock_popen().communicate.return_value = expected + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), expected) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_stderr_error(self): + self.assertRaises(ValueError, self._call, stderr=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +class VerifyExeExistsTest(unittest.TestCase): + """Tests for certbot_postfix.util.verify_exe_exists.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import verify_exe_exists + return verify_exe_exists(*args, **kwargs) + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_failure(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = mock_path_surgery.return_value = False + self.assertRaises(errors.NoInstallationError, self._call, 'foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + def test_simple_success(self, mock_exe_exists): + mock_exe_exists.return_value = True + self._call('foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = False + mock_path_surgery.return_value = True + self._call('foo') + + class CheckCallTest(unittest.TestCase): """Tests for certbot_postfix.util.check_call.""" diff --git a/tools/venv.sh b/tools/venv.sh index 1533f0e1f..a623ec529 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -25,5 +25,6 @@ fi -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-nginx \ + -e certbot-postfix \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tools/venv3.sh b/tools/venv3.sh index da56c2249..602118004 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,5 +23,6 @@ fi -e certbot-dns-nsone \ -e certbot-dns-route53 \ -e certbot-nginx \ + -e certbot-postfix \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tox.cover.sh b/tox.cover.sh index fc0c9f476..4f6ea2dab 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -43,6 +43,8 @@ cover () { min=99 elif [ "$1" = "certbot_nginx" ]; then min=97 + elif [ "$1" = "certbot_postfix" ]; then + min=100 elif [ "$1" = "letshelp_certbot" ]; then min=100 else diff --git a/tox.ini b/tox.ini index dee14b8b3..87eb77338 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ py26_packages = certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ + certbot-postfix \ letshelp-certbot non_py26_packages = certbot-dns-cloudxns \ @@ -55,6 +56,7 @@ source_paths = certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-nginx/certbot_nginx + certbot-postfix/certbot_postfix letshelp-certbot/letshelp_certbot tests/lock_test.py From 4ba153949de4fb6c4cfeda3d6400bf2bd6a33a73 Mon Sep 17 00:00:00 2001 From: Sydney Li Date: Wed, 14 Feb 2018 16:20:16 -0800 Subject: [PATCH 257/639] Fixing up postfix plugin - Finishing refactor of postconf/postfix command-line utilities - Plugin uses starttls_policy plugin to specify per-domain policies Cleaning up TLS policy code. Print warning when setting configuration parameter that is overridden by master. Update client to use new policy API Cleanup and test fixes Documentation fix smaller fixes Policy is now an enhancement and reverting works Added a README, and small documentation fixes throughout Moving testing infra from starttls repo to certbot-postfix fixing tests and lint Changes against new policy API starttls-everywhere => starttls-policy testing(postfix): Added more varieties of certificates to test against. Moar fixes against policy API. Address comments on README and setup.py Address small comments on postconf and util Address comments in installer Python 3 fixes and Postconf tester extends TempDir test class Mock out postconf calls from tests and test coverage for master overrides More various fixes. Everything minus testing done Remove STARTTLS policy enhancement from this branch. sphinx quickstart 99% test coverage some cleanup and testfixing cleanup leftover files Remove print statement testfix for python 3.4 Revert dockerfile change mypy fix fix(postfix): brad's comments test(postfix): coverage to 100 test(postfix): mypy import mypy types fix(postfix docs): add .rst files and fix build fix(postfix): tls_only and server_only params behave nicely together some cleanup lint fix more comments bump version number --- certbot-postfix/MANIFEST.in | 2 + certbot-postfix/README.rst | 9 + certbot-postfix/certbot_postfix/constants.py | 63 +++ certbot-postfix/certbot_postfix/installer.py | 434 ++++++------------ .../certbot_postfix/installer_test.py | 351 -------------- certbot-postfix/certbot_postfix/postconf.py | 172 +++++-- .../certbot_postfix/tests/__init__.py | 1 + .../certbot_postfix/tests/installer_test.py | 314 +++++++++++++ .../certbot_postfix/tests/postconf_test.py | 107 +++++ .../certbot_postfix/tests/util_test.py | 205 +++++++++ certbot-postfix/certbot_postfix/util.py | 231 +++++++--- certbot-postfix/certbot_postfix/util_test.py | 165 ------- certbot-postfix/docs/.gitignore | 1 + certbot-postfix/docs/Makefile | 20 + certbot-postfix/docs/api.rst | 8 + certbot-postfix/docs/api/installer.rst | 5 + certbot-postfix/docs/api/postconf.rst | 5 + certbot-postfix/docs/conf.py | 190 ++++++++ certbot-postfix/docs/index.rst | 28 ++ certbot-postfix/docs/make.bat | 36 ++ certbot-postfix/local-oldest-requirements.txt | 2 + certbot-postfix/setup.py | 24 +- certbot/constants.py | 2 +- certbot/plugins/disco.py | 1 + 24 files changed, 1438 insertions(+), 938 deletions(-) create mode 100644 certbot-postfix/certbot_postfix/constants.py delete mode 100644 certbot-postfix/certbot_postfix/installer_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/__init__.py create mode 100644 certbot-postfix/certbot_postfix/tests/installer_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/postconf_test.py create mode 100644 certbot-postfix/certbot_postfix/tests/util_test.py delete mode 100644 certbot-postfix/certbot_postfix/util_test.py create mode 100644 certbot-postfix/docs/.gitignore create mode 100644 certbot-postfix/docs/Makefile create mode 100644 certbot-postfix/docs/api.rst create mode 100644 certbot-postfix/docs/api/installer.rst create mode 100644 certbot-postfix/docs/api/postconf.rst create mode 100644 certbot-postfix/docs/conf.py create mode 100644 certbot-postfix/docs/index.rst create mode 100644 certbot-postfix/docs/make.bat create mode 100644 certbot-postfix/local-oldest-requirements.txt diff --git a/certbot-postfix/MANIFEST.in b/certbot-postfix/MANIFEST.in index 97e2ad3df..273381403 100644 --- a/certbot-postfix/MANIFEST.in +++ b/certbot-postfix/MANIFEST.in @@ -1,2 +1,4 @@ include LICENSE.txt include README.rst +recursive-include certbot_postfix/testdata * +recursive-include certbot_postfix/docs * diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index ee88648d3..78fd9a421 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -1 +1,10 @@ +========================== Postfix plugin for Certbot +========================== + +To install your certs with this plugin, run: + +``certbot install --installer postfix --cert-path --key-path -d `` + +And there you go! If you'd like to obtain these certificates via certbot, there's more documentation on how to do this `here `_. + diff --git a/certbot-postfix/certbot_postfix/constants.py b/certbot-postfix/certbot_postfix/constants.py new file mode 100644 index 000000000..40a263a53 --- /dev/null +++ b/certbot-postfix/certbot_postfix/constants.py @@ -0,0 +1,63 @@ +"""Postfix plugin constants.""" + +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Tuple, Union +# pylint: enable=unused-import, no-name-in-module + +MINIMUM_VERSION = (2, 11,) + +# If the value of a default VAR is a tuple, then the values which +# come LATER in the tuple are more strict/more secure. +# Certbot will default to the first value in the tuple, but will +# not override "more secure" settings. + +ACCEPTABLE_SERVER_SECURITY_LEVELS = ("may", "encrypt") +ACCEPTABLE_CLIENT_SECURITY_LEVELS = ("may", "encrypt", + "dane", "dane-only", + "fingerprint", + "verify", "secure") +ACCEPTABLE_CIPHER_LEVELS = ("medium", "high") + +# Exporting certain ciphers to prevent logjam: https://weakdh.org/sysadmin.html +EXCLUDE_CIPHERS = ("aNULL, eNULL, EXPORT, DES, RC4, MD5, PSK, aECDH, " + "EDH-DSS-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA, KRB5-DES, CBC3-SHA",) + + +TLS_VERSIONS = ("SSLv2", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2") +# Should NOT use SSLv2/3. +ACCEPTABLE_TLS_VERSIONS = ("TLSv1", "TLSv1.1", "TLSv1.2") + +# Variables associated with enabling opportunistic TLS. +TLS_SERVER_VARS = { + "smtpd_tls_security_level": ACCEPTABLE_SERVER_SECURITY_LEVELS, +} # type:Dict[str, Tuple[str, ...]] +TLS_CLIENT_VARS = { + "smtp_tls_security_level": ACCEPTABLE_CLIENT_SECURITY_LEVELS, +} # type:Dict[str, Tuple[str, ...]] +# Default variables for a secure MTA server [receiver]. +DEFAULT_SERVER_VARS = { + "smtpd_tls_auth_only": ("yes",), + "smtpd_tls_mandatory_protocols": ("!SSLv2, !SSLv3",), + "smtpd_tls_protocols": ("!SSLv2, !SSLv3",), + "smtpd_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtpd_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtpd_tls_exclude_ciphers": EXCLUDE_CIPHERS, + "smtpd_tls_eecdh_grade": ("strong",), +} # type:Dict[str, Tuple[str, ...]] + +# Default variables for a secure MTA client [sender]. +DEFAULT_CLIENT_VARS = { + "smtp_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS, + "smtp_tls_exclude_ciphers": EXCLUDE_CIPHERS, + "smtp_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS, +} # type:Dict[str, Tuple[str, ...]] + +CLI_DEFAULTS = dict( + config_dir="/etc/postfix", + ctl="postfix", + config_utility="postconf", + tls_only=False, + ignore_master_overrides=False, + server_only=False, +) +"""CLI defaults.""" diff --git a/certbot-postfix/certbot_postfix/installer.py b/certbot-postfix/certbot_postfix/installer.py index 387f01051..9ba92ef8f 100644 --- a/certbot-postfix/certbot_postfix/installer.py +++ b/certbot-postfix/certbot_postfix/installer.py @@ -1,127 +1,139 @@ -"""Certbot installer plugin for Postfix.""" +"""certbot installer plugin for postfix.""" import logging import os -import subprocess import zope.interface +import zope.component +import six from certbot import errors from certbot import interfaces from certbot import util as certbot_util from certbot.plugins import common as plugins_common -from certbot.plugins import util as plugins_util +from certbot_postfix import constants +from certbot_postfix import postconf from certbot_postfix import util +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Dict, List +# pylint: enable=unused-import, no-name-in-module logger = logging.getLogger(__name__) - @zope.interface.implementer(interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class Installer(plugins_common.Installer): """Certbot installer plugin for Postfix. :ivar str config_dir: Postfix configuration directory to modify - :ivar dict proposed_changes: configuration parameters and values to - be written to the Postfix config when save() is called :ivar list save_notes: documentation for proposed changes. This is cleared and stored in Certbot checkpoints when save() is called + :ivar postconf: Wrapper for Postfix configuration command-line tool. + :type postconf: :class: `certbot_postfix.postconf.ConfigMain` + :ivar postfix: Wrapper for Postfix command-line tool. + :type postfix: :class: `certbot_postfix.util.PostfixUtil` """ description = "Configure TLS with the Postfix MTA" @classmethod def add_parser_arguments(cls, add): - add("ctl", default="postfix", + add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the 'postfix' control program.") - add("config-dir", help="Path to the directory containing the " + # This directory points to Postfix's configuration directory. + add("config-dir", default=constants.CLI_DEFAULTS["config_dir"], + help="Path to the directory containing the " "Postfix main.cf file to modify instead of using the " "default configuration paths.") - add("config-utility", default="postconf", + add("config-utility", default=constants.CLI_DEFAULTS["config_utility"], help="Path to the 'postconf' executable.") + add("tls-only", action="store_true", default=constants.CLI_DEFAULTS["tls_only"], + help="Only set params to enable opportunistic TLS and install certificates.") + add("server-only", action="store_true", default=constants.CLI_DEFAULTS["server_only"], + help="Only set server params (prefixed with smtpd*)") + add("ignore-master-overrides", action="store_true", + default=constants.CLI_DEFAULTS["ignore_master_overrides"], + help="Ignore errors reporting overridden TLS parameters in master.cf.") def __init__(self, *args, **kwargs): super(Installer, self).__init__(*args, **kwargs) - self.config_dir = None - self.proposed_changes = {} - self.save_notes = [] + # Wrapper around postconf commands + self.postfix = None + self.postconf = None + + # Files to save + self.save_notes = [] # type: List[str] + + self._enhance_func = {} # type: Dict[str, Callable[[str, str], None]] + # Since we only need to enable TLS once for all domains, + # keep track of whether this enhancement was already called. + self._tls_enabled = False def prepare(self): """Prepare the installer. - Finish up any additional initialization. - :raises errors.PluginError: when an unexpected error occurs :raises errors.MisconfigurationError: when the config is invalid :raises errors.NoInstallationError: when can't find installation :raises errors.NotSupportedError: when version is not supported - """ + # Verify postfix and postconf are installed for param in ("ctl", "config_utility",): - self._verify_executable_is_available(param) - self._set_config_dir() - self._check_version() - self.config_test() - self._lock_config_dir() - - def _verify_executable_is_available(self, config_name): - """Asserts the program in the specified config param is found. - - :param str config_name: name of the config param - - :raises .NoInstallationError: when the executable isn't found - - """ - if not certbot_util.exe_exists(self.conf(config_name)): - if not plugins_util.path_surgery(self.conf(config_name)): - raise errors.NoInstallationError( + util.verify_exe_exists(self.conf(param), "Cannot find executable '{0}'. You can provide the " "path to this command with --{1}".format( - self.conf(config_name), - self.option_name(config_name))) + self.conf(param), + self.option_name(param))) - def _set_config_dir(self): - """Ensure self.config_dir is set to the correct path. + # Set up CLI tools + self.postfix = util.PostfixUtil(self.conf('config-dir')) + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) - If the configuration directory to use was set by the user, we'll - use that value, otherwise, we'll find the default path using - 'postconf'. + # Ensure current configuration is valid. + self.config_test() + # Check Postfix version + self._check_version() + self._lock_config_dir() + self.install_ssl_dhparams() + + def config_test(self): + """Test to see that the current Postfix configuration is valid. + + :raises errors.MisconfigurationError: If the configuration is invalid. """ - if self.conf("config-dir") is None: - self.config_dir = self._get_config_var("config_directory") - else: - self.config_dir = self.conf("config-dir") + self.postfix.test() def _check_version(self): """Verifies that the installed Postfix version is supported. :raises errors.NotSupportedError: if the version is unsupported - """ - if self._get_version() < (2, 6,): - raise errors.NotSupportedError('Postfix version is too old') + if self._get_version() < constants.MINIMUM_VERSION: + version_string = '.'.join([str(n) for n in constants.MINIMUM_VERSION]) + raise errors.NotSupportedError('Postfix version must be at least %s' % version_string) def _lock_config_dir(self): """Stop two Postfix plugins from modifying the config at once. :raises .PluginError: if unable to acquire the lock - """ try: - certbot_util.lock_dir_until_exit(self.config_dir) + certbot_util.lock_dir_until_exit(self.conf('config-dir')) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( - "Unable to lock %s", self.config_dir) + "Unable to lock %s" % self.conf('config-dir')) def more_info(self): - """Human-readable string to help the user. - Should describe the steps taken and any relevant info to help the user - decide which plugin to use. - :rtype str: + """Human-readable string to help the user. Describes steps taken and any relevant + info to help the user decide which plugin to use. + + :rtype: str """ return ( "Configures Postfix to try to authenticate mail servers, use " @@ -129,22 +141,19 @@ class Installer(plugins_common.Installer): "Server root: {root}{0}" "Version: {version}".format( os.linesep, - root=self.config_dir, + root=self.conf('config-dir'), version='.'.join([str(i) for i in self._get_version()])) ) def _get_version(self): - """Return the mail version of Postfix. - - Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) + """Return the version of Postfix, as a tuple. (e.g. '2.11.3' is (2, 11, 3)) :returns: version :rtype: tuple - :raises .PluginError: Unable to find Postfix version. - + :raises errors.PluginError: Unable to find Postfix version. """ - mail_version = self._get_config_default("mail_version") + mail_version = self.postconf.get_default("mail_version") return tuple(int(i) for i in mail_version.split('.')) def get_all_names(self): @@ -153,9 +162,34 @@ class Installer(plugins_common.Installer): :rtype: `set` of `str` """ - return set(self._get_config_var(var) + return certbot_util.get_filtered_names(self.postconf.get(var) for var in ('mydomain', 'myhostname', 'myorigin',)) + def _set_vars(self, var_dict): + """Sets all parameters in var_dict to config file. If current value is already set + as more secure (acceptable), then don't set/overwrite it. + """ + for param, acceptable in six.iteritems(var_dict): + if not util.is_acceptable_value(param, self.postconf.get(param), acceptable): + self.postconf.set(param, acceptable[0], acceptable) + + def _confirm_changes(self): + """Confirming outstanding updates for configuration parameters. + + :raises errors.PluginError: when user rejects the configuration changes. + """ + updates = self.postconf.get_changes() + output_string = "Postfix TLS configuration parameters to update in main.cf:\n" + for name, value in six.iteritems(updates): + output_string += "{0} = {1}\n".format(name, value) + output_string += "Is this okay?\n" + if not zope.component.getUtility(interfaces.IDisplay).yesno(output_string, + force_interactive=True, default=True): + raise errors.PluginError( + "Manually rejected configuration changes.\n" + "Try using --tls-only or --server-only to change a particular" + "subset of configuration parameters.") + def deploy_cert(self, domain, cert_path, key_path, chain_path, fullchain_path): """Configure the Postfix SMTP server to use the given TLS cert. @@ -171,22 +205,26 @@ class Installer(plugins_common.Installer): """ # pylint: disable=unused-argument + if self._tls_enabled: + return + self._tls_enabled = True self.save_notes.append("Configuring TLS for {0}".format(domain)) - self._set_config_var("smtpd_tls_cert_file", fullchain_path) - self._set_config_var("smtpd_tls_key_file", key_path) - self._set_config_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3") - self._set_config_var("smtpd_tls_protocols", "!SSLv2, !SSLv3") - - # Don't configure opportunistic TLS if it's currently mandatory - if self._get_config_var("smtpd_tls_security_level") != "encrypt": - self._set_config_var("smtpd_tls_security_level", "may") + self.postconf.set("smtpd_tls_cert_file", cert_path) + self.postconf.set("smtpd_tls_key_file", key_path) + self._set_vars(constants.TLS_SERVER_VARS) + if not self.conf('server_only'): + self._set_vars(constants.TLS_CLIENT_VARS) + if not self.conf('tls_only'): + self._set_vars(constants.DEFAULT_SERVER_VARS) + if not self.conf('server_only'): + self._set_vars(constants.DEFAULT_CLIENT_VARS) + # Despite the name, this option also supports 2048-bit DH params. + # http://www.postfix.org/FORWARD_SECRECY_README.html#server_fs + self.postconf.set("smtpd_tls_dh1024_param_file", self.ssl_dhparams) + self._confirm_changes() def enhance(self, domain, enhancement, options=None): - """Raises an exception for request for unsupported enhancement. - - :raises .PluginError: this is always raised as no enhancements - are currently supported - + """Raises an exception since this installer doesn't support any enhancements. """ # pylint: disable=unused-argument raise errors.PluginError( @@ -211,248 +249,40 @@ class Installer(plugins_common.Installer): be quickly reversed in the future (challenges) :raises errors.PluginError: when save is unsuccessful - """ - if self.proposed_changes: - save_files = set((os.path.join(self.config_dir, "main.cf"),)) - self.add_to_checkpoint(save_files, - "\n".join(self.save_notes), temporary) - self._write_config_changes() - self.proposed_changes.clear() + save_files = set((os.path.join(self.conf('config-dir'), "main.cf"),)) + self.add_to_checkpoint(save_files, + "\n".join(self.save_notes), temporary) + self.postconf.flush() del self.save_notes[:] if title and not temporary: self.finalize_checkpoint(title) - def config_test(self): - """Make sure the configuration is valid. + def recovery_routine(self): + super(Installer, self).recovery_routine() + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) - :raises .MisconfigurationError: if the config is invalid + def rollback_checkpoints(self, rollback=1): + """Rollback saved checkpoints. + :param int rollback: Number of checkpoints to revert + + :raises .errors.PluginError: If there is a problem with the input or + the function is unable to correctly revert the configuration """ - try: - self._run_postfix_subcommand("check") - except subprocess.CalledProcessError: - raise errors.MisconfigurationError( - "Postfix failed internal configuration check.") + super(Installer, self).rollback_checkpoints(rollback) + self.postconf = postconf.ConfigMain(self.conf('config-utility'), + self.conf('ignore-master-overrides'), + self.conf('config-dir')) def restart(self): """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted - """ - logger.info("Reloading Postfix configuration...") - if self._is_postfix_running(): - self._reload() - else: - self._start() + self.postfix.restart() - def _is_postfix_running(self): - """Is Postfix currently running? - - Uses the 'postfix status' command to determine if Postfix is - currently running using the specified configuration files. - - :returns: True if Postfix is running, otherwise, False - :rtype: bool - - """ - try: - self._run_postfix_subcommand("status") - except subprocess.CalledProcessError: - return False - return True - - def _reload(self): - """Instructions Postfix to reload its configuration. - - If Postfix isn't currently running, this method will fail. - - :raises .PluginError: when Postfix cannot reload - - """ - try: - self._run_postfix_subcommand("reload") - except subprocess.CalledProcessError: - raise errors.PluginError( - "Postfix failed to reload its configuration.") - - def _start(self): - """Instructions Postfix to start running. - - :raises .PluginError: when Postfix cannot start - - """ - try: - self._run_postfix_subcommand("start") - except subprocess.CalledProcessError: - raise errors.PluginError("Postfix failed to start") - - def _run_postfix_subcommand(self, subcommand): - """Runs a subcommand of the 'postfix' control program. - - If the command fails, the exception is logged at the DEBUG - level. - - :param str subcommand: subcommand to run - - :raises subprocess.CalledProcessError: if the command fails - - """ - cmd = [self.conf("ctl")] - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) - cmd.append(subcommand) - - util.check_call(cmd) - - def _get_config_default(self, name): - """Return the default value of the specified config parameter. - - :param str name: name of the Postfix config default to return - - :returns: default for the specified configuration parameter if it - exists, otherwise, None - :rtype: str or types.NoneType - - :raises errors.PluginError: if an error occurs while running postconf - or parsing its output - - """ - try: - return self._get_value_from_postconf(("-d", name,)) - except (subprocess.CalledProcessError, errors.PluginError): - raise errors.PluginError("Unable to determine the default value of" - " the Postfix parameter {0}".format(name)) - - def _get_config_var(self, name): - """Return the value of the specified Postfix config parameter. - - If there is an unsaved change modifying the value of the - specified config parameter, the value after this proposed change - is returned rather than the current value. If the value is - unset, `None` is returned. - - :param str name: name of the Postfix config parameter to return - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if an error occurs while running postconf - or parsing its output - - """ - if name in self.proposed_changes: - return self.proposed_changes[name] - - try: - return self._get_value_from_postconf((name,)) - except (subprocess.CalledProcessError, errors.PluginError): - raise errors.PluginError("Unable to determine the value of" - " the Postfix parameter {0}".format(name)) - - def _set_config_var(self, name, value): - """Set the Postfix config parameter name to value. - - This method only stores the requested change in memory. The - Postfix configuration is not modified until save() is called. - If there's already an identical in progress change or the - Postfix configuration parameter already has the specified value, - no changes are made. - - :param str name: name of the Postfix config parameter - :param str value: value to set the Postfix config parameter to - - """ - if self._get_config_var(name) != value: - self.proposed_changes[name] = value - self.save_notes.append("\t* Set {0} to {1}".format(name, value)) - - def _write_config_changes(self): - """Write proposed changes to the Postfix config. - - :raises errors.PluginError: if an error occurs - - """ - try: - self._run_postconf_command( - "{0}={1}".format(name, value) - for name, value in self.proposed_changes.items()) - except subprocess.CalledProcessError: - raise errors.PluginError( - "An error occurred while updating your Postfix config.") - - def _get_value_from_postconf(self, postconf_args): - """Runs postconf and extracts the specified config value. - - It is assumed that the name of the Postfix config parameter to - parse from the output is the last value in postconf_args. If the - value is unset, `None` is returned. If an error occurs, the - relevant information is logged before an exception is raised. - - :param collections.Iterable args: arguments to postconf - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if unable to parse postconf output - :raises subprocess.CalledProcessError: if postconf fails - - """ - name = postconf_args[-1] - output = self._run_postconf_command(postconf_args) - - try: - return self._parse_postconf_output(output, name) - except errors.PluginError: - logger.debug("An error occurred while parsing postconf output", - exc_info=True) - raise - - def _run_postconf_command(self, args): - """Runs a postconf command using the selected config. - - If postconf exits with a nonzero status, the error is logged - before an exception is raised. - - :param collections.Iterable args: additional arguments to postconf - - :returns: stdout output of postconf - :rtype: str - - :raises subprocess.CalledProcessError: if the command fails - - """ - - cmd = [self.conf("config-utility")] - if self.conf("config-dir") is not None: - cmd.extend(("-c", self.conf("config-dir"),)) - cmd.extend(args) - - return util.check_output(cmd) - - def _parse_postconf_output(self, output, name): - """Parses postconf output and returns the specified value. - - If the specified Postfix parameter is unset, `None` is returned. - It is assumed that most one configuration parameter will be - included in the given output. - - :param str output: output from postconf - :param str name: name of the Postfix config parameter to obtain - - :returns: value of the parameter included in postconf_args - :rtype: str or types.NoneType - - :raises errors.PluginError: if unable to parse postconf ouput - - """ - expected_prefix = name + " =" - if output.count("\n") != 1 or not output.startswith(expected_prefix): - raise errors.PluginError( - "Unexpected output '{0}' from postconf".format(output)) - - value = output[len(expected_prefix):].strip() - return value if value else None diff --git a/certbot-postfix/certbot_postfix/installer_test.py b/certbot-postfix/certbot_postfix/installer_test.py deleted file mode 100644 index c57eda746..000000000 --- a/certbot-postfix/certbot_postfix/installer_test.py +++ /dev/null @@ -1,351 +0,0 @@ -"""Tests for certbot_postfix.installer.""" -import functools -import os -import subprocess -import unittest - -import mock - -from certbot import errors -from certbot.tests import util as certbot_test_util - - -class InstallerTest(certbot_test_util.ConfigTestCase): - # pylint: disable=too-many-public-methods - - def setUp(self): - super(InstallerTest, self).setUp() - self.config.postfix_ctl = "postfix" - self.config.postfix_config_dir = self.tempdir - self.config.postfix_config_utility = "postconf" - self.mock_postfix = MockPostfix(self.tempdir, - {"mail_version": "3.1.4"}) - - def test_add_parser_arguments(self): - options = set(('ctl', 'config-dir', 'config-utility',)) - mock_add = mock.MagicMock() - - from certbot_postfix import installer - installer.Installer.add_parser_arguments(mock_add) - - for call in mock_add.call_args_list: - self.assertTrue(call[0][0] in options) - - def test_no_postconf_prepare(self): - installer = self._create_installer() - - installer_path = "certbot_postfix.installer" - exe_exists_path = installer_path + ".certbot_util.exe_exists" - path_surgery_path = installer_path + ".plugins_util.path_surgery" - - with mock.patch(path_surgery_path, return_value=False): - with mock.patch(exe_exists_path, return_value=False): - self.assertRaises(errors.NoInstallationError, - installer.prepare) - - def test_postconf_error(self): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = subprocess.CalledProcessError(42, - "a") - with mock.patch(exe_exists_path, return_value=True): - self.assertRaises(errors.PluginError, installer.prepare) - - def test_unexpected_postconf(self): - installer = self._create_installer() - - check_output_path = "certbot_postfix.installer.util.check_output" - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.return_value = "foobar" - with mock.patch(exe_exists_path, return_value=True): - self.assertRaises(errors.PluginError, installer.prepare) - - def test_set_config_dir(self): - self.config.postfix_config_dir = os.path.join(self.tempdir, "subdir") - os.mkdir(self.config.postfix_config_dir) - installer = self._create_installer() - - expected = self.config.postfix_config_dir - self.config.postfix_config_dir = None - - self.mock_postfix.set_value("config_directory", expected) - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(exe_exists_path, return_value=True): - self._mock_postfix_and_call(installer.prepare) - self.assertEqual(installer.config_dir, expected) - - @mock.patch("certbot_postfix.installer.certbot_util.exe_exists") - def test_old_version(self, mock_exe_exists): - installer = self._create_installer() - mock_exe_exists.return_value = True - self.mock_postfix.set_value("mail_version", "0.0.1") - self._mock_postfix_and_call( - self.assertRaises, errors.NotSupportedError, installer.prepare) - - def test_lock_error(self): - assert_raises = functools.partial(self.assertRaises, - errors.PluginError, - self._create_prepared_installer) - certbot_test_util.lock_and_call(assert_raises, self.tempdir) - - def test_more_info(self): - installer = self._create_prepared_installer() - version = "3.1.2" - self.mock_postfix.set_value("mail_version", version) - - output = self._mock_postfix_and_call(installer.more_info) - self.assertTrue("Postfix" in output) - self.assertTrue(self.tempdir in output) - self.assertTrue(version in output) - - def test_get_all_names(self): - config = {"mydomain": "example.org", - "myhostname": "mail.example.org", - "myorigin": "example.org"} - for name, value in config.items(): - self.mock_postfix.set_value(name, value) - - installer = self._create_prepared_installer() - result = self._mock_postfix_and_call(installer.get_all_names) - self.assertEqual(result, set(config.values())) - - def test_deploy(self): - installer = self._create_prepared_installer() - - def deploy_cert(domain): - """Calls deploy_cert for the given domain. - - :param str domain: domain to deploy cert for - - """ - installer.deploy_cert(domain, "foo", "bar", "baz", "qux") - - self._mock_postfix_and_call(deploy_cert, "example.org") - # No calls to postconf are expected so mock isn't needed - deploy_cert("mail.example.org") - - def test_deploy_and_save(self): - self._test_deploy_and_save_common({"smtpd_tls_security_level": "may"}) - - def test_deploy_and_save2(self): - self.mock_postfix.set_value("smtpd_tls_security_level", "encrypt") - self._test_deploy_and_save_common({"smtpd_tls_security_level": - "encrypt"}) - - def _test_deploy_and_save_common(self, expected_config): - key_path = "key_path" - fullchain_path = "fullchain_path" - installer = self._create_prepared_installer() - - for i, domain in enumerate(("example.org", "mail.example.org",)): - self._mock_postfix_and_call( - installer.deploy_cert, domain, "unused", - key_path, "unused", fullchain_path) - if i: - # No mock because Postfix utilities aren't expected to be used - installer.save("noop") - else: - self._mock_postfix_and_call(installer.save, "real save") - - expected_config.setdefault("smtpd_tls_cert_file", fullchain_path) - expected_config.setdefault("smtpd_tls_key_file", key_path) - for key, value in expected_config.items(): - self.assertEqual(self.mock_postfix.get_value(key), value) - - def test_save_error(self): - installer = self._create_prepared_installer() - self._mock_postfix_and_call( - installer.deploy_cert, "example.org", "foo", "bar", "baz", "qux") - - check_call_path = "certbot_postfix.installer.util.check_output" - with mock.patch(check_call_path) as mock_check_call: - mock_check_call.side_effect = subprocess.CalledProcessError(42, - "foo") - self.assertRaises(errors.PluginError, installer.save) - - def test_enhance(self): - self.assertRaises(errors.PluginError, - self._create_prepared_installer().enhance, - "example.org", "redirect") - - def test_supported_enhancements(self): - self.assertEqual( - self._create_prepared_installer().supported_enhancements(), []) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_config_test_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") - self.assertRaises(errors.MisconfigurationError, installer.config_test) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_reload_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = [ - None, subprocess.CalledProcessError(42, "foo") - ] - self.assertRaises(errors.PluginError, installer.restart) - - def test_postfix_reload_success(self): - with mock.patch("certbot_postfix.installer.subprocess.check_call"): - installer = self._create_prepared_installer() - installer.restart() - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_start_failure(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = subprocess.CalledProcessError(42, "foo") - self.assertRaises(errors.PluginError, installer.restart) - - @mock.patch("certbot_postfix.installer.subprocess.check_call") - def test_postfix_start_success(self, mock_check_call): - installer = self._create_prepared_installer() - mock_check_call.side_effect = [ - subprocess.CalledProcessError(42, "foo"), None - ] - installer.restart() - - def _create_prepared_installer(self): - """Creates and returns a new prepared Postfix Installer. - - Calls in prepare() are mocked out so the Postfix version check - is successful. - - :returns: a prepared Postfix installer - :rtype: certbot_postfix.installer.Installer - - """ - installer = self._create_installer() - - exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" - with mock.patch(exe_exists_path, return_value=True): - self._mock_postfix_and_call(installer.prepare) - - return installer - - def _create_installer(self): - """Creates and returns a new Postfix Installer. - - :returns: a new Postfix installer - :rtype: certbot_postfix.installer.Installer - - """ - name = "postfix" - - from certbot_postfix import installer - return installer.Installer(self.config, name) - - def _mock_postfix_and_call(self, func, *args, **kwargs): - """Calls func with mocked responses from Postfix utilities. - - :param callable func: function to call with mocked args - :param tuple args: positional arguments to func - :param dict kwargs: keyword arguments to func - - :returns: the return value of func - - """ - check_call_path = "certbot_postfix.installer.subprocess.check_call" - check_output_path = "certbot_postfix.installer.util.check_output" - - with mock.patch(check_call_path) as mock_check_call: - mock_check_call.side_effect = self.mock_postfix - with mock.patch(check_output_path) as mock_check_output: - mock_check_output.side_effect = self.mock_postfix - return func(*args, **kwargs) - - -class MockPostfix(object): - """A callable to mimic Postfix command line utilities. - - This is best used a side effect to a mock object. All calls to - 'postfix' are noops. For calls to 'postconf', values that are set in - the constructor or through mocked out runs of postconf are - remembered and properly returned if the installer attempts to fetch - the value. If the Postfix installer attempts to obtain a value that - hasn't yet been set, a dummy value is returned. - - :ivar str config_path: path to Postfix main.cf file - - """ - def __init__(self, config_dir, initial_values): - """Create Postfix configuration. - - :param str config_dir: path for Postfix config dir - :param dict initial_values: initial Postfix config values - - """ - initial_values["config_directory"] = config_dir - - self.config_path = os.path.join(config_dir, "main.cf") - self._write_config(initial_values) - - def __call__(self, args, *unused_args, **unused_kwargs): - cmd = os.path.basename(args[0]) - if cmd == "postfix": - return - elif cmd != "postconf": # pragma: no cover - assert False, "Unexpected command '{0}'".format(''.join(args)) - - output = [] - - skip = False - for arg in args[1:]: - if skip: - skip = False - elif arg[0] == "-": - if arg == "-c": - skip = True - elif "=" in arg: - name, _, value = arg.partition("=") - self.set_value(name, value) - else: - output.append("{0} = {1}\n".format(arg, self.get_value(arg))) - - return "\n".join(output) - - def get_value(self, name): - """Returns the value for the Postfix config parameter name. - - If the value isn't set, an empty string is returned. - - :param str name: name of the Postfix config parameter - - :returns: value of the named parameter - :rtype: str - - """ - return self._read_config().get(name, "") - - def set_value(self, name, value): - """Sets the value for a Postfix config parameter. - - :param str name: name of the Postfix config parameter - :param str value: value ot set the parameter to - - """ - config = self._read_config() - config[name] = value - self._write_config(config) - - def _read_config(self): - config = {} - with open(self.config_path) as f: - for line in f: - key, _, value = line.strip().partition(" = ") - config[key] = value - - return config - - def _write_config(self, config): - with open(self.config_path, "w") as f: - f.writelines("{0} = {1}\n".format(key, value) - for key, value in config.items()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/certbot-postfix/certbot_postfix/postconf.py b/certbot-postfix/certbot_postfix/postconf.py index 7738562dd..466e0e63e 100644 --- a/certbot-postfix/certbot_postfix/postconf.py +++ b/certbot-postfix/certbot_postfix/postconf.py @@ -1,62 +1,152 @@ """Classes that wrap the postconf command line utility. - -These classes allow you to interact with a Postfix config like it is a -dictionary, with the getting and setting of values in the config being -handled automatically by the class. - """ -import collections - +import six +from certbot import errors from certbot_postfix import util +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, List, Tuple +# pylint: enable=unused-import, no-name-in-module -class ReadOnlyMainMap(util.PostfixUtilBase, collections.Mapping): - """A read-only view of a Postfix main.cf file.""" +class ConfigMain(util.PostfixUtilBase): + """A parser for Postfix's main.cf file.""" - _modifiers = None - """An iterable containing additional CLI flags for postconf.""" - - def __getitem__(self, name): - return next(_parse_main_output(self._get_output([name])))[1] - - def __iter__(self): - for name, _ in _parse_main_output(self._get_output()): - yield name - - def __len__(self): - return sum(1 for _ in _parse_main_output(self._get_output())) - - def _call(self, extra_args=None): - """Runs Postconf and returns the result. - - If self._modifiers is set, it is provided on the command line to - postconf before any values in extra_args. - - :param list extra_args: additional arguments for the command - - :returns: data written to stdout and stderr - :rtype: `tuple` of `str` - - :raises subprocess.CalledProcessError: if the command fails + def __init__(self, executable, ignore_master_overrides=False, config_dir=None): + super(ConfigMain, self).__init__(executable, config_dir) + # Whether to ignore overrides from master. + self._ignore_master_overrides = ignore_master_overrides + # List of all current Postfix parameters, from `postconf` command. + self._db = {} # type: Dict[str, str] + # List of current master.cf overrides from Postfix config. Dictionary + # of parameter name => list of tuples (service name, paramter value) + # Note: We should never modify master without explicit permission. + self._master_db = {} # type: Dict[str, List[Tuple[str, str]]] + # List of all changes requested to the Postfix parameters as they are now + # in _db. These changes are flushed to `postconf` on `flush`. + self._updated = {} # type: Dict[str, str] + self._read_from_conf() + def _read_from_conf(self): + """Reads initial parameter state from `main.cf` into this object. """ - all_extra_args = [] - for args_list in (self._modifiers, extra_args,): - if args_list is not None: - all_extra_args.extend(args_list) + out = self._get_output() + for name, value in _parse_main_output(out): + self._db[name] = value + out = self._get_output_master() + for name, value in _parse_main_output(out): + service, param_name = name.rsplit("/", 1) + if param_name not in self._master_db: + self._master_db[param_name] = [] + self._master_db[param_name].append((service, value)) - return super(ReadOnlyMainMap, self)._call(all_extra_args) + def _get_output_master(self): + """Retrieves output for `master.cf` parameters.""" + return self._get_output('-P') + def get_default(self, name): + """Retrieves default value of parameter `name` from postfix parameters. + + :param str name: The name of the parameter to fetch. + :returns: The default value of parameter `name`. + :rtype: str + """ + out = self._get_output(['-d', name]) + _, value = next(_parse_main_output(out), (None, None)) + return value + + def get(self, name): + """Retrieves working value of parameter `name` from postfix parameters. + + :param str name: The name of the parameter to fetch. + :returns: The value of parameter `name`. + :rtype: str + """ + if name in self._updated: + return self._updated[name] + return self._db[name] + + def get_master_overrides(self, name): + """Retrieves list of overrides for parameter `name` in postfix's Master config + file. + + :returns: List of tuples (service, value), meaning that parameter `name` + is overridden as `value` for `service`. + :rtype: `list` of `tuple` of `str` + """ + if name in self._master_db: + return self._master_db[name] + return None + + def set(self, name, value, acceptable_overrides=None): + """Sets parameter `name` to `value`. If `name` is overridden by a particular service in + `master.cf`, reports any of these parameter conflicts as long as + `ignore_master_overrides` was not set. + + .. note:: that this function does not flush these parameter values to main.cf; + To do that, use `flush`. + + :param str name: The name of the parameter to set. + :param str value: The value of the parameter. + :param tuple acceptable_overrides: If the master configuration file overrides `value` + with a value in acceptable_overrides. + """ + if name not in self._db: + raise KeyError("Parameter name %s is not a valid Postfix parameter name.", name) + # Check to see if this parameter is overridden by master. + overrides = self.get_master_overrides(name) + if not self._ignore_master_overrides and overrides is not None: + util.report_master_overrides(name, overrides, acceptable_overrides) + if value != self._db[name]: + # _db contains the "original" state of parameters. We only care about + # writes if they cause a delta from the original state. + self._updated[name] = value + elif name in self._updated: + # If this write reverts a previously updated parameter back to the + # original DB's state, we don't have to keep track of it in _updated. + del self._updated[name] + + def flush(self): + """Flushes all parameter changes made using `self.set`, to `main.cf` + + :raises error.PluginError: When flush to main.cf fails for some reason. + """ + if len(self._updated) == 0: + return + args = ['-e'] + for name, value in six.iteritems(self._updated): + args.append('{0}={1}'.format(name, value)) + try: + self._get_output(args) + except IOError as e: + raise errors.PluginError("Unable to save to Postfix config: %v", e) + for name, value in six.iteritems(self._updated): + self._db[name] = value + self._updated = {} + + def get_changes(self): + """ Return queued changes to main.cf. + + :rtype: dict[str, str] + """ + return self._updated def _parse_main_output(output): """Parses the raw output from Postconf about main.cf. + Expects the output to look like: + + .. code-block:: none + + name1 = value1 + name2 = value2 + :param str output: data postconf wrote to stdout about main.cf :returns: generator providing key-value pairs from main.cf - :rtype: generator - + :rtype: Iterator[tuple(str, str)] """ for line in output.splitlines(): name, _, value = line.partition(" =") yield name, value.strip() + + diff --git a/certbot-postfix/certbot_postfix/tests/__init__.py b/certbot-postfix/certbot_postfix/tests/__init__.py new file mode 100644 index 000000000..7316b5888 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/__init__.py @@ -0,0 +1 @@ +""" Certbot Postfix Tests """ diff --git a/certbot-postfix/certbot_postfix/tests/installer_test.py b/certbot-postfix/certbot_postfix/tests/installer_test.py new file mode 100644 index 000000000..1bdd2c8b3 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/installer_test.py @@ -0,0 +1,314 @@ +"""Tests for certbot_postfix.installer.""" +from contextlib import contextmanager +import copy +import functools +import os +import pkg_resources +import six +import unittest + +import mock + +from certbot import errors +from certbot.tests import util as certbot_test_util + +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Tuple, Union +# pylint: enable=unused-import, no-name-in-module + +DEFAULT_MAIN_CF = { + "smtpd_tls_cert_file": "", + "smtpd_tls_key_file": "", + "smtpd_tls_dh1024_param_file": "", + "smtpd_tls_security_level": "none", + "smtpd_tls_auth_only": "", + "smtpd_tls_mandatory_protocols": "", + "smtpd_tls_protocols": "", + "smtpd_tls_ciphers": "", + "smtpd_tls_exclude_ciphers": "", + "smtpd_tls_mandatory_ciphers": "", + "smtpd_tls_eecdh_grade": "medium", + "smtp_tls_security_level": "", + "smtp_tls_ciphers": "", + "smtp_tls_exclude_ciphers": "", + "smtp_tls_mandatory_ciphers": "", + "mail_version": "3.2.3" +} + +def _main_cf_with(obj): + main_cf = copy.copy(DEFAULT_MAIN_CF) + main_cf.update(obj) + return main_cf + +class InstallerTest(certbot_test_util.ConfigTestCase): + # pylint: disable=too-many-public-methods + + def setUp(self): + super(InstallerTest, self).setUp() + _config_file = pkg_resources.resource_filename("certbot_postfix.tests", + os.path.join("testdata", "config.json")) + self.config.postfix_ctl = "postfix" + self.config.postfix_config_dir = self.tempdir + self.config.postfix_config_utility = "postconf" + self.config.postfix_tls_only = False + self.config.postfix_server_only = False + self.config.config_dir = self.tempdir + + @mock.patch("certbot_postfix.installer.util.is_acceptable_value") + def test_set_vars(self, mock_is_acceptable_value): + mock_is_acceptable_value.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + mock_is_acceptable_value.return_value = False + + @mock.patch("certbot_postfix.installer.util.is_acceptable_value") + def test_acceptable_value(self, mock_is_acceptable_value): + mock_is_acceptable_value.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + mock_is_acceptable_value.return_value = False + + @certbot_test_util.patch_get_utility() + def test_confirm_changes_no_raises_error(self, mock_util): + mock_util().yesno.return_value = False + with create_installer(self.config) as installer: + installer.prepare() + self.assertRaises(errors.PluginError, installer.deploy_cert, + "example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + + @certbot_test_util.patch_get_utility() + def test_save(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.postconf.flush = mock.Mock() + installer.reverter = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.save() + self.assertEqual(installer.save_notes, []) + self.assertEqual(installer.postconf.flush.call_count, 1) + self.assertEqual(installer.reverter.add_to_checkpoint.call_count, 1) + + @certbot_test_util.patch_get_utility() + def test_save_with_title(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.postconf.flush = mock.Mock() + installer.reverter = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.save(title="new_file!") + self.assertEqual(installer.reverter.finalize_checkpoint.call_count, 1) + + @certbot_test_util.patch_get_utility() + def test_rollback_checkpoints_resets_postconf(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.rollback_checkpoints() + self.assertEqual(installer.postconf.get_changes(), {}) + + @certbot_test_util.patch_get_utility() + def test_recovery_routine_resets_postconf(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + installer.recovery_routine() + self.assertEqual(installer.postconf.get_changes(), {}) + + def test_restart(self): + with create_installer(self.config) as installer: + installer.prepare() + installer.restart() + self.assertEqual(installer.postfix.restart.call_count, 1) + + def test_add_parser_arguments(self): + options = set(("ctl", "config-dir", "config-utility", + "tls-only", "server-only", "ignore-master-overrides")) + mock_add = mock.MagicMock() + + from certbot_postfix import installer + installer.Installer.add_parser_arguments(mock_add) + + for call in mock_add.call_args_list: + self.assertTrue(call[0][0] in options) + + def test_no_postconf_prepare(self): + with create_installer(self.config) as installer: + installer_path = "certbot_postfix.installer" + exe_exists_path = installer_path + ".certbot_util.exe_exists" + path_surgery_path = "certbot_postfix.util.plugins_util.path_surgery" + with mock.patch(path_surgery_path, return_value=False): + with mock.patch(exe_exists_path, return_value=False): + self.assertRaises(errors.NoInstallationError, + installer.prepare) + + def test_old_version(self): + with create_installer(self.config, main_cf=_main_cf_with({"mail_version": "0.0.1"}))\ + as installer: + self.assertRaises(errors.NotSupportedError, installer.prepare) + + def test_lock_error(self): + with create_installer(self.config) as installer: + assert_raises = functools.partial(self.assertRaises, + errors.PluginError, + installer.prepare) + certbot_test_util.lock_and_call(assert_raises, self.tempdir) + + + @mock.patch('certbot.util.lock_dir_until_exit') + def test_dir_locked(self, lock_dir): + with create_installer(self.config) as installer: + lock_dir.side_effect = errors.LockError + self.assertRaises(errors.PluginError, installer.prepare) + + def test_more_info(self): + with create_installer(self.config) as installer: + installer.prepare() + output = installer.more_info() + self.assertTrue("Postfix" in output) + self.assertTrue(self.tempdir in output) + self.assertTrue(DEFAULT_MAIN_CF["mail_version"] in output) + + def test_get_all_names(self): + config = {"mydomain": "example.org", + "myhostname": "mail.example.org", + "myorigin": "example.org"} + with create_installer(self.config, main_cf=_main_cf_with(config)) as installer: + installer.prepare() + result = installer.get_all_names() + self.assertEqual(result, set(config.values())) + + @certbot_test_util.patch_get_utility() + def test_deploy(self, mock_util): + mock_util().yesno.return_value = True + from certbot_postfix import constants + with create_installer(self.config) as installer: + installer.prepare() + + # pylint: disable=protected-access + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + changes = installer.postconf.get_changes() + expected = {} # type: Dict[str, Tuple[str, ...]] + expected.update(constants.TLS_SERVER_VARS) + expected.update(constants.DEFAULT_SERVER_VARS) + expected.update(constants.DEFAULT_CLIENT_VARS) + self.assertEqual(changes["smtpd_tls_key_file"], "key_path") + self.assertEqual(changes["smtpd_tls_cert_file"], "cert_path") + for name, value in six.iteritems(expected): + self.assertEqual(changes[name], value[0]) + + @certbot_test_util.patch_get_utility() + def test_tls_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: x == "tls_only" + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 4) + + @certbot_test_util.patch_get_utility() + def test_server_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: x == "server_only" + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 11) + + @certbot_test_util.patch_get_utility() + def test_tls_and_server_only(self, mock_util): + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + installer.conf = lambda x: True + installer.postconf.set = mock.Mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(installer.postconf.set.call_count, 3) + + @certbot_test_util.patch_get_utility() + def test_deploy_twice(self, mock_util): + # Deploying twice on the same installer shouldn't do anything! + mock_util().yesno.return_value = True + with create_installer(self.config) as installer: + installer.prepare() + from certbot_postfix.postconf import ConfigMain + with mock.patch.object(ConfigMain, "set", wraps=installer.postconf.set) as fake_set: + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + self.assertEqual(fake_set.call_count, 15) + fake_set.reset_mock() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + fake_set.assert_not_called() + + @certbot_test_util.patch_get_utility() + def test_deploy_already_secure(self, mock_util): + # Should not overwrite "more-secure" parameters + mock_util().yesno.return_value = True + more_secure = { + "smtpd_tls_security_level": "encrypt", + "smtpd_tls_protocols": "!SSLv3, !SSLv2, !TLSv1", + "smtpd_tls_eecdh_grade": "strong" + } + with create_installer(self.config,\ + main_cf=_main_cf_with(more_secure)) as installer: + installer.prepare() + installer.deploy_cert("example.com", "cert_path", "key_path", + "chain_path", "fullchain_path") + for param in more_secure.keys(): + self.assertFalse(param in installer.postconf.get_changes()) + + def test_enhance(self): + with create_installer(self.config) as installer: + installer.prepare() + self.assertRaises(errors.PluginError, + installer.enhance, + "example.org", "redirect") + + def test_supported_enhancements(self): + with create_installer(self.config) as installer: + installer.prepare() + self.assertEqual(installer.supported_enhancements(), []) + +@contextmanager +def create_installer(config, main_cf=DEFAULT_MAIN_CF): +# pylint: disable=dangerous-default-value + """Creates a Postfix installer with calls to `postconf` and `postfix` mocked out. + + In particular, creates a ConfigMain object that does regular things, but seeds it + with values from `main_cf` and `master_cf` dicts. + """ + from certbot_postfix.postconf import ConfigMain + from certbot_postfix import installer + def _mock_init_postconf(postconf, executable, ignore_master_overrides=False, config_dir=None): + # pylint: disable=protected-access,unused-argument + postconf._ignore_master_overrides = ignore_master_overrides + postconf._db = main_cf + postconf._master_db = {} + postconf._updated = {} + # override get_default to get from main + postconf.get_default = lambda name: main_cf[name] + with mock.patch.object(ConfigMain, "__init__", _mock_init_postconf): + exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists" + with mock.patch(exe_exists_path, return_value=True): + with mock.patch("certbot_postfix.installer.util.PostfixUtil", + return_value=mock.Mock()): + yield installer.Installer(config, "postfix") + +if __name__ == "__main__": + unittest.main() # pragma: no cover + diff --git a/certbot-postfix/certbot_postfix/tests/postconf_test.py b/certbot-postfix/certbot_postfix/tests/postconf_test.py new file mode 100644 index 000000000..91617d410 --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/postconf_test.py @@ -0,0 +1,107 @@ +"""Tests for certbot_postfix.postconf.""" + +import mock +import unittest + +from certbot import errors + +class PostConfTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostConf.""" + def setUp(self): + from certbot_postfix.postconf import ConfigMain + super(PostConfTest, self).setUp() + with mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') as mock_call: + with mock.patch('certbot_postfix.postconf.ConfigMain._get_output_master') as \ + mock_master_call: + with mock.patch('certbot_postfix.postconf.util.verify_exe_exists') as verify_exe: + verify_exe.return_value = True + mock_call.return_value = ('default_parameter = value\n' + 'extra_param =\n' + 'overridden_by_master = default\n') + mock_master_call.return_value = ( + 'service/type/overridden_by_master = master_value\n' + 'service2/type/overridden_by_master = master_value2\n' + ) + self.config = ConfigMain('postconf', False) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + @mock.patch('certbot_postfix.postconf.util.verify_exe_exists') + def test_get_output_master(self, mock_verify_exe, mock_get_output): + from certbot_postfix.postconf import ConfigMain + mock_verify_exe.return_value = True + ConfigMain('postconf', lambda x, y, z: None) + mock_get_output.assert_called_with('-P') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_read_default(self, mock_get_output): + mock_get_output.return_value = 'param = default_value' + self.assertEqual(self.config.get_default('param'), 'default_value') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_set(self, mock_call): + self.config.set('extra_param', 'other_value') + self.assertEqual(self.config.get('extra_param'), 'other_value') + self.config.flush() + mock_call.assert_called_with(['-e', 'extra_param=other_value']) + + def test_set_bad_param_name(self): + self.assertRaises(KeyError, self.config.set, 'nonexistent_param', 'some_value') + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_write_revert(self, mock_call): + self.config.set('default_parameter', 'fake_news') + # revert config set + self.config.set('default_parameter', 'value') + self.config.flush() + mock_call.assert_not_called() + + @mock.patch('certbot_postfix.util.PostfixUtilBase._call') + def test_write_default(self, mock_call): + self.config.set('default_parameter', 'value') + self.config.flush() + mock_call.assert_not_called() + + def test_master_overrides(self): + self.assertEqual(self.config.get_master_overrides('overridden_by_master'), + [('service/type', 'master_value'), + ('service2/type', 'master_value2')]) + + def test_set_check_override(self): + self.assertRaises(errors.PluginError, self.config.set, + 'overridden_by_master', 'new_value') + + def test_ignore_check_override(self): + # pylint: disable=protected-access + self.config._ignore_master_overrides = True + self.config.set('overridden_by_master', 'new_value') + + def test_check_acceptable_overrides(self): + self.config.set('overridden_by_master', 'new_value', + ('master_value', 'master_value2')) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush(self, mock_out): + self.config.set('default_parameter', 'new_value') + self.config.set('extra_param', 'another_value') + self.config.flush() + arguments = mock_out.call_args_list[-1][0][0] + self.assertEquals('-e', arguments[0]) + self.assertTrue('default_parameter=new_value' in arguments) + self.assertTrue('extra_param=another_value' in arguments) + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush_updates_object(self, mock_out): + self.config.set('default_parameter', 'new_value') + self.config.flush() + mock_out.reset_mock() + self.config.set('default_parameter', 'new_value') + mock_out.assert_not_called() + + @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') + def test_flush_throws_error_on_fail(self, mock_out): + mock_out.side_effect = [IOError("oh no!")] + self.config.set('default_parameter', 'new_value') + self.assertRaises(errors.PluginError, self.config.flush) + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/certbot-postfix/certbot_postfix/tests/util_test.py b/certbot-postfix/certbot_postfix/tests/util_test.py new file mode 100644 index 000000000..fa38f83ab --- /dev/null +++ b/certbot-postfix/certbot_postfix/tests/util_test.py @@ -0,0 +1,205 @@ +"""Tests for certbot_postfix.util.""" + +import subprocess +import unittest + +import mock + +from certbot import errors + + +class PostfixUtilBaseTest(unittest.TestCase): + """Tests for certbot_postfix.util.PostfixUtilBase.""" + + @classmethod + def _create_object(cls, *args, **kwargs): + from certbot_postfix.util import PostfixUtilBase + return PostfixUtilBase(*args, **kwargs) + + @mock.patch('certbot_postfix.util.verify_exe_exists') + def test_no_exe(self, mock_verify): + expected_error = errors.NoInstallationError + mock_verify.side_effect = expected_error + self.assertRaises(expected_error, self._create_object, 'nonexistent') + + def test_object_creation(self): + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self._create_object('existent') + + @mock.patch('certbot_postfix.util.check_all_output') + def test_call_extends_args(self, mock_output): + # pylint: disable=protected-access + with mock.patch('certbot_postfix.util.verify_exe_exists'): + mock_output.return_value = 'expected' + postfix = self._create_object('executable') + postfix._call(['many', 'extra', 'args']) + mock_output.assert_called_with(['executable', 'many', 'extra', 'args']) + postfix._call() + mock_output.assert_called_with(['executable']) + + def test_create_with_config(self): + # pylint: disable=protected-access + with mock.patch('certbot_postfix.util.verify_exe_exists'): + postfix = self._create_object('exec', 'config_dir') + self.assertEqual(postfix._base_command, ['exec', '-c', 'config_dir']) + +class PostfixUtilTest(unittest.TestCase): + def setUp(self): + # pylint: disable=protected-access + from certbot_postfix.util import PostfixUtil + with mock.patch('certbot_postfix.util.verify_exe_exists'): + self.postfix = PostfixUtil() + self.postfix._call = mock.Mock() + self.mock_call = self.postfix._call + + def test_test(self): + self.postfix.test() + self.mock_call.assert_called_with(['check']) + + def test_test_raises_error_when_check_fails(self): + self.mock_call.side_effect = [subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.MisconfigurationError, self.postfix.test) + self.mock_call.assert_called_with(['check']) + + def test_restart_while_running(self): + self.mock_call.side_effect = [subprocess.CalledProcessError(1, ""), None] + self.postfix.restart() + self.mock_call.assert_called_with(['start']) + + def test_restart_while_not_running(self): + self.postfix.restart() + self.mock_call.assert_called_with(['reload']) + + def test_restart_raises_error_when_reload_fails(self): + self.mock_call.side_effect = [None, subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.PluginError, self.postfix.restart) + self.mock_call.assert_called_with(['reload']) + + def test_restart_raises_error_when_start_fails(self): + self.mock_call.side_effect = [ + subprocess.CalledProcessError(1, ""), + subprocess.CalledProcessError(1, "")] + self.assertRaises(errors.PluginError, self.postfix.restart) + self.mock_call.assert_called_with(['start']) + +class CheckAllOutputTest(unittest.TestCase): + """Tests for certbot_postfix.util.check_all_output.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import check_all_output + return check_all_output(*args, **kwargs) + + @mock.patch('certbot_postfix.util.logger') + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_command_error(self, mock_popen, mock_logger): + command = 'foo' + retcode = 42 + output = 'bar' + err = 'baz' + + mock_popen().communicate.return_value = (output, err) + mock_popen().poll.return_value = 42 + + self.assertRaises(subprocess.CalledProcessError, self._call, command) + log_args = mock_logger.debug.call_args[0] + for value in (command, retcode, output, err,): + self.assertTrue(value in log_args) + + @mock.patch('certbot_postfix.util.subprocess.Popen') + def test_success(self, mock_popen): + command = 'foo' + expected = ('bar', '') + mock_popen().communicate.return_value = expected + mock_popen().poll.return_value = 0 + + self.assertEqual(self._call(command), expected) + + def test_stdout_error(self): + self.assertRaises(ValueError, self._call, stdout=None) + + def test_stderr_error(self): + self.assertRaises(ValueError, self._call, stderr=None) + + def test_universal_newlines_error(self): + self.assertRaises(ValueError, self._call, universal_newlines=False) + + +class VerifyExeExistsTest(unittest.TestCase): + """Tests for certbot_postfix.util.verify_exe_exists.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot_postfix.util import verify_exe_exists + return verify_exe_exists(*args, **kwargs) + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_failure(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = mock_path_surgery.return_value = False + self.assertRaises(errors.NoInstallationError, self._call, 'foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + def test_simple_success(self, mock_exe_exists): + mock_exe_exists.return_value = True + self._call('foo') + + @mock.patch('certbot_postfix.util.certbot_util.exe_exists') + @mock.patch('certbot_postfix.util.plugins_util.path_surgery') + def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): + mock_exe_exists.return_value = False + mock_path_surgery.return_value = True + self._call('foo') + +class TestUtils(unittest.TestCase): + """ Testing random utility functions in util.py + """ + def test_report_master_overrides(self): + from certbot_postfix.util import report_master_overrides + self.assertRaises(errors.PluginError, report_master_overrides, 'name', + [('service/type', 'value')]) + # Shouldn't raise error + report_master_overrides('name', [('service/type', 'value')], + acceptable_overrides=('value',)) + + def test_no_acceptable_value(self): + from certbot_postfix.util import is_acceptable_value + self.assertFalse(is_acceptable_value('name', 'value', None)) + + def test_is_acceptable_value(self): + from certbot_postfix.util import is_acceptable_value + self.assertTrue(is_acceptable_value('name', 'value', ('value',))) + self.assertFalse(is_acceptable_value('name', 'bad', ('value',))) + + def test_is_acceptable_tuples(self): + from certbot_postfix.util import is_acceptable_value + self.assertTrue(is_acceptable_value('name', 'value', ('value', 'value1'))) + self.assertFalse(is_acceptable_value('name', 'bad', ('value', 'value1'))) + + def test_is_acceptable_protocols(self): + from certbot_postfix.util import is_acceptable_value + # SSLv2 and SSLv3 are both not supported, unambiguously + self.assertFalse(is_acceptable_value('tls_mandatory_protocols_lol', + 'SSLv2, SSLv3', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'SSLv2, SSLv3', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + '!SSLv2, !TLSv1', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + '!SSLv2, SSLv3, !SSLv3, ', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + '!SSLv2, !SSLv3', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + '!SSLv3, !TLSv1, !SSLv2', None)) + # TLSv1.2 is supported unambiguously + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'TLSv1, TLSv1.1,', None)) + self.assertFalse(is_acceptable_value('tls_protocols_lol', + 'TLSv1.2, !TLSv1.2,', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + 'TLSv1.2, ', None)) + self.assertTrue(is_acceptable_value('tls_protocols_lol', + 'TLSv1, TLSv1.1, TLSv1.2', None)) + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/certbot-postfix/certbot_postfix/util.py b/certbot-postfix/certbot_postfix/util.py index 57196017f..f06989903 100644 --- a/certbot-postfix/certbot_postfix/util.py +++ b/certbot-postfix/certbot_postfix/util.py @@ -1,14 +1,16 @@ """Utility functions for use in the Postfix installer.""" import logging +import re import subprocess from certbot import errors from certbot import util as certbot_util from certbot.plugins import util as plugins_util - +from certbot_postfix import constants logger = logging.getLogger(__name__) +COMMAND = "postfix" class PostfixUtilBase(object): """A base class for wrapping Postfix command line utilities.""" @@ -22,9 +24,13 @@ class PostfixUtilBase(object): :raises .NoInstallationError: when the executable isn't found """ + self.executable = executable verify_exe_exists(executable) + self._set_base_command(config_dir) + self.config_dir = None - self._base_command = [executable] + def _set_base_command(self, config_dir): + self._base_command = [self.executable] if config_dir is not None: self._base_command.extend(('-c', config_dir,)) @@ -59,6 +65,77 @@ class PostfixUtilBase(object): """ return self._call(extra_args)[0] +class PostfixUtil(PostfixUtilBase): + """Wrapper around Postfix CLI tool. + """ + + def __init__(self, config_dir=None): + super(PostfixUtil, self).__init__(COMMAND, config_dir) + + def test(self): + """Make sure the configuration is valid. + + :raises .MisconfigurationError: if the config is invalid + """ + try: + self._call(["check"]) + except subprocess.CalledProcessError as e: + logger.debug("Could not check postfix configuration:\n%s", e) + raise errors.MisconfigurationError( + "Postfix failed internal configuration check.") + + def restart(self): + """Restart or refresh the server content. + + :raises .PluginError: when server cannot be restarted + + """ + logger.info("Reloading Postfix configuration...") + if self._is_running(): + self._reload() + else: + self._start() + + + def _is_running(self): + """Is Postfix currently running? + + Uses the 'postfix status' command to determine if Postfix is + currently running using the specified configuration files. + + :returns: True if Postfix is running, otherwise, False + :rtype: bool + + """ + try: + self._call(["status"]) + except subprocess.CalledProcessError: + return False + return True + + def _start(self): + """Instructions Postfix to start running. + + :raises .PluginError: when Postfix cannot start + + """ + try: + self._call(["start"]) + except subprocess.CalledProcessError: + raise errors.PluginError("Postfix failed to start") + + def _reload(self): + """Instructs Postfix to reload its configuration. + + If Postfix isn't currently running, this method will fail. + + :raises .PluginError: when Postfix cannot reload + """ + try: + self._call(["reload"]) + except subprocess.CalledProcessError: + raise errors.PluginError( + "Postfix failed to reload its configuration") def check_all_output(*args, **kwargs): """A version of subprocess.check_output that also captures stderr. @@ -107,7 +184,7 @@ def check_all_output(*args, **kwargs): return (output, err) -def verify_exe_exists(exe): +def verify_exe_exists(exe, message=None): """Ensures an executable with the given name is available. If an executable isn't found for the given path or name, extra @@ -115,83 +192,101 @@ def verify_exe_exists(exe): utilities that may not be available in the default cron PATH. :param str exe: executable path or name + :param str message: Error message to print. :raises .NoInstallationError: when the executable isn't found """ + if message is None: + message = "Cannot find executable '{0}'.".format(exe) if not (certbot_util.exe_exists(exe) or plugins_util.path_surgery(exe)): - raise errors.NoInstallationError( - "Cannot find executable '{0}'.".format(exe)) + raise errors.NoInstallationError(message) +def report_master_overrides(name, overrides, acceptable_overrides=None): + """If the value for a parameter `name` is overridden by other services, + report a warning to notify the user. If `parameter` is a TLS version parameter + (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then + `acceptable_overrides` isn't used each value in overrides is inspected for secure TLS + versions. -def check_call(*args, **kwargs): - """A simple wrapper of subprocess.check_call that logs errors. - - :param tuple args: positional arguments to subprocess.check_call - :param dict kargs: keyword arguments to subprocess.check_call - - :raises subprocess.CalledProcessError: if the call fails - + :param str name: The name of the parameter that is being overridden. + :param list overrides: The values that other services are setting for `name`. + Each override is a tuple: (service name, value) + :param tuple acceptable_overrides: Override values that are acceptable. For instance, if + another service is overriding our parameter with a more secure option, we don't have + to warn. If this is set to None, errors are raised for *any* overrides of `name`! """ - try: - subprocess.check_call(*args, **kwargs) - except subprocess.CalledProcessError: - cmd = _get_cmd(*args, **kwargs) - logger.debug("%s exited with a non-zero status.", - "".join(cmd), exc_info=True) - raise + error_string = "" + for override in overrides: + service, value = override + # If this override is acceptable: + if acceptable_overrides is not None and \ + is_acceptable_value(name, value, acceptable_overrides): + continue + error_string += " {0}: {1}\n".format(service, value) + if error_string: + raise errors.PluginError("{0} is overridden with less secure options by the " + "following services in master.cf:\n".format(name) + error_string) -def check_output(*args, **kwargs): - """Backported version of subprocess.check_output for Python 2.6+. - - This is the same as subprocess.check_output from newer versions of - Python, except: - - 1. The return value is a string rather than a byte string. To - accomplish this, the caller cannot set the parameter - universal_newlines. - 2. If the command exits with a nonzero status, output is not - included in the raised subprocess.CalledProcessError because - subprocess.CalledProcessError on Python 2.6 does not support this. - Instead, the failure including the output is logged. - - :param tuple args: positional arguments for Popen - :param dict kwargs: keyword arguments for Popen - - :returns: data printed to stdout - :rtype: str - - :raises ValueError: if arguments are invalid - :raises subprocess.CalledProcessError: if the command fails +def is_acceptable_value(parameter, value, acceptable=None): + """ Returns whether the `value` for this `parameter` is acceptable, + given a tuple of `acceptable` values. If `parameter` is a TLS version parameter + (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then + `acceptable` isn't used and `value` is inspected for secure TLS versions. + :param str parameter: The name of the parameter being set. + :param str value: Proposed new value for parameter. + :param tuple acceptable: List of acceptable values for parameter. """ - for keyword in ('stdout', 'universal_newlines',): - if keyword in kwargs: - raise ValueError( - keyword + ' argument not allowed, it will be overridden.') - - kwargs['stdout'] = subprocess.PIPE - kwargs['universal_newlines'] = True - - process = subprocess.Popen(*args, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = _get_cmd(*args, **kwargs) - logger.debug( - "'%s' exited with %d. Output was:\n%s", - cmd, retcode, output, exc_info=True) - raise subprocess.CalledProcessError(retcode, cmd) - return output + # Check if param value is a comma-separated list of protocols. + # Otherwise, just check whether the value is in the acceptable list. + if 'tls_protocols' in parameter or 'tls_mandatory_protocols' in parameter: + return _has_acceptable_tls_versions(value) + if acceptable is not None: + return value in acceptable + return False -def _get_cmd(*args, **kwargs): - """Return the command from Popen args. - - :param tuple args: Popen args - :param dict kwargs: Popen kwargs - +def _has_acceptable_tls_versions(parameter_string): """ - cmd = kwargs.get('args') - return args[0] if cmd is None else cmd + Checks to see if the list of TLS protocols is acceptable. + This requires that TLSv1.2 is supported, and neither SSLv2 nor SSLv3 are supported. + + Should be a string of protocol names delimited by commas, spaces, or colons. + + Postfix's documents suggest listing protocols to exclude, like "!SSLv2, !SSLv3". + Listing the protocols to include, like "TLSv1, TLSv1.1, TLSv1.2" is okay as well, + though not recommended + + When these two modes are interspersed, the presence of a single non-negated protocol name + (i.e. "TLSv1" rather than "!TLSv1") automatically excludes all other unnamed protocols. + + In addition, the presence of both a protocol name inclusion and exclusion isn't explicitly + documented, so this method should return False if it encounters contradicting statements + about TLSv1.2, SSLv2, or SSLv3. (for instance, "SSLv3, !SSLv3"). + """ + if not parameter_string: + return False + bad_versions = list(constants.TLS_VERSIONS) + for version in constants.ACCEPTABLE_TLS_VERSIONS: + del bad_versions[bad_versions.index(version)] + supported_version_list = re.split("[, :]+", parameter_string) + # The presence of any non-"!" protocol listing excludes the others by default. + inclusion_list = False + for version in supported_version_list: + if not version: + continue + if version in bad_versions: # short-circuit if we recognize any bad version + return False + if version[0] != "!": + inclusion_list = True + if inclusion_list: # For any inclusion list, we still require TLS 1.2. + if "TLSv1.2" not in supported_version_list or "!TLSv1.2" in supported_version_list: + return False + else: + for bad_version in bad_versions: + if "!" + bad_version not in supported_version_list: + return False + return True + diff --git a/certbot-postfix/certbot_postfix/util_test.py b/certbot-postfix/certbot_postfix/util_test.py deleted file mode 100644 index 4a014ca9b..000000000 --- a/certbot-postfix/certbot_postfix/util_test.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Tests for certbot_postfix.util.""" - -import subprocess -import unittest - -import mock - -from certbot import errors - - -class PostfixUtilBaseTest(unittest.TestCase): - """Tests for certbot_postfix.util.PostfixUtilBase.""" - - @classmethod - def _create_object(cls, *args, **kwargs): - from certbot_postfix.util import PostfixUtilBase - return PostfixUtilBase(*args, **kwargs) - - @mock.patch('certbot_postfix.util.verify_exe_exists') - def test_no_exe(self, mock_verify): - expected_error = errors.NoInstallationError - mock_verify.side_effect = expected_error - self.assertRaises(expected_error, self._create_object, 'nonexistent') - - def test_object_creation(self): - with mock.patch('certbot_postfix.util.verify_exe_exists'): - self._create_object('existent') - - -class CheckAllOutputTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_all_output.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_all_output - return check_all_output(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_command_error(self, mock_popen, mock_logger): - command = 'foo' - retcode = 42 - output = 'bar' - err = 'baz' - - mock_popen().communicate.return_value = (output, err) - mock_popen().poll.return_value = 42 - - self.assertRaises(subprocess.CalledProcessError, self._call, command) - log_args = mock_logger.debug.call_args[0] - for value in (command, retcode, output, err,): - self.assertTrue(value in log_args) - - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_success(self, mock_popen): - command = 'foo' - expected = ('bar', '') - mock_popen().communicate.return_value = expected - mock_popen().poll.return_value = 0 - - self.assertEqual(self._call(command), expected) - - def test_stdout_error(self): - self.assertRaises(ValueError, self._call, stdout=None) - - def test_stderr_error(self): - self.assertRaises(ValueError, self._call, stderr=None) - - def test_universal_newlines_error(self): - self.assertRaises(ValueError, self._call, universal_newlines=False) - - -class VerifyExeExistsTest(unittest.TestCase): - """Tests for certbot_postfix.util.verify_exe_exists.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import verify_exe_exists - return verify_exe_exists(*args, **kwargs) - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - @mock.patch('certbot_postfix.util.plugins_util.path_surgery') - def test_failure(self, mock_exe_exists, mock_path_surgery): - mock_exe_exists.return_value = mock_path_surgery.return_value = False - self.assertRaises(errors.NoInstallationError, self._call, 'foo') - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - def test_simple_success(self, mock_exe_exists): - mock_exe_exists.return_value = True - self._call('foo') - - @mock.patch('certbot_postfix.util.certbot_util.exe_exists') - @mock.patch('certbot_postfix.util.plugins_util.path_surgery') - def test_successful_surgery(self, mock_exe_exists, mock_path_surgery): - mock_exe_exists.return_value = False - mock_path_surgery.return_value = True - self._call('foo') - - -class CheckCallTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_call.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_call - return check_call(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.check_call') - def test_failure(self, mock_check_call, mock_logger): - cmd = "postconf smtpd_use_tls=yes".split() - mock_check_call.side_effect = subprocess.CalledProcessError(42, cmd) - self.assertRaises(subprocess.CalledProcessError, self._call, cmd) - self.assertTrue(mock_logger.method_calls) - - @mock.patch('certbot_postfix.util.subprocess.check_call') - def test_success(self, mock_check_call): - cmd = "postconf smtpd_use_tls=yes".split() - self._call(cmd) - mock_check_call.assert_called_once_with(cmd) - - -class CheckOutputTest(unittest.TestCase): - """Tests for certbot_postfix.util.check_output.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot_postfix.util import check_output - return check_output(*args, **kwargs) - - @mock.patch('certbot_postfix.util.logger') - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_command_error(self, mock_popen, mock_logger): - command = 'foo' - retcode = 42 - output = 'bar' - - mock_popen().communicate.return_value = (output, '') - mock_popen().poll.return_value = 42 - - self.assertRaises(subprocess.CalledProcessError, self._call, command) - - log_args = mock_logger.debug.call_args[0] - self.assertTrue(command in log_args) - self.assertTrue(retcode in log_args) - self.assertTrue(output in log_args) - - @mock.patch('certbot_postfix.util.subprocess.Popen') - def test_success(self, mock_popen): - command = 'foo' - output = 'bar' - mock_popen().communicate.return_value = (output, '') - mock_popen().poll.return_value = 0 - - self.assertEqual(self._call(command), output) - - def test_stdout_error(self): - self.assertRaises(ValueError, self._call, stdout=None) - - def test_universal_newlines_error(self): - self.assertRaises(ValueError, self._call, universal_newlines=False) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/certbot-postfix/docs/.gitignore b/certbot-postfix/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-postfix/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-postfix/docs/Makefile b/certbot-postfix/docs/Makefile new file mode 100644 index 000000000..717ff654f --- /dev/null +++ b/certbot-postfix/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-postfix +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-postfix/docs/api.rst b/certbot-postfix/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-postfix/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-postfix/docs/api/installer.rst b/certbot-postfix/docs/api/installer.rst new file mode 100644 index 000000000..121d58d5b --- /dev/null +++ b/certbot-postfix/docs/api/installer.rst @@ -0,0 +1,5 @@ +:mod:`certbot_postfix.installer` +-------------------------------------- + +.. automodule:: certbot_postfix.installer + :members: diff --git a/certbot-postfix/docs/api/postconf.rst b/certbot-postfix/docs/api/postconf.rst new file mode 100644 index 000000000..917150e45 --- /dev/null +++ b/certbot-postfix/docs/api/postconf.rst @@ -0,0 +1,5 @@ +:mod:`certbot_postfix.postconf` +------------------------------- + +.. automodule:: certbot_postfix.postconf + :members: diff --git a/certbot-postfix/docs/conf.py b/certbot-postfix/docs/conf.py new file mode 100644 index 000000000..51d99aab5 --- /dev/null +++ b/certbot-postfix/docs/conf.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'certbot-postfix' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The short X.Y version +version = u'0' +# The full version, including alpha/beta/rc tags +release = u'0' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', +] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = u'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-postfixdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-postfix.tex', u'certbot-postfix Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-postfix', u'certbot-postfix Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-postfix', u'certbot-postfix Documentation', + author, 'certbot-postfix', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/certbot-postfix/docs/index.rst b/certbot-postfix/docs/index.rst new file mode 100644 index 000000000..3d6697bcb --- /dev/null +++ b/certbot-postfix/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-postfix documentation master file, created by + sphinx-quickstart on Wed May 2 16:01:06 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-postfix's documentation! +=========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: certbot_postfix + :members: + +.. toctree:: + :maxdepth: 1 + + api + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-postfix/docs/make.bat b/certbot-postfix/docs/make.bat new file mode 100644 index 000000000..23fbdc93c --- /dev/null +++ b/certbot-postfix/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-postfix + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-postfix/local-oldest-requirements.txt b/certbot-postfix/local-oldest-requirements.txt new file mode 100644 index 000000000..bc0cdbf00 --- /dev/null +++ b/certbot-postfix/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.25.0 +certbot[dev]==0.23.0 diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 2fdcd46aa..cde1ac7c2 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -1,21 +1,23 @@ -import sys - from setuptools import setup from setuptools import find_packages -version = '0.18.0.dev0' +version = '0.26.0.dev0' install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'acme>=0.25.0', + 'certbot>0.23.0', + 'setuptools', 'six', + 'zope.component', 'zope.interface', ] +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + setup( name='certbot-postfix', version=version, @@ -24,6 +26,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -32,10 +35,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -50,6 +51,9 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, entry_points={ 'certbot.plugins': [ 'postfix = certbot_postfix:Installer', diff --git a/certbot/constants.py b/certbot/constants.py index dfdfcc0e8..cd77fb72d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,7 +64,7 @@ RENEWER_DEFAULTS = dict( """Defaults for renewer script.""" -ENHANCEMENTS = ["redirect", "http-header", "ocsp-stapling", "spdy"] +ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling", "spdy", "starttls-policy"] """List of possible :class:`certbot.interfaces.IInstaller` enhancements. diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..01b6ad5e9 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -39,6 +39,7 @@ class PluginEntryPoint(object): "certbot-dns-rfc2136", "certbot-dns-route53", "certbot-nginx", + "certbot-postfix", ] """Distributions for which prefix will be omitted.""" From 3877af6619f690e64fa750f83e12c7a744336a25 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 21 Jun 2018 17:27:19 +0300 Subject: [PATCH 258/639] Gradually increasing HSTS max-age (#5912) This PR adds the functionality to enhance Apache configuration to include HTTP Strict Transport Security header with a low initial max-age value. The max-age value will get increased on every (scheduled) run of certbot renew regardless of the certificate actually getting renewed, if the last increase took place longer than ten hours ago. The increase steps are visible in constants.AUTOHSTS_STEPS. Upon the first actual renewal after reaching the maximum increase step, the max-age value will be made "permanent" and will get value of one year. To achieve accurate VirtualHost discovery on subsequent runs, a comment with unique id string will be added to each enhanced VirtualHost. * AutoHSTS code rebased on master * Fixes to match the changes in master * Make linter happy with metaclass registration * Address small review comments * Use new enhancement interfaces * New style enhancement changes * Do not allow --hsts and --auto-hsts simultaneuously * MyPy annotation fixes and added test * Change oldest requrements to point to local certbot core version * Enable new style enhancements for run and install verbs * Test refactor * New test class for main.install tests * Move a test to a correct test class --- certbot-apache/certbot_apache/apache_util.py | 6 + certbot-apache/certbot_apache/configurator.py | 312 +++++++++++++++++- certbot-apache/certbot_apache/constants.py | 13 + certbot-apache/certbot_apache/parser.py | 32 ++ .../certbot_apache/tests/autohsts_test.py | 181 ++++++++++ .../certbot_apache/tests/configurator_test.py | 15 + .../certbot_apache/tests/parser_test.py | 7 + certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 2 +- certbot/cli.py | 8 + certbot/constants.py | 1 + certbot/main.py | 44 ++- certbot/plugins/enhancements.py | 159 +++++++++ certbot/plugins/enhancements_test.py | 65 ++++ certbot/tests/main_test.py | 82 ++++- certbot/tests/renewupdater_test.py | 87 +++-- certbot/updater.py | 46 +++ 17 files changed, 1027 insertions(+), 35 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/autohsts_test.py create mode 100644 certbot/plugins/enhancements.py create mode 100644 certbot/plugins/enhancements_test.py diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index f03c9da87..62342004f 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -1,4 +1,5 @@ """ Utility functions for certbot-apache plugin """ +import binascii import os from certbot import util @@ -98,3 +99,8 @@ def parse_define_file(filepath, varname): var_parts = v[2:].partition("=") return_vars[var_parts[0]] = var_parts[2] return return_vars + + +def unique_id(): + """ Returns an unique id to be used as a VirtualHost identifier""" + return binascii.hexlify(os.urandom(16)).decode("utf-8") diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 861fe4458..ab83a5332 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -13,7 +13,7 @@ import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict, Dict, List, Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, DefaultDict, Dict, List, Set, Union # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces @@ -22,6 +22,7 @@ from certbot import util from certbot.achallenges import KeyAuthorizationAnnotatedChallenge # pylint: disable=unused-import from certbot.plugins import common from certbot.plugins.util import path_surgery +from certbot.plugins.enhancements import AutoHSTSEnhancement from certbot_apache import apache_util from certbot_apache import augeas_configurator @@ -160,6 +161,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + # Temporary state for AutoHSTS enhancement + self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] # These will be set in the prepare function self.parser = None @@ -1472,6 +1475,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if need_to_save: self.save() + def find_vhost_by_id(self, id_str): + """ + Searches through VirtualHosts and tries to match the id in a comment + + :param str id_str: Id string for matching + + :returns: The matched VirtualHost or None + :rtype: :class:`~certbot_apache.obj.VirtualHost` or None + + :raises .errors.PluginError: If no VirtualHost is found + """ + + for vh in self.vhosts: + if self._find_vhost_id(vh) == id_str: + return vh + msg = "No VirtualHost with ID {} was found.".format(id_str) + logger.warning(msg) + raise errors.PluginError(msg) + + def _find_vhost_id(self, vhost): + """Tries to find the unique ID from the VirtualHost comments. This is + used for keeping track of VirtualHost directive over time. + + :param vhost: Virtual host to add the id + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :returns: The unique ID or None + :rtype: str or None + """ + + # Strip the {} off from the format string + search_comment = constants.MANAGED_COMMENT_ID.format("") + + id_comment = self.parser.find_comments(search_comment, vhost.path) + if id_comment: + # Use the first value, multiple ones shouldn't exist + comment = self.parser.get_arg(id_comment[0]) + return comment.split(" ")[-1] + return None + + def add_vhost_id(self, vhost): + """Adds an unique ID to the VirtualHost as a comment for mapping back + to it on later invocations, as the config file order might have changed. + If ID already exists, returns that instead. + + :param vhost: Virtual host to add or find the id + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :returns: The unique ID for vhost + :rtype: str or None + """ + + vh_id = self._find_vhost_id(vhost) + if vh_id: + return vh_id + + id_string = apache_util.unique_id() + comment = constants.MANAGED_COMMENT_ID.format(id_string) + self.parser.add_comment(vhost.path, comment) + return id_string + def _escape(self, fp): fp = fp.replace(",", "\\,") fp = fp.replace("[", "\\[") @@ -1531,6 +1595,78 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.warning("Failed %s for %s", enhancement, domain) raise + def _autohsts_increase(self, vhost, id_str, nextstep): + """Increase the AutoHSTS max-age value + + :param vhost: Virtual host object to modify + :type vhost: :class:`~certbot_apache.obj.VirtualHost` + + :param str id_str: The unique ID string of VirtualHost + + :param int nextstep: Next AutoHSTS max-age value index + + """ + nextstep_value = constants.AUTOHSTS_STEPS[nextstep] + self._autohsts_write(vhost, nextstep_value) + self._autohsts[id_str] = {"laststep": nextstep, "timestamp": time.time()} + + def _autohsts_write(self, vhost, nextstep_value): + """ + Write the new HSTS max-age value to the VirtualHost file + """ + + hsts_dirpath = None + header_path = self.parser.find_dir("Header", None, vhost.path) + if header_path: + pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)' + for match in header_path: + if re.search(pat, self.aug.get(match).lower()): + hsts_dirpath = match + if not hsts_dirpath: + err_msg = ("Certbot was unable to find the existing HSTS header " + "from the VirtualHost at path {0}.").format(vhost.filep) + raise errors.PluginError(err_msg) + + # Prepare the HSTS header value + hsts_maxage = "\"max-age={0}\"".format(nextstep_value) + + # Update the header + # Our match statement was for string strict-transport-security, but + # we need to update the value instead. The next index is for the value + hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]") + self.aug.set(hsts_dirpath, hsts_maxage) + note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost " + "in {1}\n".format(nextstep_value, vhost.filep)) + logger.debug(note_msg) + self.save_notes += note_msg + self.save(note_msg) + + def _autohsts_fetch_state(self): + """ + Populates the AutoHSTS state from the pluginstorage + """ + try: + self._autohsts = self.storage.fetch("autohsts") + except KeyError: + self._autohsts = dict() + + def _autohsts_save_state(self): + """ + Saves the state of AutoHSTS object to pluginstorage + """ + self.storage.put("autohsts", self._autohsts) + self.storage.save() + + def _autohsts_vhost_in_lineage(self, vhost, lineage): + """ + Searches AutoHSTS managed VirtualHosts that belong to the lineage. + Matches the private key path. + """ + + return bool( + self.parser.find_dir("SSLCertificateKeyFile", + lineage.key_path, vhost.path)) + def _enable_ocsp_stapling(self, ssl_vhost, unused_options): """Enables OCSP Stapling @@ -2158,3 +2294,177 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + + def enable_autohsts(self, _unused_lineage, domains): + """ + Enable the AutoHSTS enhancement for defined domains + + :param _unused_lineage: Certificate lineage object, unused + :type _unused_lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + """ + + self._autohsts_fetch_state() + _enhanced_vhosts = [] + for d in domains: + matched_vhosts = self.choose_vhosts(d, create_if_no_ssl=False) + # We should be handling only SSL vhosts for AutoHSTS + vhosts = [vhost for vhost in matched_vhosts if vhost.ssl] + + if not vhosts: + msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a " + "domain {0} for enabling AutoHSTS enhancement.") + msg = msg_tmpl.format(d) + logger.warning(msg) + raise errors.PluginError(msg) + for vh in vhosts: + try: + self._enable_autohsts_domain(vh) + _enhanced_vhosts.append(vh) + except errors.PluginEnhancementAlreadyPresent: + if vh in _enhanced_vhosts: + continue + msg = ("VirtualHost for domain {0} in file {1} has a " + + "String-Transport-Security header present, exiting.") + raise errors.PluginEnhancementAlreadyPresent( + msg.format(d, vh.filep)) + if _enhanced_vhosts: + note_msg = "Enabling AutoHSTS" + self.save(note_msg) + logger.info(note_msg) + self.restart() + + # Save the current state to pluginstorage + self._autohsts_save_state() + + def _enable_autohsts_domain(self, ssl_vhost): + """Do the initial AutoHSTS deployment to a vhost + + :param ssl_vhost: The VirtualHost object to deploy the AutoHSTS + :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost` or None + + :raises errors.PluginEnhancementAlreadyPresent: When already enhanced + + """ + # This raises the exception + self._verify_no_matching_http_header(ssl_vhost, + "Strict-Transport-Security") + + if "headers_module" not in self.parser.modules: + self.enable_mod("headers") + # Prepare the HSTS header value + hsts_header = constants.HEADER_ARGS["Strict-Transport-Security"][:-1] + initial_maxage = constants.AUTOHSTS_STEPS[0] + hsts_header.append("\"max-age={0}\"".format(initial_maxage)) + + # Add ID to the VirtualHost for mapping back to it later + uniq_id = self.add_vhost_id(ssl_vhost) + self.save_notes += "Adding unique ID {0} to VirtualHost in {1}\n".format( + uniq_id, ssl_vhost.filep) + # Add the actual HSTS header + self.parser.add_dir(ssl_vhost.path, "Header", hsts_header) + note_msg = ("Adding gradually increasing HSTS header with initial value " + "of {0} to VirtualHost in {1}\n".format( + initial_maxage, ssl_vhost.filep)) + self.save_notes += note_msg + + # Save the current state to pluginstorage + self._autohsts[uniq_id] = {"laststep": 0, "timestamp": time.time()} + + def update_autohsts(self, _unused_domain): + """ + Increase the AutoHSTS values of VirtualHosts that the user has enabled + this enhancement for. + + :param _unused_domain: Not currently used + :type _unused_domain: Not Available + + """ + self._autohsts_fetch_state() + if not self._autohsts: + # No AutoHSTS enabled for any domain + return + curtime = time.time() + save_and_restart = False + for id_str, config in list(self._autohsts.items()): + if config["timestamp"] + constants.AUTOHSTS_FREQ > curtime: + # Skip if last increase was < AUTOHSTS_FREQ ago + continue + nextstep = config["laststep"] + 1 + if nextstep < len(constants.AUTOHSTS_STEPS): + # Have not reached the max value yet + try: + vhost = self.find_vhost_by_id(id_str) + except errors.PluginError: + msg = ("Could not find VirtualHost with ID {0}, disabling " + "AutoHSTS for this VirtualHost").format(id_str) + logger.warning(msg) + # Remove the orphaned AutoHSTS entry from pluginstorage + self._autohsts.pop(id_str) + continue + self._autohsts_increase(vhost, id_str, nextstep) + msg = ("Increasing HSTS max-age value for VirtualHost with id " + "{0}").format(id_str) + self.save_notes += msg + save_and_restart = True + + if save_and_restart: + self.save("Increased HSTS max-age values") + self.restart() + + self._autohsts_save_state() + + def deploy_autohsts(self, lineage): + """ + Checks if autohsts vhost has reached maximum auto-increased value + and changes the HSTS max-age to a high value. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + self._autohsts_fetch_state() + if not self._autohsts: + # No autohsts enabled for any vhost + return + + vhosts = [] + affected_ids = [] + # Copy, as we are removing from the dict inside the loop + for id_str, config in list(self._autohsts.items()): + if config["laststep"]+1 >= len(constants.AUTOHSTS_STEPS): + # max value reached, try to make permanent + try: + vhost = self.find_vhost_by_id(id_str) + except errors.PluginError: + msg = ("VirtualHost with id {} was not found, unable to " + "make HSTS max-age permanent.").format(id_str) + logger.warning(msg) + self._autohsts.pop(id_str) + continue + if self._autohsts_vhost_in_lineage(vhost, lineage): + vhosts.append(vhost) + affected_ids.append(id_str) + + save_and_restart = False + for vhost in vhosts: + self._autohsts_write(vhost, constants.AUTOHSTS_PERMANENT) + msg = ("Strict-Transport-Security max-age value for " + "VirtualHost in {0} was made permanent.").format(vhost.filep) + logger.debug(msg) + self.save_notes += msg+"\n" + save_and_restart = True + + if save_and_restart: + self.save("Made HSTS max-age permanent") + self.restart() + + for id_str in affected_ids: + self._autohsts.pop(id_str) + + # Update AutoHSTS storage (We potentially removed vhosts from managed) + self._autohsts_save_state() + + +AutoHSTSEnhancement.register(ApacheConfigurator) # pylint: disable=no-member diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index fd6a9eb11..23a7b7afd 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -48,3 +48,16 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy", HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, "Upgrade-Insecure-Requests": UIR_ARGS} + +AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400] +"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h""" + +AUTOHSTS_PERMANENT = 31536000 +"""Value for the last max-age of HSTS""" + +AUTOHSTS_FREQ = 172800 +"""Minimum time since last increase to perform a new one: 48h""" + +MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot" +MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}" +"""Managed by Certbot comments and the VirtualHost identification template""" diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 43878eda2..02337f8d4 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) class ApacheParser(object): + # pylint: disable=too-many-public-methods """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... @@ -350,6 +351,37 @@ class ApacheParser(object): else: self.aug.set(first_dir + "/arg", args) + def add_comment(self, aug_conf_path, comment): + """Adds the comment to the augeas path + + :param str aug_conf_path: Augeas configuration path to add directive + :param str comment: Comment content + + """ + self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment) + + def find_comments(self, arg, start=None): + """Finds a comment with specified content from the provided DOM path + + :param str arg: Comment content to search + :param str start: Beginning Augeas path to begin looking + + :returns: List of augeas paths containing the comment content + :rtype: list + + """ + if not start: + start = get_aug_path(self.root) + + comments = self.aug.match("%s//*[label() = '#comment']" % start) + + results = [] + for comment in comments: + c_content = self.aug.get(comment) + if c_content and arg in c_content: + results.append(comment) + return results + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py new file mode 100644 index 000000000..86d985079 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -0,0 +1,181 @@ +# pylint: disable=too-many-public-methods,too-many-lines +"""Test for certbot_apache.configurator AutoHSTS functionality""" +import re +import unittest +import mock +# six is used in mock.patch() +import six # pylint: disable=unused-import + +from certbot import errors +from certbot_apache import constants +from certbot_apache.tests import util + + +class AutoHSTSTest(util.ApacheTest): + """Tests for AutoHSTS feature""" + # pylint: disable=protected-access + + def setUp(self): # pylint: disable=arguments-differ + super(AutoHSTSTest, self).setUp() + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config.parser.modules.add("headers_module") + self.config.parser.modules.add("mod_headers.c") + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") + + self.vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multiple_vhosts") + + def get_autohsts_value(self, vh_path): + """ Get value from Strict-Transport-Security header """ + header_path = self.config.parser.find_dir("Header", None, vh_path) + if header_path: + pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)' + for head in header_path: + if re.search(pat, self.config.parser.aug.get(head).lower()): + return self.config.parser.aug.get(head.replace("arg[3]", + "arg[4]")) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_autohsts_enable_headers_mod(self, mock_enable, _restart): + self.config.parser.modules.discard("headers_module") + self.config.parser.modules.discard("mod_header.c") + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + self.assertTrue(mock_enable.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_deploy_already_exists(self, _restart): + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + self.assertRaises(errors.PluginEnhancementAlreadyPresent, + self.config.enable_autohsts, + mock.MagicMock(), ["ocspvhost.com"]) + + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_increase(self, _mock_restart): + maxage = "\"max-age={0}\"" + initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) + inc_val = maxage.format(constants.AUTOHSTS_STEPS[1]) + + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Verify initial value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + initial_val) + # Increase + self.config.update_autohsts(mock.MagicMock()) + # Verify increased value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + inc_val) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase") + def test_autohsts_increase_noop(self, mock_increase, _restart): + maxage = "\"max-age={0}\"" + initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Verify initial value + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + initial_val) + + self.config.update_autohsts(mock.MagicMock()) + # Freq not patched, so value shouldn't increase + self.assertFalse(mock_increase.called) + + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + def test_autohsts_increase_no_header(self, _restart): + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + # Remove the header + dir_locs = self.config.parser.find_dir("Header", None, + self.vh_truth[7].path) + dir_loc = "/".join(dir_locs[0].split("/")[:-1]) + self.config.parser.aug.remove(dir_loc) + self.assertRaises(errors.PluginError, + self.config.update_autohsts, + mock.MagicMock()) + + @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + def test_autohsts_increase_and_make_permanent(self, _mock_restart): + maxage = "\"max-age={0}\"" + max_val = maxage.format(constants.AUTOHSTS_PERMANENT) + mock_lineage = mock.MagicMock() + mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem" + self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) + for i in range(len(constants.AUTOHSTS_STEPS)-1): + # Ensure that value is not made permanent prematurely + self.config.deploy_autohsts(mock_lineage) + self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path), + max_val) + self.config.update_autohsts(mock.MagicMock()) + # Value should match pre-permanent increment step + cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + cur_val) + # Make permanent + self.config.deploy_autohsts(mock_lineage) + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + max_val) + + def test_autohsts_update_noop(self): + with mock.patch("time.time") as mock_time: + # Time mock is used to make sure that the execution does not + # continue when no autohsts entries exist in pluginstorage + self.config.update_autohsts(mock.MagicMock()) + self.assertFalse(mock_time.called) + + def test_autohsts_make_permanent_noop(self): + self.config.storage.put = mock.MagicMock() + self.config.deploy_autohsts(mock.MagicMock()) + # Make sure that the execution does not continue when no entries in store + self.assertFalse(self.config.storage.put.called) + + @mock.patch("certbot_apache.display_ops.select_vhost") + def test_autohsts_no_ssl_vhost(self, mock_select): + mock_select.return_value = self.vh_truth[0] + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.assertRaises(errors.PluginError, + self.config.enable_autohsts, + mock.MagicMock(), "invalid.example.com") + self.assertTrue( + "Certbot was not able to find SSL" in mock_log.call_args[0][0]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id") + def test_autohsts_dont_enhance_twice(self, mock_id, _restart): + mock_id.return_value = "1234567" + self.config.enable_autohsts(mock.MagicMock(), + ["ocspvhost.com", "ocspvhost.com"]) + self.assertEquals(mock_id.call_count, 1) + + def test_autohsts_remove_orphaned(self): + # pylint: disable=protected-access + self.config._autohsts_fetch_state() + self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0} + + self.config._autohsts_save_state() + self.config.update_autohsts(mock.MagicMock()) + self.assertFalse("orphan_id" in self.config._autohsts) + # Make sure it's removed from the pluginstorage file as well + self.config._autohsts = None + self.config._autohsts_fetch_state() + self.assertFalse(self.config._autohsts) + + def test_autohsts_make_permanent_vhost_not_found(self): + # pylint: disable=protected-access + self.config._autohsts_fetch_state() + self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0} + self.config._autohsts_save_state() + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.config.deploy_autohsts(mock.MagicMock()) + self.assertTrue(mock_log.called) + self.assertTrue( + "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0]) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 23c1ee82b..350262634 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1487,6 +1487,21 @@ class MultipleVhostsTest(util.ApacheTest): "Upgrade-Insecure-Requests") self.assertTrue(mock_choose.called) + def test_add_vhost_id(self): + for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]: + vh_id = self.config.add_vhost_id(vh) + self.assertEqual(vh, self.config.find_vhost_by_id(vh_id)) + + def test_find_vhost_by_id_404(self): + self.assertRaises(errors.PluginError, + self.config.find_vhost_by_id, + "nonexistent") + + def test_add_vhost_id_already_exists(self): + first_id = self.config.add_vhost_id(self.vh_truth[0]) + second_id = self.config.add_vhost_id(self.vh_truth[0]) + self.assertEqual(first_id, second_id) + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index 4496781c9..f95f1b346 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -299,6 +299,13 @@ class BasicParserTest(util.ParserTest): errors.MisconfigurationError, self.parser.update_runtime_variables) + def test_add_comment(self): + from certbot_apache.parser import get_aug_path + self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456") + comm = self.parser.find_comments("123456") + self.assertEquals(len(comm), 1) + self.assertTrue(self.parser.loc["name"] in comm[0]) + class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 4e4aadbd8..e70ac0c7f 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.25.0 -certbot[dev]==0.21.1 +-e .[dev] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7a0dc43bf..8d15e7971 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -8,7 +8,7 @@ version = '0.26.0.dev0' # acme/certbot version. install_requires = [ 'acme>0.24.0', - 'certbot>=0.21.1', + 'certbot>0.25.1', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot/cli.py b/certbot/cli.py index 24e0bddac..81e926e2f 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -32,6 +32,7 @@ from certbot import util from certbot.display import util as display_util from certbot.plugins import disco as plugins_disco +import certbot.plugins.enhancements as enhancements import certbot.plugins.selection as plugin_selection logger = logging.getLogger(__name__) @@ -627,6 +628,10 @@ class HelpfulArgumentParser(object): raise errors.Error("Using --allow-subset-of-names with a" " wildcard domain is not supported.") + if parsed_args.hsts and parsed_args.auto_hsts: + raise errors.Error( + "Parameters --hsts and --auto-hsts cannot be used simultaneously.") + possible_deprecation_warning(parsed_args) return parsed_args @@ -1228,6 +1233,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) + # Populate the command line parameters for new style enhancements + enhancements.populate_cli(helpful.add) + _create_subparsers(helpful) _paths_parser(helpful) # _plugins_parsing should be the last thing to act upon the main diff --git a/certbot/constants.py b/certbot/constants.py index 93bc269af..7047424e5 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -58,6 +58,7 @@ CLI_DEFAULTS = dict( rsa_key_size=2048, must_staple=False, redirect=None, + auto_hsts=False, hsts=None, uir=None, staple=None, diff --git a/certbot/main.py b/certbot/main.py index 089eab0c3..a457919d4 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -36,7 +36,7 @@ from certbot import util from certbot.display import util as display_util, ops as display_ops from certbot.plugins import disco as plugins_disco from certbot.plugins import selection as plug_sel - +from certbot.plugins import enhancements USER_CANCELLED = ("User chose to cancel the operation and may " "reinvoke the client.") @@ -750,8 +750,8 @@ def _install_cert(config, le_client, domains, lineage=None): :param le_client: Client object :type le_client: client.Client - :param plugins: List of domains - :type plugins: `list` of `str` + :param domains: List of domains + :type domains: `list` of `str` :param lineage: Certificate lineage object. Defaults to `None` :type lineage: storage.RenewableCert @@ -789,11 +789,19 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") # If cert-path is defined, populate missing (ie. not overridden) values. # Unfortunately this can't be done in argument parser, as certificate # manager needs the access to renewal directory paths if config.certname: config = _populate_from_certname(config) + elif enhancements.are_requested(config): + # Preflight config check + raise errors.ConfigurationError("One or more of the requested enhancements " + "require --cert-name to be provided") + if config.key_path and config.cert_path: _check_certificate_and_key(config) domains, _ = _find_domains_or_certname(config, installer) @@ -804,6 +812,11 @@ def install(config, plugins): "If your certificate is managed by Certbot, please use --cert-name " "to define which certificate you would like to install.") + if enhancements.are_requested(config): + # In the case where we don't have certname, we have errored out already + lineage = cert_manager.lineage_for_certname(config, config.certname) + enhancements.enable(lineage, domains, installer, config) + def _populate_from_certname(config): """Helper function for install to populate missing config values from lineage defined by --cert-name.""" @@ -881,7 +894,8 @@ def enhance(config, plugins): """ supported_enhancements = ["hsts", "redirect", "uir", "staple"] # Check that at least one enhancement was requested on command line - if not any([getattr(config, enh) for enh in supported_enhancements]): + oldstyle_enh = any([getattr(config, enh) for enh in supported_enhancements]) + if not enhancements.are_requested(config) and not oldstyle_enh: msg = ("Please specify one or more enhancement types to configure. To list " "the available enhancement types, run:\n\n%s --help enhance\n") logger.warning(msg, sys.argv[0]) @@ -892,6 +906,10 @@ def enhance(config, plugins): except errors.PluginSelectionError as e: return str(e) + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") + certname_question = ("Which certificate would you like to use to enhance " "your configuration?") config.certname = cert_manager.get_certnames( @@ -907,11 +925,15 @@ def enhance(config, plugins): if not domains: raise errors.Error("User cancelled the domain selection. No domains " "defined, exiting.") + + lineage = cert_manager.lineage_for_certname(config, config.certname) if not config.chain_path: - lineage = cert_manager.lineage_for_certname(config, config.certname) config.chain_path = lineage.chain_path - le_client = _init_le_client(config, authenticator=None, installer=installer) - le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + if oldstyle_enh: + le_client = _init_le_client(config, authenticator=None, installer=installer) + le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + if enhancements.are_requested(config): + enhancements.enable(lineage, domains, installer, config) def rollback(config, plugins): @@ -1073,6 +1095,11 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals except errors.PluginSelectionError as e: return str(e) + # Preflight check for enhancement support by the selected installer + if not enhancements.are_supported(config, installer): + raise errors.NotSupportedError("One ore more of the requested enhancements " + "are not supported by the selected installer") + # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) @@ -1091,6 +1118,9 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals _install_cert(config, le_client, domains, new_lineage) + if enhancements.are_requested(config) and new_lineage: + enhancements.enable(new_lineage, domains, installer, config) + if lineage is None or not should_get_cert: display_ops.success_installation(domains) else: diff --git a/certbot/plugins/enhancements.py b/certbot/plugins/enhancements.py new file mode 100644 index 000000000..506abe433 --- /dev/null +++ b/certbot/plugins/enhancements.py @@ -0,0 +1,159 @@ +"""New interface style Certbot enhancements""" +import abc +import six + +from certbot import constants + +from acme.magic_typing import Dict, List, Any # pylint: disable=unused-import, no-name-in-module + +def enabled_enhancements(config): + """ + Generator to yield the enabled new style enhancements. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + for enh in _INDEX: + if getattr(config, enh["cli_dest"]): + yield enh + +def are_requested(config): + """ + Checks if one or more of the requested enhancements are those of the new + enhancement interfaces. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + return any(enabled_enhancements(config)) + +def are_supported(config, installer): + """ + Checks that all of the requested enhancements are supported by the + installer. + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: If all the requested enhancements are supported by the installer + :rtype: bool + """ + for enh in enabled_enhancements(config): + if not isinstance(installer, enh["class"]): + return False + return True + +def enable(lineage, domains, installer, config): + """ + Run enable method for each requested enhancement that is supported. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration. + :type config: :class:`certbot.interfaces.IConfig` + """ + for enh in enabled_enhancements(config): + getattr(installer, enh["enable_function"])(lineage, domains) + +def populate_cli(add): + """ + Populates the command line flags for certbot.cli.HelpfulParser + + :param add: Add function of certbot.cli.HelpfulParser + :type add: func + """ + for enh in _INDEX: + add(enh["cli_groups"], enh["cli_flag"], action=enh["cli_action"], + dest=enh["cli_dest"], default=enh["cli_flag_default"], + help=enh["cli_help"]) + + +@six.add_metaclass(abc.ABCMeta) +class AutoHSTSEnhancement(object): + """ + Enhancement interface that installer plugins can implement in order to + provide functionality that configures the software to have a + 'Strict-Transport-Security' with initially low max-age value that will + increase over time. + + The plugins implementing new style enhancements are responsible of handling + the saving of configuration checkpoints as well as calling possible restarts + of managed software themselves. + + Methods: + enable_autohsts is called when the header is initially installed using a + low max-age value. + + update_autohsts is called every time when Certbot is run using 'renew' + verb. The max-age value should be increased over time using this method. + + deploy_autohsts is called for every lineage that has had its certificate + renewed. A long HSTS max-age value should be set here, as we should be + confident that the user is able to automatically renew their certificates. + + + """ + + @abc.abstractmethod + def update_autohsts(self, lineage, *args, **kwargs): + """ + Gets called for each lineage every time Certbot is run with 'renew' verb. + Implementation of this method should increase the max-age value. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + + @abc.abstractmethod + def deploy_autohsts(self, lineage, *args, **kwargs): + """ + Gets called for a lineage when its certificate is successfully renewed. + Long max-age value should be set in implementation of this method. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + """ + + @abc.abstractmethod + def enable_autohsts(self, lineage, domains, *args, **kwargs): + """ + Enables the AutoHSTS enhancement, installing + Strict-Transport-Security header with a low initial value to be increased + over the subsequent runs of Certbot renew. + + :param lineage: Certificate lineage object + :type lineage: certbot.storage.RenewableCert + + :param domains: List of domains in certificate to enhance + :type domains: str + """ + +# This is used to configure internal new style enhancements in Certbot. These +# enhancement interfaces need to be defined in this file. Please do not modify +# this list from plugin code. +_INDEX = [ + { + "name": "AutoHSTS", + "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ + "Security security header", + "cli_flag": "--auto-hsts", + "cli_flag_default": constants.CLI_DEFAULTS["auto_hsts"], + "cli_groups": ["security", "enhance"], + "cli_dest": "auto_hsts", + "cli_action": "store_true", + "class": AutoHSTSEnhancement, + "updater_function": "update_autohsts", + "deployer_function": "deploy_autohsts", + "enable_function": "enable_autohsts" + } +] # type: List[Dict[str, Any]] diff --git a/certbot/plugins/enhancements_test.py b/certbot/plugins/enhancements_test.py new file mode 100644 index 000000000..b69dc9836 --- /dev/null +++ b/certbot/plugins/enhancements_test.py @@ -0,0 +1,65 @@ +"""Tests for new style enhancements""" +import unittest +import mock + +from certbot.plugins import enhancements +from certbot.plugins import null + +import certbot.tests.util as test_util + + +class EnhancementTest(test_util.ConfigTestCase): + """Tests for new style enhancements in certbot.plugins.enhancements""" + + def setUp(self): + super(EnhancementTest, self).setUp() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) + + + @test_util.patch_get_utility() + def test_enhancement_enabled_enhancements(self, _): + FAKEINDEX = [ + { + "name": "autohsts", + "cli_dest": "auto_hsts", + }, + { + "name": "somethingelse", + "cli_dest": "something", + } + ] + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + self.config.auto_hsts = True + self.config.something = True + enabled = list(enhancements.enabled_enhancements(self.config)) + self.assertEqual(len(enabled), 2) + self.assertTrue([i for i in enabled if i["name"] == "autohsts"]) + self.assertTrue([i for i in enabled if i["name"] == "somethingelse"]) + + def test_are_requested(self): + self.assertEquals( + len([i for i in enhancements.enabled_enhancements(self.config)]), 0) + self.assertFalse(enhancements.are_requested(self.config)) + self.config.auto_hsts = True + self.assertEquals( + len([i for i in enhancements.enabled_enhancements(self.config)]), 1) + self.assertTrue(enhancements.are_requested(self.config)) + + def test_are_supported(self): + self.config.auto_hsts = True + unsupported = null.Installer(self.config, "null") + self.assertTrue(enhancements.are_supported(self.config, self.mockinstaller)) + self.assertFalse(enhancements.are_supported(self.config, unsupported)) + + def test_enable(self): + self.config.auto_hsts = True + domains = ["example.com", "www.example.com"] + lineage = "lineage" + enhancements.enable(lineage, domains, self.mockinstaller, self.config) + self.assertTrue(self.mockinstaller.enable_autohsts.called) + self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0], + (lineage, domains)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 4b251c421..24d059e53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -29,7 +29,9 @@ from certbot import updater from certbot import util from certbot.plugins import disco +from certbot.plugins import enhancements from certbot.plugins import manual +from certbot.plugins import null import certbot.tests.util as test_util @@ -52,10 +54,11 @@ class TestHandleIdenticalCerts(unittest.TestCase): self.assertEqual(ret, ("reinstall", mock_lineage)) -class RunTest(unittest.TestCase): +class RunTest(test_util.ConfigTestCase): """Tests for certbot.main.run.""" def setUp(self): + super(RunTest, self).setUp() self.domain = 'example.org' self.patches = [ mock.patch('certbot.main._get_and_save_cert'), @@ -105,6 +108,15 @@ class RunTest(unittest.TestCase): self._call() self.mock_success_renewal.assert_called_once_with([self.domain]) + @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') + def test_run_enhancement_not_supported(self, mock_choose): + mock_choose.return_value = (null.Installer(self.config, "null"), None) + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.assertRaises(errors.NotSupportedError, + main.run, + self.config, plugins) + class CertonlyTest(unittest.TestCase): """Tests for certbot.main.certonly.""" @@ -1573,12 +1585,14 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): strict=self.config.strict_permissions) -class EnhanceTest(unittest.TestCase): +class EnhanceTest(test_util.ConfigTestCase): """Tests for certbot.main.enhance.""" def setUp(self): + super(EnhanceTest, self).setUp() self.get_utility_patch = test_util.patch_get_utility() self.mock_get_utility = self.get_utility_patch.start() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) def tearDown(self): self.get_utility_patch.stop() @@ -1670,7 +1684,7 @@ class EnhanceTest(unittest.TestCase): def test_no_enhancements_defined(self): self.assertRaises(errors.MisconfigurationError, - self._call, ['enhance']) + self._call, ['enhance', '-a', 'null']) @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') @mock.patch('certbot.main.display_ops.choose_values') @@ -1682,5 +1696,67 @@ class EnhanceTest(unittest.TestCase): mock_client = self._call(['enhance', '--hsts']) self.assertFalse(mock_client.enhance_config.called) + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @test_util.patch_get_utility() + def test_enhancement_enable(self, _, _rec, mock_inst, mock_choose, mock_lineage): + mock_inst.return_value = self.mockinstaller + mock_choose.return_value = ["example.com", "another.tld"] + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + self._call(['enhance', '--auto-hsts']) + self.assertTrue(self.mockinstaller.enable_autohsts.called) + self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0][1], + ["example.com", "another.tld"]) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @test_util.patch_get_utility() + def test_enhancement_enable_not_supported(self, _, _rec, mock_inst, mock_choose, mock_lineage): + mock_inst.return_value = null.Installer(self.config, "null") + mock_choose.return_value = ["example.com", "another.tld"] + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + self.assertRaises( + errors.NotSupportedError, + self._call, ['enhance', '--auto-hsts']) + + def test_enhancement_enable_conflict(self): + self.assertRaises( + errors.Error, + self._call, ['enhance', '--auto-hsts', '--hsts']) + + +class InstallTest(test_util.ConfigTestCase): + """Tests for certbot.main.install.""" + + def setUp(self): + super(InstallTest, self).setUp() + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_install_enhancement_not_supported(self, mock_inst, _rec): + mock_inst.return_value = null.Installer(self.config, "null") + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.assertRaises(errors.NotSupportedError, + main.install, + self.config, plugins) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_install_enhancement_no_certname(self, mock_inst, _rec): + mock_inst.return_value = self.mockinstaller + plugins = disco.PluginsRegistry.find_all() + self.config.auto_hsts = True + self.config.certname = None + self.assertRaises(errors.ConfigurationError, + main.install, + self.config, plugins) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index bd1cd891e..c1b97843e 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -6,6 +6,8 @@ from certbot import interfaces from certbot import main from certbot import updater +from certbot.plugins import enhancements + import certbot.tests.util as test_util @@ -14,25 +16,10 @@ class RenewUpdaterTest(test_util.ConfigTestCase): def setUp(self): super(RenewUpdaterTest, self).setUp() - class MockInstallerGenericUpdater(interfaces.GenericUpdater): - """Mock class that implements GenericUpdater""" - def __init__(self, *args, **kwargs): - # pylint: disable=unused-argument - self.restart = mock.MagicMock() - self.callcounter = mock.MagicMock() - def generic_updates(self, lineage, *args, **kwargs): - self.callcounter(*args, **kwargs) - - class MockInstallerRenewDeployer(interfaces.RenewDeployer): - """Mock class that implements RenewDeployer""" - def __init__(self, *args, **kwargs): - # pylint: disable=unused-argument - self.callcounter = mock.MagicMock() - def renew_deploy(self, lineage, *args, **kwargs): - self.callcounter(*args, **kwargs) - - self.generic_updater = MockInstallerGenericUpdater() - self.renew_deployer = MockInstallerRenewDeployer() + self.generic_updater = mock.MagicMock(spec=interfaces.GenericUpdater) + self.generic_updater.restart = mock.MagicMock() + self.renew_deployer = mock.MagicMock(spec=interfaces.RenewDeployer) + self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') @@ -48,16 +35,16 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertTrue(mock_generic_updater.restart.called) mock_generic_updater.restart.reset_mock() - mock_generic_updater.callcounter.reset_mock() + mock_generic_updater.generic_updates.reset_mock() updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertEqual(mock_generic_updater.callcounter.call_count, 1) + self.assertEqual(mock_generic_updater.generic_updates.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): lineage = mock.MagicMock() mock_deployer = self.renew_deployer updater.run_renewal_deployer(self.config, lineage, mock_deployer) - self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + self.assertTrue(mock_deployer.renew_deploy.called_with(lineage)) @mock.patch("certbot.updater.logger.debug") def test_updater_skip_dry_run(self, mock_log): @@ -75,6 +62,62 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertEquals(mock_log.call_args[0][0], "Skipping renewal deployer in dry-run mode.") + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_updates(self, mock_select): + mock_select.return_value = (self.mockinstaller, None) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertTrue(self.mockinstaller.update_autohsts.called) + self.assertEqual(self.mockinstaller.update_autohsts.call_count, 1) + + def test_enhancement_deployer(self): + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertTrue(self.mockinstaller.deploy_autohsts.called) + + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_updates_not_called(self, mock_select): + self.config.disable_renew_updates = True + mock_select.return_value = (self.mockinstaller, None) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertFalse(self.mockinstaller.update_autohsts.called) + + def test_enhancement_deployer_not_called(self): + self.config.disable_renew_updates = True + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertFalse(self.mockinstaller.deploy_autohsts.called) + + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_enhancement_no_updater(self, mock_select): + FAKEINDEX = [ + { + "name": "Test", + "class": enhancements.AutoHSTSEnhancement, + "updater_function": None, + "deployer_function": "deploy_autohsts", + "enable_function": "enable_autohsts" + } + ] + mock_select.return_value = (self.mockinstaller, None) + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + updater.run_generic_updaters(self.config, mock.MagicMock(), None) + self.assertFalse(self.mockinstaller.update_autohsts.called) + + def test_enhancement_no_deployer(self): + FAKEINDEX = [ + { + "name": "Test", + "class": enhancements.AutoHSTSEnhancement, + "updater_function": "deploy_autohsts", + "deployer_function": None, + "enable_function": "enable_autohsts" + } + ] + with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): + updater.run_renewal_deployer(self.config, mock.MagicMock(), + self.mockinstaller) + self.assertFalse(self.mockinstaller.deploy_autohsts.called) + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py index 112cf06ef..fb7c52f77 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -5,6 +5,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import selection as plug_sel +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) @@ -33,6 +34,7 @@ def run_generic_updaters(config, lineage, plugins): logger.warning("Could not choose appropriate plugin for updaters: %s", e) return _run_updaters(lineage, installer, config) + _run_enhancement_updaters(lineage, installer, config) def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used @@ -57,6 +59,7 @@ def run_renewal_deployer(config, lineage, installer): if not config.disable_renew_updates and isinstance(installer, interfaces.RenewDeployer): installer.renew_deploy(lineage) + _run_enhancement_deployers(lineage, installer, config) def _run_updaters(lineage, installer, config): """Helper function to run the updater interface methods if supported by the @@ -74,3 +77,46 @@ def _run_updaters(lineage, installer, config): if not config.disable_renew_updates: if isinstance(installer, interfaces.GenericUpdater): installer.generic_updates(lineage) + +def _run_enhancement_updaters(lineage, installer, config): + """Iterates through known enhancement interfaces. If the installer implements + an enhancement interface and the enhance interface has an updater method, the + updater method gets run. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration object + :type config: interfaces.IConfig + """ + + if config.disable_renew_updates: + return + for enh in enhancements._INDEX: # pylint: disable=protected-access + if isinstance(installer, enh["class"]) and enh["updater_function"]: + getattr(installer, enh["updater_function"])(lineage) + + +def _run_enhancement_deployers(lineage, installer, config): + """Iterates through known enhancement interfaces. If the installer implements + an enhancement interface and the enhance interface has an deployer method, the + deployer method gets run. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :param config: Configuration object + :type config: interfaces.IConfig + """ + + if config.disable_renew_updates: + return + for enh in enhancements._INDEX: # pylint: disable=protected-access + if isinstance(installer, enh["class"]) and enh["deployer_function"]: + getattr(installer, enh["deployer_function"])(lineage) From 6771b8e05bfe438165df67d8cc671857510f3957 Mon Sep 17 00:00:00 2001 From: Harlan Lieberman-Berg Date: Thu, 21 Jun 2018 15:52:08 -0400 Subject: [PATCH 259/639] docs: move warning about distro provided renewal (#6133) Currently, you must read ten paragraphs about writing renewal hooks before you find that most distributions will automatically renew certs for you. This is burying the lede in a major way; moving it up to the header seems a better choice. --- docs/using.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 40d8f8452..46599a06e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -454,6 +454,12 @@ Renewing certificates days). Make sure you renew the certificates at least once in 3 months. +.. seealso:: Many of the certbot clients obtained through a + distribution come with automatic renewal out of the box, + such as Debian and Ubuntu versions installed through `apt`, + CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ + for more details. + As of version 0.10.0, Certbot supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew them. The simplest form is simply @@ -560,12 +566,6 @@ can run on a regular basis, like every week or every day). In that case, you are likely to want to use the ``-q`` or ``--quiet`` quiet flag to silence all output except errors. -.. seealso:: Many of the certbot clients obtained through a - distribution come with automatic renewal out of the box, - such as Debian and Ubuntu versions installed through `apt`, - CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ - for more details. - If you are manually renewing all of your certificates, the ``--force-renewal`` flag may be helpful; it causes the expiration time of the certificate(s) to be ignored when considering renewal, and attempts to From 2ac0b5520833b83b093b10f02e5a02686842e17a Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 21 Jun 2018 13:23:09 -0700 Subject: [PATCH 260/639] Reuse ACMEv1 accounts for ACMEv2 in production (#6134) * Reuse accounts made with ACMEv1 when using an ACMEv2 Let's Encrypt server. This commit turns the feature on for the production server; the bulk of the work was done in 8e4303a. * add upgrade test for production server --- certbot/constants.py | 1 + certbot/tests/account_test.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/certbot/constants.py b/certbot/constants.py index 7047424e5..d31faa71c 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -161,6 +161,7 @@ ACCOUNTS_DIR = "accounts" """Directory where all accounts are saved.""" LE_REUSE_SERVERS = { + 'acme-v02.api.letsencrypt.org/directory': 'acme-v01.api.letsencrypt.org/directory', 'acme-staging-v02.api.letsencrypt.org/directory': 'acme-staging.api.letsencrypt.org/directory' } diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index a8059fbcf..a4fe5edb7 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -218,12 +218,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) - def test_upgrade_version(self): + def test_upgrade_version_staging(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([self.acc], self.storage.find_all()) + def test_upgrade_version_production(self): + self._set_server('https://acme-v01.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-v02.api.letsencrypt.org/directory') + self.assertEqual([self.acc], self.storage.find_all()) + @mock.patch('os.rmdir') def test_corrupted_account(self, mock_rmdir): # pylint: disable=protected-access From 1e1e7d8e973be0b7376d96fb07c59816f694a83a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 21 Jun 2018 15:40:43 -0700 Subject: [PATCH 261/639] Improve UA default in docs (#6120) * Use less informative UA values in docs. * set CERTBOT_DOCS during release --- certbot/client.py | 12 ++++++++++-- certbot/tests/client_test.py | 33 +++++++++++++++++++++++++++++++++ tools/release.sh | 3 ++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d97de0571..4d4915a27 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -65,9 +65,17 @@ def determine_user_agent(config): if config.user_agent is None: ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") - ua = ua.format(certbot.__version__, cli.cli_command, util.get_os_info_ua(), + if os.environ.get("CERTBOT_DOCS") == "1": + cli_command = "certbot(-auto)" + os_info = "OS_NAME OS_VERSION" + python_version = "major.minor.patchlevel" + else: + cli_command = cli.cli_command + os_info = util.get_os_info_ua() + python_version = platform.python_version() + ua = ua.format(certbot.__version__, cli_command, os_info, config.authenticator, config.installer, config.verb, - ua_flags(config), platform.python_version(), + ua_flags(config), python_version, "; " + config.user_agent_comment if config.user_agent_comment else "") else: ua = config.user_agent diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a4425bca9..70ab4c798 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -1,5 +1,6 @@ """Tests for certbot.client.""" import os +import platform import shutil import tempfile import unittest @@ -16,6 +17,38 @@ KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") +class DetermineUserAgentTest(test_util.ConfigTestCase): + """Tests for certbot.client.determine_user_agent.""" + + def _call(self): + from certbot.client import determine_user_agent + return determine_user_agent(self.config) + + @mock.patch.dict(os.environ, {"CERTBOT_DOCS": "1"}) + def test_docs_value(self): + self._test(expect_doc_values=True) + + @mock.patch.dict(os.environ, {}) + def test_real_values(self): + self._test(expect_doc_values=False) + + def _test(self, expect_doc_values): + ua = self._call() + + if expect_doc_values: + doc_value_check = self.assertIn + real_value_check = self.assertNotIn + else: + doc_value_check = self.assertNotIn + real_value_check = self.assertIn + + doc_value_check("certbot(-auto)", ua) + doc_value_check("OS_NAME OS_VERSION", ua) + doc_value_check("major.minor.patchlevel", ua) + real_value_check(util.get_os_info_ua(), ua) + real_value_check(platform.python_version(), ua) + + class RegisterTest(test_util.ConfigTestCase): """Tests for certbot.client.register.""" diff --git a/tools/release.sh b/tools/release.sh index a8de208b5..069d311d1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -153,7 +153,8 @@ kill $! cd ~- # get a snapshot of the CLI help for the docs -certbot --help all > docs/cli-help.txt +# We set CERTBOT_DOCS to use dummy values in example user-agent string. +CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt jws --help > acme/docs/jws-help.txt cd .. From 7890de62ecfd0a3de60b81fabba4659cf80605d1 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 21 Jun 2018 16:11:02 -0700 Subject: [PATCH 262/639] doc(postfix): install instructions (#6136) fixes #6131 * doc(postfix): install instructions * address brad's comments --- certbot-postfix/README.rst | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index 78fd9a421..e6367e365 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -2,9 +2,22 @@ Postfix plugin for Certbot ========================== -To install your certs with this plugin, run: +Note: this MTA installer is in **developer beta**-- we appreciate any testing, feedback, or +feature requests for this plugin. -``certbot install --installer postfix --cert-path --key-path -d `` +To install this plugin, in the root of this repo, run:: -And there you go! If you'd like to obtain these certificates via certbot, there's more documentation on how to do this `here `_. + ./tools/venv.sh + source venv/bin/activate +You can use this installer with any `authenticator plugin +`_. +For instance, with the `standalone authenticator +`_, which requires no extra server +software, you might run:: + + sudo ./venv/bin/certbot run --standalone -i postfix -d + +To just install existing certs with this plugin, run:: + + sudo ./venv/bin/certbot install -i postfix --cert-path --key-path -d From 80cd134847edd1b860315796d3accf1a46171b16 Mon Sep 17 00:00:00 2001 From: r5d Date: Mon, 25 Jun 2018 21:02:07 -0400 Subject: [PATCH 263/639] certbot.cli: Remove debug-challenges option for `renew` subcommand. (#6141) Addresses issue #5005. --- certbot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 81e926e2f..5c4313ea4 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1082,7 +1082,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="Show tracebacks in case of errors, and allow certbot-auto " "execution on experimental platforms") helpful.add( - [None, "certonly", "renew", "run"], "--debug-challenges", action="store_true", + [None, "certonly", "run"], "--debug-challenges", action="store_true", default=flag_default("debug_challenges"), help="After setting up challenges, wait for user input before " "submitting to CA") From 87e1912bf91cf91e8ff1d3e4d5928353ae1a294c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 25 Jun 2018 18:09:30 -0700 Subject: [PATCH 264/639] Show both possible Nginx default server root values in docs (#6137) See https://github.com/certbot/website/pull/348#issuecomment-399257703. ``` $ certbot --help all | grep -C 3 nginx-server-root nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and ``` ``` $ CERTBOT_DOCS=1 certbot --help all | grep -C 3 nginx-server-root nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx or /usr/local/etc/nginx) --nginx-ctl NGINX_CTL ``` * Show both possible Nginx default server root values in docs * add test * check that exactly one server root is in the default * use default magic --- certbot-nginx/certbot_nginx/configurator.py | 11 +++++++- certbot-nginx/certbot_nginx/constants.py | 7 +++-- .../certbot_nginx/tests/configurator_test.py | 26 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 41b5124b8..b80d95613 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -69,8 +69,9 @@ class NginxConfigurator(common.Installer): @classmethod def add_parser_arguments(cls, add): + default_server_root = _determine_default_server_root() add("server-root", default=constants.CLI_DEFAULTS["server_root"], - help="Nginx server root directory.") + help="Nginx server root directory. (default: %s)" % default_server_root) add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the " "'nginx' binary, used for 'configtest' and retrieving nginx " "version number.") @@ -1129,3 +1130,11 @@ def install_ssl_options_conf(options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" return common.install_version_controlled_file(options_ssl, options_ssl_digest, constants.MOD_SSL_CONF_SRC, constants.ALL_SSL_OPTIONS_HASHES) + +def _determine_default_server_root(): + if os.environ.get("CERTBOT_DOCS") == "1": + default_server_root = "%s or %s" % (constants.LINUX_SERVER_ROOT, + constants.FREEBSD_DARWIN_SERVER_ROOT) + else: + default_server_root = constants.CLI_DEFAULTS["server_root"] + return default_server_root diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index dfc451202..d749b6989 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -2,10 +2,13 @@ import pkg_resources import platform +FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" +LINUX_SERVER_ROOT = "/etc/nginx" + if platform.system() in ('FreeBSD', 'Darwin'): - server_root_tmp = "/usr/local/etc/nginx" + server_root_tmp = FREEBSD_DARWIN_SERVER_ROOT else: - server_root_tmp = "/etc/nginx" + server_root_tmp = LINUX_SERVER_ROOT CLI_DEFAULTS = dict( server_root=server_root_tmp, diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 9386c3cd9..75dfca563 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -949,5 +949,31 @@ class InstallSslOptionsConfTest(util.NginxTest): " with the sha256 hash of self.config.mod_ssl_conf when it is updated.") +class DetermineDefaultServerRootTest(certbot_test_util.ConfigTestCase): + """Tests for certbot_nginx.configurator._determine_default_server_root.""" + + def _call(self): + from certbot_nginx.configurator import _determine_default_server_root + return _determine_default_server_root() + + @mock.patch.dict(os.environ, {"CERTBOT_DOCS": "1"}) + def test_docs_value(self): + self._test(expect_both_values=True) + + @mock.patch.dict(os.environ, {}) + def test_real_values(self): + self._test(expect_both_values=False) + + def _test(self, expect_both_values): + server_root = self._call() + + if expect_both_values: + self.assertIn("/usr/local/etc/nginx", server_root) + self.assertIn("/etc/nginx", server_root) + else: + self.assertTrue("/etc/nginx" in server_root or "/usr/local/etc/nginx" in server_root) + self.assertFalse("/etc/nginx" in server_root and "/usr/local/etc/nginx" in server_root) + + if __name__ == "__main__": unittest.main() # pragma: no cover From a4760cfe56ee88bbc776859fb95a141d128adcaf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 26 Jun 2018 15:33:41 -0700 Subject: [PATCH 265/639] Partially revert "Implement TLS-ALPN-01 challenge and standalone TLS-ALPN server (#5894)" (#6144) This partially reverts commit 15f1405fff7083bf5d4f599a58c54a43be499740. A basic tls-alpn-01 implementation is left so we can successfully parse the challenge so it can be used in boulder's tests. --- acme/acme/challenges.py | 158 +++------------------------- acme/acme/challenges_test.py | 93 +--------------- acme/acme/crypto_util.py | 61 +++-------- acme/acme/crypto_util_test.py | 16 +-- acme/acme/standalone.py | 48 +-------- acme/acme/standalone_test.py | 57 ---------- acme/acme/testdata/README | 6 +- acme/acme/testdata/rsa1024_cert.pem | 13 --- 8 files changed, 32 insertions(+), 420 deletions(-) delete mode 100644 acme/acme/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 30983e28f..2f0bf004d 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,6 +1,5 @@ """ACME Identifier Validation Challenges.""" import abc -import codecs import functools import hashlib import logging @@ -8,7 +7,7 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -from OpenSSL import crypto +import OpenSSL import requests import six @@ -412,8 +411,8 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ if key is None: - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, bits) + key = OpenSSL.crypto.PKey() + key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) return crypto_util.gen_ss_cert(key, [ # z_domain is too big to fit into CN, hence first dummy domain 'dummy', self.z_domain.decode()], force_san=True), key @@ -508,152 +507,19 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) -@ChallengeResponse.register -class TLSALPN01Response(KeyAuthorizationChallengeResponse): - """ACME tls-alpn-01 challenge response.""" - typ = "tls-alpn-01" - - PORT = 443 - """Verification port as defined by the protocol. - - You can override it (e.g. for testing) by passing ``port`` to - `simple_verify`. - - """ - - ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" - ACME_TLS_1_PROTOCOL = "acme-tls/1" - - @property - def h(self): - """Hash value stored in challenge certificate""" - return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() - - def gen_cert(self, domain, key=None, bits=2048): - """Generate tls-alpn-01 certificate. - - :param unicode domain: Domain verified by the challenge. - :param OpenSSL.crypto.PKey key: Optional private key used in - certificate generation. If not provided (``None``), then - fresh key will be generated. - :param int bits: Number of bits for newly generated key. - - :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` - - """ - if key is None: - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, bits) - - - # Instead of using a ASN.1 encoding library just append the OCTET STRING tag (0x04) - # and the length of the SHA256 hash (0x20) since both of these should never change - der_value = b"DER:0420" + codecs.encode(self.h, 'hex') - acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, - critical=True, value=der_value) - - return crypto_util.gen_ss_cert(key, [domain], force_san=True, - extensions=[acme_extension]), key - - def probe_cert(self, domain, host=None, port=None): - """Probe tls-alpn-01 challenge certificate. - - :param unicode domain: domain being validated, required. - :param string host: IP address used to probe the certificate. - :param int port: Port used to probe the certificate. - - """ - if host is None: - host = socket.gethostbyname(domain) - logger.debug('%s resolved to %s', domain, host) - if port is None: - port = self.PORT - - return crypto_util.probe_sni(host=host, port=port, name=domain, - alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) - - def verify_cert(self, domain, cert): - """Verify tls-alpn-01 challenge certificate. - - :param unicode domain: Domain name being validated. - :param OpensSSL.crypto.X509 cert: Challenge certificate. - - :returns: Whether the certificate was successfully verified. - :rtype: bool - - """ - # pylint: disable=protected-access - names = crypto_util._pyopenssl_cert_or_req_all_names(cert) - logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) - if len(names) != 1 or names[0].lower() != domain.lower(): - return False - - for i in range(cert.get_extension_count()): - ext = cert.get_extension(i) - # FIXME: assume this is the ACME extension. Currently there is no - # way to get full OID of an unknown extension from pyopenssl. - if ext.get_short_name() == b'UNDEF': - data = ext.get_data() - # Add the ASN.1 tag/length prefix to the hash before comparison - return data == b'\x04\x20' + self.h - - return False - - # pylint: disable=too-many-arguments - def simple_verify(self, chall, domain, account_public_key, - cert=None, host=None, port=None): - """Simple verify. - - Verify ``validation`` using ``account_public_key``, optionally - probe tls-alpn-01 certificate and check using `verify_cert`. - - :param .challenges.TLSALPN01 chall: Corresponding challenge. - :param str domain: Domain name being validated. - :param JWK account_public_key: - :param OpenSSL.crypto.X509 cert: Optional certificate. If not - provided (``None``) certificate will be retrieved using - `probe_cert`. - :param string host: IP address used to probe the certificate. - :param int port: Port used to probe the certificate. - - - :returns: ``True`` iff client's control of the domain has been - verified. - :rtype: bool - - """ - if not self.verify(chall, account_public_key): - logger.debug("Verification of key authorization in response failed") - return False - - if cert is None: - try: - cert = self.probe_cert(domain=domain, host=host, port=port) - except errors.Error as error: - logger.debug(str(error), exc_info=True) - return False - - return self.verify_cert(cert, domain) - - @Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): - """ACME tls-alpn-01 challenge.""" - response_cls = TLSALPN01Response - typ = response_cls.typ + """ACME tls-alpn-01 challenge. + + This class simply allows parsing the TLS-ALPN-01 challenge returned from + the CA. Full TLS-ALPN-01 support is not currently provided. + + """ + typ = "tls-alpn-01" def validation(self, account_key, **kwargs): - """Generate validation. - - :param JWK account_key: - :param OpenSSL.crypto.PKey cert_key: Optional private key used - in certificate generation. If not provided (``None``), then - fresh key will be generated. - - :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` - - """ - return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) + """Generate validation for the challenge.""" + raise NotImplementedError() @Challenge.register # pylint: disable=too-many-ancestors diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index b929d4939..661d25a35 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -393,91 +393,6 @@ class TLSSNI01Test(unittest.TestCase): mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) -class TLSALPN01ResponseTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes - - def setUp(self): - from acme.challenges import TLSALPN01 - self.chall = TLSALPN01( - token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) - self.domain = u'example.com' - self.domain2 = u'example2.com' - - self.response = self.chall.response(KEY) - self.jmsg = { - 'resource': 'challenge', - 'type': 'tls-alpn-01', - 'keyAuthorization': self.response.key_authorization, - } - - def test_to_partial_json(self): - self.assertEqual(self.jmsg, self.response.to_partial_json()) - - def test_from_json(self): - from acme.challenges import TLSALPN01Response - self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) - - def test_from_json_hashable(self): - from acme.challenges import TLSALPN01Response - hash(TLSALPN01Response.from_json(self.jmsg)) - - def test_gen_verify_cert(self): - key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') - cert, key2 = self.response.gen_cert(self.domain, key1) - self.assertEqual(key1, key2) - self.assertTrue(self.response.verify_cert(self.domain, cert)) - - def test_gen_verify_cert_gen_key(self): - cert, key = self.response.gen_cert(self.domain) - self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) - self.assertTrue(self.response.verify_cert(self.domain, cert)) - - def test_verify_bad_cert(self): - self.assertFalse(self.response.verify_cert(self.domain, - test_util.load_cert('cert.pem'))) - - def test_verify_bad_domain(self): - key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') - cert, key2 = self.response.gen_cert(self.domain, key1) - self.assertEqual(key1, key2) - self.assertFalse(self.response.verify_cert(self.domain2, cert)) - - def test_simple_verify_bad_key_authorization(self): - key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) - self.response.simple_verify(self.chall, "local", key2.public_key()) - - @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) - def test_simple_verify(self, mock_verify_cert): - mock_verify_cert.return_value = mock.sentinel.verification - self.assertEqual( - mock.sentinel.verification, self.response.simple_verify( - self.chall, self.domain, KEY.public_key(), - cert=mock.sentinel.cert)) - mock_verify_cert.assert_called_once_with( - self.response, mock.sentinel.cert, self.domain) - - @mock.patch('acme.challenges.socket.gethostbyname') - @mock.patch('acme.challenges.crypto_util.probe_sni') - def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): - mock_gethostbyname.return_value = '127.0.0.1' - self.response.probe_cert('foo.com') - mock_gethostbyname.assert_called_once_with('foo.com') - mock_probe_sni.assert_called_once_with( - host='127.0.0.1', port=self.response.PORT, name='foo.com', - alpn_protocols=['acme-tls/1']) - - self.response.probe_cert('foo.com', host='8.8.8.8') - mock_probe_sni.assert_called_with( - host='8.8.8.8', port=mock.ANY, name='foo.com', - alpn_protocols=['acme-tls/1']) - - @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') - def test_simple_verify_false_on_probe_error(self, mock_probe_cert): - mock_probe_cert.side_effect = errors.Error - self.assertFalse(self.response.simple_verify( - self.chall, self.domain, KEY.public_key())) - - class TLSALPN01Test(unittest.TestCase): def setUp(self): @@ -506,12 +421,8 @@ class TLSALPN01Test(unittest.TestCase): self.assertRaises( jose.DeserializationError, TLSALPN01.from_json, self.jmsg) - @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') - def test_validation(self, mock_gen_cert): - mock_gen_cert.return_value = ('cert', 'key') - self.assertEqual(('cert', 'key'), self.msg.validation( - KEY, cert_key=mock.sentinel.cert_key)) - mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + def test_validation(self): + self.assertRaises(NotImplementedError, self.msg.validation, KEY) class DNSTest(unittest.TestCase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d25c2340b..d0e203417 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -31,15 +31,6 @@ logger = logging.getLogger(__name__) _DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): - def __init__(self, certs): - self.certs = certs - - def __call__(self, connection): - server_name = connection.get_servername() - return self.certs.get(server_name, None) - - class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. @@ -47,25 +38,12 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. - :ivar alpn_selection: Hook to select negotiated ALPN protocol for - connection. - :ivar cert_selection: Hook to select certificate for connection. If given, - `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs=None, - method=_DEFAULT_TLSSNI01_SSL_METHOD, alpn_selection=None, - cert_selection=None): + def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): self.sock = sock - self.alpn_selection = alpn_selection + self.certs = certs self.method = method - if not cert_selection and not certs: - raise ValueError("Neither cert_selection or certs specified.") - if cert_selection and certs: - raise ValueError("Both cert_selection and certs specified.") - if cert_selection is None: - cert_selection = _DefaultCertSelection(certs) - self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -82,19 +60,18 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :type connection: :class:`OpenSSL.Connection` """ - pair = self.cert_selection(connection) - if pair is None: - logger.debug("Certificate selection for server name %s failed, dropping SSL", - connection.get_servername()) + server_name = connection.get_servername() + try: + key, cert = self.certs[server_name] + except KeyError: + logger.debug("Server name (%s) not recognized, dropping SSL", + server_name) return - key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) - if self.alpn_selection is not None: - new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -119,8 +96,6 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) - if self.alpn_selection is not None: - context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -136,9 +111,8 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0), - alpn_protocols=None): +def probe_sni(name, host, port=443, timeout=300, + method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -150,8 +124,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. - :param alpn_protocols: Protocols to request using ALPN. - :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -188,8 +160,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 - if alpn_protocols is not None: - client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -281,14 +251,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): + validity=(7 * 24 * 60 * 60), force_san=True): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: - :param extensions: List of additional extensions to include in the cert. - :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -301,13 +269,10 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - if extensions is None: - extensions = [] - - extensions.append( + extensions = [ crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ) + ] cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index b661e4e70..36d62b324 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -19,6 +19,7 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" + def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -33,8 +34,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), - certs) + self.socket = SSLSocket(socket.socket(), certs=certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -66,18 +66,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') -class SSLSocketTest(unittest.TestCase): - """Tests for acme.crypto_util.SSLSocket.""" - - def test_ssl_socket_invalid_arguments(self): - from acme.crypto_util import SSLSocket - with self.assertRaises(ValueError): - _ = SSLSocket(None, {'sni': ('key', 'cert')}, - cert_selection=lambda _: None) - with self.assertRaises(ValueError): - _ = SSLSocket(None) - - class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 3bcb0b230..ff9159933 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -43,14 +43,7 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, cert_selection=self._cert_selection, - alpn_selection=getattr(self, '_alpn_selection', None), - method=self.method) - - def _cert_selection(self, connection): - """Callback selecting certificate for connection.""" - server_name = connection.get_servername() - return self.certs.get(server_name, None) + self.socket, certs=self.certs, method=self.method) def server_bind(self): # pylint: disable=missing-docstring self._wrap_sock() @@ -154,45 +147,6 @@ class TLSSNI01DualNetworkedServers(BaseDualNetworkedServers): BaseDualNetworkedServers.__init__(self, TLSSNI01Server, *args, **kwargs) -class BadALPNProtos(Exception): - """Error raised when cannot negotiate ALPN protocol.""" - pass - - -class TLSALPN01Server(TLSServer, ACMEServerMixin): - """TLSALPN01 Server.""" - - ACME_TLS_1_PROTOCOL = b"acme-tls/1" - - def __init__(self, server_address, certs, challenge_certs, ipv6=False): - TLSServer.__init__( - self, server_address, BaseRequestHandlerWithLogging, certs=certs, - ipv6=ipv6) - self.challenge_certs = challenge_certs - - def _cert_selection(self, connection): - # TODO: We would like to serve challenge cert only if asked for it via - # ALPN. To do this, we need to retrieve the list of protos from client - # hello, but this is currently impossible with openssl [0], and ALPN - # negotiation is done after cert selection. - # Therefore, currently we always return challenge cert, and terminate - # handshake in alpn_selection() if ALPN protos are not what we expect. - # [0] https://github.com/openssl/openssl/issues/4952 - server_name = connection.get_servername() - logger.debug("Serving challenge cert for server name %s", server_name) - return self.challenge_certs.get(server_name, None) - - def _alpn_selection(self, _connection, alpn_protos): - """Callback to select alpn protocol.""" - if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: - logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) - return self.ACME_TLS_1_PROTOCOL - # Raising an exception causes openssl to terminate handshake and - # send fatal tls alert. - logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) - raise BadALPNProtos("Got: %s" % str(alpn_protos)) - - class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): """BaseRequestHandler with logging.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index aee592187..1591187e5 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -10,7 +10,6 @@ import unittest from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose import mock import requests @@ -120,62 +119,6 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) -@unittest.skipUnless( - hasattr(SSL.Connection, "set_alpn_protos") and - hasattr(SSL.Context, "set_alpn_select_callback"), - "pyOpenSSL too old") -class TLSALPN01ServerTest(unittest.TestCase): - """Test for acme.standalone.TLSALPN01Server.""" - - def setUp(self): - self.certs = {b'localhost': ( - test_util.load_pyopenssl_private_key('rsa2048_key.pem'), - test_util.load_cert('rsa2048_cert.pem'), - )} - # Use different certificate for challenge. - self.challenge_certs = {b'localhost': ( - test_util.load_pyopenssl_private_key('rsa1024_key.pem'), - test_util.load_cert('rsa1024_cert.pem'), - )} - from acme.standalone import TLSALPN01Server - self.server = TLSALPN01Server(("", 0), certs=self.certs, - challenge_certs=self.challenge_certs) - # pylint: disable=no-member - self.thread = threading.Thread(target=self.server.serve_forever) - self.thread.start() - - def tearDown(self): - self.server.shutdown() # pylint: disable=no-member - self.thread.join() - - #TODO: This is not implemented yet, see comments in standalone.py - #def test_certs(self): - # host, port = self.server.socket.getsockname()[:2] - # cert = crypto_util.probe_sni( - # b'localhost', host=host, port=port, timeout=1) - # # Expect normal cert when connecting without ALPN. - # self.assertEqual(jose.ComparableX509(cert), - # jose.ComparableX509(self.certs[b'localhost'][1])) - - def test_challenge_certs(self): - host, port = self.server.socket.getsockname()[:2] - cert = crypto_util.probe_sni( - b'localhost', host=host, port=port, timeout=1, - alpn_protocols=[b"acme-tls/1"]) - # Expect challenge cert when connecting with ALPN. - self.assertEqual( - jose.ComparableX509(cert), - jose.ComparableX509(self.challenge_certs[b'localhost'][1]) - ) - - def test_bad_alpn(self): - host, port = self.server.socket.getsockname()[:2] - with self.assertRaises(errors.Error): - crypto_util.probe_sni( - b'localhost', host=host, port=port, timeout=1, - alpn_protocols=[b"bad-alpn"]) - - class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" diff --git a/acme/acme/testdata/README b/acme/acme/testdata/README index d65cc3018..dfe3f5405 100644 --- a/acme/acme/testdata/README +++ b/acme/acme/testdata/README @@ -10,8 +10,6 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificates: +and for the certificate: - openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der - openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem - openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem + openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der diff --git a/acme/acme/testdata/rsa1024_cert.pem b/acme/acme/testdata/rsa1024_cert.pem deleted file mode 100644 index 1b7912181..000000000 --- a/acme/acme/testdata/rsa1024_cert.pem +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV -BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow -FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ -AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr -Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW -l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G -A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X -XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB -ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI -Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY -qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x ------END CERTIFICATE----- From 2304f7fcdadd6baad61a925e9a11541d2b534d6f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 27 Jun 2018 19:32:53 +0300 Subject: [PATCH 266/639] Remove unnecessary dotfiles (#6151) --- .../augeas_vhosts/apache2/mods-enabled/.gitignore | 0 .../multiple_vhosts/apache2/mods-enabled/.gitignore | 0 .../apache/apache2/modules.d/.keep_www-servers_apache-2 | 0 .../apache/apache2/vhosts.d/.keep_www-servers_apache-2 | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore delete mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore delete mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 delete mode 100644 certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/mods-enabled/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/mods-enabled/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/modules.d/.keep_www-servers_apache-2 deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 b/certbot-apache/certbot_apache/tests/testdata/gentoo_apache/apache/apache2/vhosts.d/.keep_www-servers_apache-2 deleted file mode 100644 index e69de29bb..000000000 From f169e7374bedf11a77c860d522f7e9b7eaad0824 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 27 Jun 2018 21:35:01 +0300 Subject: [PATCH 267/639] Interactive certificate selection with install verb (#6097) If either --cert-name or both --key-path and --cert-path (in which case the user requests installation for a certificate not managed by Certbot) are not provided, prompt the user with managed certificates and let them choose. Fixes: #5824 --- certbot/main.py | 7 +++++++ certbot/tests/main_test.py | 26 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index a457919d4..6078f87a6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -789,6 +789,13 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) + custom_cert = (config.key_path and config.cert_path) + if not config.certname and not custom_cert: + certname_question = "Which certificate would you like to install?" + config.certname = cert_manager.get_certnames( + config, "install", allow_multiple=False, + custom_prompt=certname_question)[0] + if not enhancements.are_supported(config, installer): raise errors.NotSupportedError("One ore more of the requested enhancements " "are not supported by the selected installer") diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 24d059e53..a2115a486 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -761,20 +761,25 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_param_error(self, _inst, _rec): - self.assertRaises(errors.ConfigurationError, - self._call, - ['install', '--key-path', '/tmp/key_path']) - self.assertRaises(errors.ConfigurationError, - self._call, - ['install', '--cert-path', '/tmp/key_path']) - self.assertRaises(errors.ConfigurationError, - self._call, - ['install']) self.assertRaises(errors.ConfigurationError, self._call, ['install', '--cert-name', 'notfound', '--key-path', 'invalid']) + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + @mock.patch('certbot.cert_manager.get_certnames') + @mock.patch('certbot.main._install_cert') + def test_installer_select_cert(self, mock_inst, mock_getcert, _inst, _rec): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install'], mockisfile=True) + self.assertTrue(mock_getcert.called) + self.assertTrue(mock_inst.called) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -1742,6 +1747,7 @@ class InstallTest(test_util.ConfigTestCase): mock_inst.return_value = null.Installer(self.config, "null") plugins = disco.PluginsRegistry.find_all() self.config.auto_hsts = True + self.config.certname = "nonexistent" self.assertRaises(errors.NotSupportedError, main.install, self.config, plugins) @@ -1753,6 +1759,8 @@ class InstallTest(test_util.ConfigTestCase): plugins = disco.PluginsRegistry.find_all() self.config.auto_hsts = True self.config.certname = None + self.config.key_path = "/tmp/nonexistent" + self.config.cert_path = "/tmp/nonexistent" self.assertRaises(errors.ConfigurationError, main.install, self.config, plugins) From ad3c547e1fcd88d7877f0abbed3cf999ba7fc94b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Jun 2018 14:29:21 -0700 Subject: [PATCH 268/639] Update cli-help.txt to use generic values (#6143) --- docs/cli-help.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 49821a7b5..594bcfd04 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,12 +108,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.1 (certbot; - darwin 10.13.5) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.15). The flags - encoded in the user agent are: --duplicate, --force- - renew, --allow-subset-of-names, -n, and whether any - hooks are set. + "". (default: CertbotACMEClient/0.25.1 + (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX + Installer/YYY (SUBCOMMAND; flags: FLAGS) + Py/major.minor.patchlevel). The flags encoded in the + user agent are: --duplicate, --force-renew, --allow- + subset-of-names, -n, and whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -638,7 +638,7 @@ nginx: Nginx Web Server plugin - Alpha --nginx-server-root NGINX_SERVER_ROOT - Nginx server root directory. (default: + Nginx server root directory. (default: /etc/nginx or /usr/local/etc/nginx) --nginx-ctl NGINX_CTL Path to the 'nginx' binary, used for 'configtest' and From 742a57722ba99812920baae47a8d6b094859a952 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Jun 2018 17:35:43 -0700 Subject: [PATCH 269/639] fix server_root default tests on macOS (#6149) --- certbot-nginx/certbot_nginx/tests/configurator_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 75dfca563..4d23f3518 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -971,8 +971,7 @@ class DetermineDefaultServerRootTest(certbot_test_util.ConfigTestCase): self.assertIn("/usr/local/etc/nginx", server_root) self.assertIn("/etc/nginx", server_root) else: - self.assertTrue("/etc/nginx" in server_root or "/usr/local/etc/nginx" in server_root) - self.assertFalse("/etc/nginx" in server_root and "/usr/local/etc/nginx" in server_root) + self.assertTrue(server_root == "/etc/nginx" or server_root == "/usr/local/etc/nginx") if __name__ == "__main__": From d00a31622dd4f797a75b140b95320e99bcc4c953 Mon Sep 17 00:00:00 2001 From: Bahram Aghaei Date: Thu, 28 Jun 2018 17:43:52 +0000 Subject: [PATCH 270/639] run Isort on imported packages (#6138) --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e32925583..8176883ad 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,7 @@ import codecs import os import re -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup # Workaround for http://bugs.python.org/issue8876, see # http://bugs.python.org/issue8876#msg208792 From 64e06d4201a8695580533c3cb07370fac226ebea Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 28 Jun 2018 10:55:21 -0700 Subject: [PATCH 271/639] Use greater than or equal to in requirements. (#6117) * Use greater than or equal to in requirements. This changes the existing requirements using strictly greater than to greater than or equal to so that they're more conventional. * Use >= for certbot-postfix. Despite it previously saying 'certbot>0.23.0', certbot-postfix/local-oldest-requirements.txt was pinned to 0.23.0 so let's just use certbot>=0.23.0. --- certbot-apache/setup.py | 4 ++-- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot-postfix/setup.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 8d15e7971..ffaa6a863 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -7,8 +7,8 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>0.24.0', - 'certbot>0.25.1', + 'acme>=0.25.0', + 'certbot>=0.26.0.dev0', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8e2821332..c8806e862 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -6,7 +6,7 @@ version = '0.26.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>0.24.0', + 'acme>=0.25.0', 'certbot>=0.21.1', 'boto3', 'mock', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index d9cb4a9c2..b486b2778 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -8,7 +8,7 @@ version = '0.26.0.dev0' # acme/certbot version. install_requires = [ 'acme>=0.25.0', - 'certbot>0.21.1', + 'certbot>=0.22.0', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index cde1ac7c2..4c53477d2 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -6,7 +6,7 @@ version = '0.26.0.dev0' install_requires = [ 'acme>=0.25.0', - 'certbot>0.23.0', + 'certbot>=0.23.0', 'setuptools', 'six', 'zope.component', diff --git a/setup.py b/setup.py index 8176883ad..9ef9ec0d2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>0.24.0', + 'acme>=0.25.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From a816cc89798823cc043058f016e4d89cb3fca4f6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 28 Jun 2018 13:34:16 -0700 Subject: [PATCH 272/639] Use account reuse symlink logic when loading an account (#6156) Fixes #6154. * add symlinking to load flow * test account reuse on load --- certbot/account.py | 36 ++++++++++++++++++++++------------- certbot/tests/account_test.py | 8 ++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index c4eeb1388..5e9455048 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -175,33 +175,43 @@ class AccountFileStorage(interfaces.AccountStorage): except errors.AccountStorageError: logger.debug("Account loading problem", exc_info=True) - if not accounts and server_path in constants.LE_REUSE_SERVERS: # find all for the next link down prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_accounts = self._find_all_for_server_path(prev_server_path) # if we found something, link to that if prev_accounts: - if os.path.islink(accounts_dir): - os.unlink(accounts_dir) - else: - try: - os.rmdir(accounts_dir) - except OSError: - return [] - prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) - os.symlink(prev_account_dir, accounts_dir) + try: + self._symlink_to_accounts_dir(prev_server_path, server_path) + except OSError: + return [] accounts = prev_accounts return accounts def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + if os.path.islink(accounts_dir): + os.unlink(accounts_dir) + else: + os.rmdir(accounts_dir) + prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) + os.symlink(prev_account_dir, accounts_dir) + def _load_for_server_path(self, account_id, server_path): account_dir_path = self._account_dir_path_for_server_path(account_id, server_path) - if not os.path.isdir(account_dir_path): - raise errors.AccountNotFound( - "Account at %s does not exist" % account_dir_path) + if not os.path.isdir(account_dir_path): # isdir is also true for symlinks + if server_path in constants.LE_REUSE_SERVERS: + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) + # we didn't error so we found something, so create a symlink to that + self._symlink_to_accounts_dir(prev_server_path, server_path) + return prev_loaded_account + else: + raise errors.AccountNotFound( + "Account at %s does not exist" % account_dir_path) try: with open(self._regr_path(account_dir_path)) as regr_file: diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index a4fe5edb7..e7f82a5b8 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -241,6 +241,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + def test_upgrade_load(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() From 6e13c2ccc71e7629f451f89339870013fad7dc30 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 28 Jun 2018 15:06:52 -0700 Subject: [PATCH 273/639] Add --disable=locally-enabled to .pylintrc. (#6159) --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 36d8c286f..1d3f0ac4f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -41,7 +41,7 @@ load-plugins=linter_plugin # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code +disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code # abstract-class-not-used cannot be disabled locally (at least in # pylint 1.4.1), same for abstract-class-little-used From 552e60a12691d434ea86a7c6bf5bc4a270651ba8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 29 Jun 2018 05:27:58 -0700 Subject: [PATCH 274/639] Don't use hardcoded port in tests (#6145) * Don't use port 1234 in standalone tests. * rename unused variable * add back failure case * Add back probe connection error test. * fix lint * remove unused import * fix test file coverage * prevent future heisenbug --- acme/acme/crypto_util_test.py | 26 ++++++++++++++++-------- acme/acme/standalone_test.py | 37 ++++++++++++++--------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 36d62b324..168489d86 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -42,28 +42,38 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.server_thread = threading.Thread( # pylint: disable=no-member target=self.server.handle_request) - self.server_thread.start() - time.sleep(1) # TODO: avoid race conditions in other way def tearDown(self): - self.server_thread.join() + if self.server_thread.is_alive(): + # The thread may have already terminated. + self.server_thread.join() # pragma: no cover def _probe(self, name): from acme.crypto_util import probe_sni return jose.ComparableX509(probe_sni( name, host='127.0.0.1', port=self.port)) + def _start_server(self): + self.server_thread.start() + time.sleep(1) # TODO: avoid race conditions in other way + def test_probe_ok(self): + self._start_server() self.assertEqual(self.cert, self._probe(b'foo')) def test_probe_not_recognized_name(self): + self._start_server() self.assertRaises(errors.Error, self._probe, b'bar') - # TODO: py33/py34 tox hangs forever on do_handshake in second probe - #def probe_connection_error(self): - # self._probe(b'foo') - # #time.sleep(1) # TODO: avoid race conditions in other way - # self.assertRaises(errors.Error, self._probe, b'bar') + def test_probe_connection_error(self): + # pylint has a hard time with six + self.server.server_close() # pylint: disable=no-member + original_timeout = socket.getdefaulttimeout() + try: + socket.setdefaulttimeout(1) + self.assertRaises(errors.Error, self._probe, b'bar') + finally: + socket.setdefaulttimeout(original_timeout) class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 1591187e5..6beea038e 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -4,10 +4,10 @@ import shutil import socket import threading import tempfile -import time import unittest from six.moves import http_client # pylint: disable=import-error +from six.moves import queue # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error import josepy as jose @@ -16,7 +16,6 @@ import requests from acme import challenges from acme import crypto_util -from acme import errors from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module @@ -261,10 +260,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): os.path.join(localhost_dir, 'key.pem')) from acme.standalone import simple_tls_sni_01_server - self.port = 1234 self.thread = threading.Thread( target=simple_tls_sni_01_server, kwargs={ - 'cli_args': ('xxx', '--port', str(self.port)), + 'cli_args': ('filename',), 'forever': False, }, ) @@ -276,25 +274,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): self.thread.join() shutil.rmtree(self.test_cwd) - def test_it(self): - max_attempts = 5 - for attempt in range(max_attempts): - try: - cert = crypto_util.probe_sni( - b'localhost', b'0.0.0.0', self.port) - except errors.Error: - self.assertTrue(attempt + 1 < max_attempts, "Timeout!") - time.sleep(1) # wait until thread starts - else: - self.assertEqual(jose.ComparableX509(cert), - test_util.load_comparable_cert( - 'rsa2048_cert.pem')) - break + @mock.patch('acme.standalone.logger') + def test_it(self, mock_logger): + # Use a Queue because mock objects aren't thread safe. + q = queue.Queue() # type: queue.Queue[int] + # Add port number to the queue. + mock_logger.info.side_effect = lambda *args: q.put(args[-1]) + self.thread.start() - if attempt == 0: - # the first attempt is always meant to fail, so we can test - # the socket failure code-path for probe_sni, as well - self.thread.start() + # After the timeout, an exception is raised if the queue is empty. + port = q.get(timeout=5) + cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port) + self.assertEqual(jose.ComparableX509(cert), + test_util.load_comparable_cert( + 'rsa2048_cert.pem')) if __name__ == "__main__": From ab9851d97e7f58290d0299da8fec49f072226750 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Jul 2018 06:57:58 -0700 Subject: [PATCH 275/639] Upgrade to the latest cryptography version (#6169) This allows certbot-auto and our development setup to work with Python 3.7. --- letsencrypt-auto-source/letsencrypt-auto | 51 ++++++++----------- .../pieces/dependency-requirements.txt | 51 ++++++++----------- 2 files changed, 40 insertions(+), 62 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index dc9190630..d571a5a8d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 376e19deb..54498cb3e 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -64,37 +64,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 From cad95466b05e6be51c1c29eaa91e6e3b7ea3cefd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Jul 2018 02:11:04 -0700 Subject: [PATCH 276/639] Allow py37 testing (#6170) * Reorganize packages in tox to allow for py37 tests certbot-dns-cloudflare doesn't currently work in Python 3.7 because it transitively depends on pyYAML which doesn't yet support Python 3.7. See https://github.com/yaml/pyyaml/issues/126 for more info. * add py37 tox environment --- tox.ini | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ef71b52be..b44d30449 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -dns_packages = - certbot-dns-cloudflare \ +python37_compatible_dns_packages = certbot-dns-cloudxns \ certbot-dns-digitalocean \ certbot-dns-dnsimple \ @@ -25,14 +24,22 @@ dns_packages = certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 -all_packages = +dns_packages = + certbot-dns-cloudflare \ + {[base]python37_compatible_dns_packages} +nondns_packages = acme[dev] \ .[dev] \ certbot-apache \ - {[base]dns_packages} \ certbot-nginx \ certbot-postfix \ letshelp-certbot +python37_compatible_packages = + {[base]nondns_packages} \ + {[base]python37_compatible_dns_packages} +all_packages = + {[base]nondns_packages} \ + {[base]dns_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -62,6 +69,13 @@ commands = setenv = PYTHONHASHSEED = 0 +[testenv:py37] +commands = + {[base]install_and_test} {[base]python37_compatible_packages} + python tests/lock_test.py +setenv = + {[testenv]setenv} + [testenv:py27-oldest] commands = {[testenv]commands} From cb076539ec99423b8e3f8ff7f0bc4fa2b7452766 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Jul 2018 08:26:42 -0700 Subject: [PATCH 277/639] Remove .dev0 from version numbers during releases. (#6116) This allows us to depend on packages like acme>=0.26.0.dev0 during development and automatically change it to acme>=0.26.0 during the release. We use `git add -p` to be safe, but if .dev0 is used at all in our released setup.py files, we're probably doing something wrong. --- tools/release.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/release.sh b/tools/release.sh index 069d311d1..02c7ccca5 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -87,6 +87,14 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" +for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +do + sed -i 's/\.dev0//' "$pkg_dir/setup.py" +done +# We only add Certbot's setup.py here because the other files are added in the +# call to SetVersion below. +git add -p setup.py + SetVersion() { ver="$1" # bumping Certbot's version number is done differently From 08378203df8978ec8b0c971abb88a0c51d094521 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 6 Jul 2018 09:08:40 -0700 Subject: [PATCH 278/639] Add Python 3.7 tests (#6179) * Remove apacheconftest packages. The apacheconftests handle installing Apache dependencies, so let's remove it from the general case. * We don't need to run dpkg -s in before_install. * Remove augeas sources. We only needed it for Ubuntu Precise which is dead and it doesn't work in Ubuntu Xenial. * Upgrade Python 3.6 tests to 3.7. Let's continue the approach of testing on the oldest and newest versions of Python 3. We will continue testing on Python 3.6 in the nightly tests. * Revert "We don't need to run dpkg -s in before_install." This reverts commit e5d35099a79985ee97a26931e08451620d711522. * let apacheconftest handle deps --- .travis.yml | 11 +++-------- .../tests/apache-conf-files/apache-conf-test | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3d964326..937d24610 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,8 +41,9 @@ matrix: env: TOXENV=py34 sudo: required services: docker - - python: "3.6" - env: TOXENV=py36 + - python: "3.7" + dist: xenial + env: TOXENV=py37 sudo: required services: docker - sudo: required @@ -77,8 +78,6 @@ sudo: false addons: apt: - sources: - - augeas packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder. - python-dev - python-virtualenv @@ -90,10 +89,6 @@ addons: # For certbot-nginx integration testing - nginx-light - openssl - # for apacheconftest - - apache2 - - libapache2-mod-wsgi - - libapache2-mod-macro install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" script: diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test index e0715a46b..dcbba9d3e 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test @@ -46,6 +46,7 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then + sudo apt-get install -y apache2 sudo apt-get install -y libapache2-mod-wsgi sudo apt-get install -y libapache2-mod-macro From 2564566e1c512619f71bb8bbdc77821907a46ebe Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 6 Jul 2018 23:19:29 +0300 Subject: [PATCH 279/639] Do not call IPlugin.prepare() for updaters when running renew (#6167) interfaces.GenericUpdater and new enhancement interface updater functions get run on every invocation of Certbot with "renew" verb for every lineage. This causes performance problems for users with large configurations, because of plugin plumbing and preparsing happening in prepare() method of installer plugins. This PR moves the responsibility to call prepare() to the plugin (possibly) implementing a new style enhancement interface. Fixes: #6153 * Do not call IPlugin.prepare() for updaters when running renew * Check prepare called in tests * Refine pydoc and make the function name more informative * Verify the plugin type --- certbot-apache/certbot_apache/configurator.py | 5 ++ .../certbot_apache/tests/autohsts_test.py | 5 +- certbot/interfaces.py | 3 ++ certbot/main.py | 2 +- certbot/plugins/enhancements.py | 7 ++- certbot/plugins/selection.py | 29 +++++++++++ certbot/plugins/selection_test.py | 48 ++++++++++++++++++- certbot/tests/main_test.py | 14 +++--- certbot/tests/renewupdater_test.py | 22 +++++---- certbot/updater.py | 10 ++-- 10 files changed, 119 insertions(+), 26 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index ab83a5332..bb77e2e41 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -165,6 +165,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] # These will be set in the prepare function + self._prepared = False self.parser = None self.version = version self.vhosts = None @@ -249,6 +250,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( "Unable to lock %s", self.conf("server-root")) + self._prepared = True def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. @@ -2394,6 +2396,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): continue nextstep = config["laststep"] + 1 if nextstep < len(constants.AUTOHSTS_STEPS): + # If installer hasn't been prepared yet, do it now + if not self._prepared: + self.prepare() # Have not reached the max value yet try: vhost = self.find_vhost_by_id(id_str) diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 86d985079..73da33f15 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -55,7 +55,9 @@ class AutoHSTSTest(util.ApacheTest): @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_autohsts_increase(self, _mock_restart): + @mock.patch("certbot_apache.configurator.ApacheConfigurator.prepare") + def test_autohsts_increase(self, mock_prepare, _mock_restart): + self.config._prepared = False maxage = "\"max-age={0}\"" initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) inc_val = maxage.format(constants.AUTOHSTS_STEPS[1]) @@ -69,6 +71,7 @@ class AutoHSTSTest(util.ApacheTest): # Verify increased value self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), inc_val) + self.assertTrue(mock_prepare.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase") diff --git a/certbot/interfaces.py b/certbot/interfaces.py index a5fb426e6..2e837d1d2 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -620,6 +620,9 @@ class GenericUpdater(object): methods, and interfaces.GenericUpdater.register(InstallerClass) should be called from the installer code. + The plugins implementing this enhancement are responsible of handling + the saving of configuration checkpoints as well as other calls to + interface methods of `interfaces.IInstaller` such as prepare() and restart() """ @abc.abstractmethod diff --git a/certbot/main.py b/certbot/main.py index 6078f87a6..556722104 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1199,11 +1199,11 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. + # Run deployer updater.run_renewal_deployer(config, renewed_lineage, installer) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) - # Run deployer def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. diff --git a/certbot/plugins/enhancements.py b/certbot/plugins/enhancements.py index 506abe433..7ca096610 100644 --- a/certbot/plugins/enhancements.py +++ b/certbot/plugins/enhancements.py @@ -88,7 +88,8 @@ class AutoHSTSEnhancement(object): The plugins implementing new style enhancements are responsible of handling the saving of configuration checkpoints as well as calling possible restarts - of managed software themselves. + of managed software themselves. For update_autohsts method, the installer may + have to call prepare() to finalize the plugin initialization. Methods: enable_autohsts is called when the header is initially installed using a @@ -112,6 +113,10 @@ class AutoHSTSEnhancement(object): :param lineage: Certificate lineage object :type lineage: certbot.storage.RenewableCert + + .. note:: prepare() method inherited from `interfaces.IPlugin` might need + to be called manually within implementation of this interface method + to finalize the plugin initialization. """ @abc.abstractmethod diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 030d5b6db..95f123a46 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -39,6 +39,35 @@ def pick_authenticator( return pick_plugin( config, default, plugins, question, (interfaces.IAuthenticator,)) +def get_unprepared_installer(config, plugins): + """ + Get an unprepared interfaces.IInstaller object. + + :param certbot.interfaces.IConfig config: Configuration + :param certbot.plugins.disco.PluginsRegistry plugins: + All plugins registered as entry points. + + :returns: Unprepared installer plugin or None + :rtype: IPlugin or None + """ + + _, req_inst = cli_plugin_requests(config) + if not req_inst: + return None + installers = plugins.filter(lambda p_ep: p_ep.name == req_inst) + installers.init(config) + installers = installers.verify((interfaces.IInstaller,)) + if len(installers) > 1: + raise errors.PluginSelectionError( + "Found multiple installers with the name %s, Certbot is unable to " + "determine which one to use. Skipping." % req_inst) + if installers: + inst = list(installers.values())[0] + logger.debug("Selecting plugin: %s", inst) + return inst.init(config) + else: + raise errors.PluginSelectionError( + "Could not select or initialize the requested installer %s." % req_inst) def pick_plugin(config, default, plugins, question, ifaces): """Pick plugin. diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index ab480544a..44d64ab8e 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -6,10 +6,13 @@ import unittest import mock import zope.component +from certbot import errors +from certbot import interfaces + from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.display import util as display_util +from certbot.plugins.disco import PluginsRegistry from certbot.tests import util as test_util -from certbot import interfaces class ConveniencePickPluginTest(unittest.TestCase): @@ -170,5 +173,48 @@ class ChoosePluginTest(unittest.TestCase): self.assertTrue("default" in mock_util().menu.call_args[1]) +class GetUnpreparedInstallerTest(test_util.ConfigTestCase): + """Tests for certbot.plugins.selection.get_unprepared_installer.""" + + def setUp(self): + super(GetUnpreparedInstallerTest, self).setUp() + self.mock_apache_fail_ep = mock.Mock( + description_with_name="afail") + self.mock_apache_fail_ep.name = "afail" + self.mock_apache_ep = mock.Mock( + description_with_name="apache") + self.mock_apache_ep.name = "apache" + self.mock_apache_plugin = mock.MagicMock() + self.mock_apache_ep.init.return_value = self.mock_apache_plugin + self.plugins = PluginsRegistry({ + "afail": self.mock_apache_fail_ep, + "apache": self.mock_apache_ep, + }) + + def _call(self): + from certbot.plugins.selection import get_unprepared_installer + return get_unprepared_installer(self.config, self.plugins) + + def test_no_installer_defined(self): + self.config.configurator = None + self.assertEquals(self._call(), None) + + def test_no_available_installers(self): + self.config.configurator = "apache" + self.plugins = PluginsRegistry({}) + self.assertRaises(errors.PluginSelectionError, self._call) + + def test_get_plugin(self): + self.config.configurator = "apache" + installer = self._call() + self.assertTrue(installer is self.mock_apache_plugin) + + def test_multiple_installers_returned(self): + self.config.configurator = "apache" + # Two plugins with the same name + self.mock_apache_fail_ep.name = "apache" + self.assertRaises(errors.PluginSelectionError, self._call) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index a2115a486..cc4e6c293 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1493,17 +1493,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(mock_handle.called) @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_plugin_selection_error(self, mock_choose): + @mock.patch('certbot.updater._run_updaters') + def test_plugin_selection_error(self, mock_run, mock_choose): mock_choose.side_effect = errors.PluginSelectionError self.assertRaises(errors.PluginSelectionError, main.renew_cert, None, None, None) - with mock.patch('certbot.updater.logger.warning') as mock_log: - self.config.dry_run = False - updater.run_generic_updaters(self.config, None, None) - self.assertTrue(mock_log.called) - self.assertTrue("Could not choose appropriate plugin for updaters" - in mock_log.call_args[0][0]) + self.config.dry_run = False + updater.run_generic_updaters(self.config, None, None) + # Make sure we're returning None, and hence not trying to run the + # without installer + self.assertFalse(mock_run.called) class UnregisterTest(unittest.TestCase): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c1b97843e..5a362072c 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -23,13 +23,15 @@ class RenewUpdaterTest(test_util.ConfigTestCase): @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + @mock.patch('certbot.plugins.selection.get_unprepared_installer') @test_util.patch_get_utility() - def test_server_updates(self, _, mock_select, mock_getsave): + def test_server_updates(self, _, mock_geti, mock_select, mock_getsave): mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater mock_select.return_value = (mock_generic_updater, None) + mock_geti.return_value = mock_generic_updater with mock.patch('certbot.main._init_le_client'): main.renew_cert(self.config, None, mock.MagicMock()) self.assertTrue(mock_generic_updater.restart.called) @@ -62,9 +64,9 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.assertEquals(mock_log.call_args[0][0], "Skipping renewal deployer in dry-run mode.") - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_updates(self, mock_select): - mock_select.return_value = (self.mockinstaller, None) + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_updates(self, mock_geti): + mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertTrue(self.mockinstaller.update_autohsts.called) self.assertEqual(self.mockinstaller.update_autohsts.call_count, 1) @@ -74,10 +76,10 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.mockinstaller) self.assertTrue(self.mockinstaller.deploy_autohsts.called) - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_updates_not_called(self, mock_select): + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_updates_not_called(self, mock_geti): self.config.disable_renew_updates = True - mock_select.return_value = (self.mockinstaller, None) + mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertFalse(self.mockinstaller.update_autohsts.called) @@ -87,8 +89,8 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.mockinstaller) self.assertFalse(self.mockinstaller.deploy_autohsts.called) - @mock.patch('certbot.plugins.selection.choose_configurator_plugins') - def test_enhancement_no_updater(self, mock_select): + @mock.patch('certbot.plugins.selection.get_unprepared_installer') + def test_enhancement_no_updater(self, mock_geti): FAKEINDEX = [ { "name": "Test", @@ -98,7 +100,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): "enable_function": "enable_autohsts" } ] - mock_select.return_value = (self.mockinstaller, None) + mock_geti.return_value = self.mockinstaller with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertFalse(self.mockinstaller.update_autohsts.called) diff --git a/certbot/updater.py b/certbot/updater.py index fb7c52f77..58df6fcb4 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -28,13 +28,13 @@ def run_generic_updaters(config, lineage, plugins): logger.debug("Skipping updaters in dry-run mode.") return try: - # installers are used in auth mode to determine domain names - installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: + installer = plug_sel.get_unprepared_installer(config, plugins) + except errors.Error as e: logger.warning("Could not choose appropriate plugin for updaters: %s", e) return - _run_updaters(lineage, installer, config) - _run_enhancement_updaters(lineage, installer, config) + if installer: + _run_updaters(lineage, installer, config) + _run_enhancement_updaters(lineage, installer, config) def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used From dd600db436542f79363f1fbdc50e1a2f6cf0ba42 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:16:08 -0700 Subject: [PATCH 280/639] Upgrade pinned josepy version (#6184) We released josepy 1.1.0 a while ago to work around newer versions of cryptography deprecating some of the functionality we were using. We haven't yet upgraded our pinned josepy version though and since #6169 has landed, we're now seeing these deprecation warnings in our tests. This would be shown to certbot-auto users as well. This PR removes these warnings by upgrading our pinned version of josepy. * update pinned josepy version * build leauto * update pinned dev version of josepy --- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/dependency-requirements.txt | 6 +++--- tools/dev_constraints.txt | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d571a5a8d..9afa86849 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1092,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 54498cb3e..ae6079d96 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -96,9 +96,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 777222ffb..4a392e0b4 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -26,7 +26,7 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 -josepy==1.0.1 +josepy==1.1.0 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 From cdf93de33899a95a544bbf7b99ddc1c327cc2728 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:16:44 -0700 Subject: [PATCH 281/639] Full Python 3.7 support (#6182) Now that yaml/pyyaml#126 is resolved, #6170 can be reverted by bumping the pinned version of PyYAML. You can see this code passing with full macOS and integration tests at https://travis-ci.org/certbot/certbot/builds/400957729. * Revert "Allow py37 testing (#6170)" This reverts commit cad95466b05e6be51c1c29eaa91e6e3b7ea3cefd. * Bump pyyaml pinning to work on Python 3.7. --- tools/dev_constraints.txt | 2 +- tox.ini | 22 ++++------------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 4a392e0b4..5a16b8cba 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,7 +51,7 @@ pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 python-digitalocean==1.11 -PyYAML==3.12 +PyYAML==3.13 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 requests-toolbelt==0.8.0 diff --git a/tox.ini b/tox.ini index b44d30449..ef71b52be 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,8 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -python37_compatible_dns_packages = +dns_packages = + certbot-dns-cloudflare \ certbot-dns-cloudxns \ certbot-dns-digitalocean \ certbot-dns-dnsimple \ @@ -24,22 +25,14 @@ python37_compatible_dns_packages = certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 -dns_packages = - certbot-dns-cloudflare \ - {[base]python37_compatible_dns_packages} -nondns_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ + {[base]dns_packages} \ certbot-nginx \ certbot-postfix \ letshelp-certbot -python37_compatible_packages = - {[base]nondns_packages} \ - {[base]python37_compatible_dns_packages} -all_packages = - {[base]nondns_packages} \ - {[base]dns_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -69,13 +62,6 @@ commands = setenv = PYTHONHASHSEED = 0 -[testenv:py37] -commands = - {[base]install_and_test} {[base]python37_compatible_packages} - python tests/lock_test.py -setenv = - {[testenv]setenv} - [testenv:py27-oldest] commands = {[testenv]commands} From 43f2bfd6f18b9ab1375b27fc0fd9bf73be64d8ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Jul 2018 09:17:03 -0700 Subject: [PATCH 282/639] Advertise our packages work on Python 3.7. (#6183) --- acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot-postfix/setup.py | 1 + letshelp-certbot/setup.py | 1 + setup.py | 1 + 17 files changed, 17 insertions(+) diff --git a/acme/setup.py b/acme/setup.py index ecafac61a..8332febc1 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -68,6 +68,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index ffaa6a863..aca7f3bce 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bf86df5da..34e39ec5e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -46,6 +46,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 055a8cffc..9cf26aa52 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 2c0c074be..8077fe8f5 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index bd5f2ff26..5877d2d0d 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 581c478b8..8c00e6d58 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c5d310d71..34772c422 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 01d979c2b..ad56e58be 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -47,6 +47,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 44f67c0a7..796f7a489 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 87a7da4cc..d87470c3a 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c1fffc4a2..bbfbcbba1 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -42,6 +42,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index c8806e862..8836dc6d8 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b486b2778..09192a956 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-postfix/setup.py b/certbot-postfix/setup.py index 4c53477d2..0ff2908df 100644 --- a/certbot-postfix/setup.py +++ b/certbot-postfix/setup.py @@ -40,6 +40,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Communications :: Email :: Mail Transport Agents', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 28ce0e962..3e9e31725 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -35,6 +35,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/setup.py b/setup.py index 9ef9ec0d2..5a68ff40e 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', From 83f7e72fefb8d9087a5ad488153a644e1b905572 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 10 Jul 2018 13:03:25 -0700 Subject: [PATCH 283/639] Update and delete registration with account reuse (#6174) * find the correct url when deactivating an acmev1 account on the acmev2 endpoint * set regr in ClientNetwork.account after deactivating on the server * update self.net.account * move logic into update_registration * return methods to their original order to please git * factor out common code * update test_fowarding to use a method that still gets forwarded * add acme module test coverage * pragma no cover on correct line * use previous regr uri * strip unnecessary items from regr before saving * add explanation to main.py * add extra check to client_test.py * use empty dict instead of empty string to indicate lack of body that we save to disk --- acme/acme/client.py | 26 +++++++++++++++++++++++++- acme/acme/client_test.py | 22 +++++++++++++++++++++- acme/acme/messages.py | 1 + certbot/account.py | 9 ++++++--- certbot/main.py | 5 +++++ 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index bdc07fb1c..a0bfe460d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -50,7 +50,6 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :ivar .ClientNetwork net: Client network. :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, net, acme_version): """Initialize. @@ -588,6 +587,30 @@ class ClientV2(ClientBase): self.net.account = regr return regr + def update_registration(self, regr, update=None): + """Update registration. + + :param messages.RegistrationResource regr: Registration Resource. + :param messages.Registration update: Updated body of the + resource. If not provided, body will be taken from `regr`. + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + # https://github.com/certbot/certbot/issues/6155 + new_regr = self._get_v2_account(regr) + return super(ClientV2, self).update_registration(new_regr, update) + + def _get_v2_account(self, regr): + self.net.account = None + only_existing_reg = regr.body.update(only_return_existing=True) + response = self._post(self.directory['newAccount'], only_existing_reg) + updated_uri = response.headers['Location'] + new_regr = regr.update(uri=updated_uri) + self.net.account = new_regr + return new_regr + def new_order(self, csr_pem): """Request a new Order object from the server. @@ -910,6 +933,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes if acme_version == 2: kwargs["url"] = url # newAccount and revokeCert work without the kid + # newAccount must not have kid if self.account is not None: kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index f3018ed81..cd31c4ac3 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -139,7 +139,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - self.assertEqual(client.update_registration, client.client.update_registration) + self.assertEqual(client.deactivate_registration, client.client.deactivate_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') self.assertRaises(AttributeError, client.__getattr__, 'new_account') @@ -270,6 +270,13 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.revoke(messages_test.CERT, self.rsn) mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + def test_update_registration(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.update_registration(mock.sentinel.regr, None) + mock_client().update_registration.assert_called_once_with(mock.sentinel.regr, None) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -789,6 +796,19 @@ class ClientV2Test(ClientTestBase): self.net.post.assert_called_once_with( self.directory["revokeCert"], mock.ANY, acme_version=2) + def test_update_registration(self): + # "Instance of 'Field' has no to_json/update member" bug: + # pylint: disable=no-member + self.response.headers['Location'] = self.regr.uri + self.response.json.return_value = self.regr.body.to_json() + self.assertEqual(self.regr, self.client.update_registration(self.regr)) + self.assertNotEqual(self.client.net.account, None) + self.assertEqual(self.client.net.post.call_count, 2) + self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0]) + + self.response.json.return_value = self.regr.body.update( + contact=()).to_json() + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 827a4dd11..5be458580 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -274,6 +274,7 @@ class Registration(ResourceBody): agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) + only_return_existing = jose.Field('onlyReturnExisting', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/certbot/account.py b/certbot/account.py index 5e9455048..2f261f759 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -270,9 +270,12 @@ class AccountFileStorage(interfaces.AccountStorage): if hasattr(acme.directory, "new-authz"): regr = RegistrationResourceWithNewAuthzrURI( new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) + body={}, + uri=regr.uri) + else: + regr = messages.RegistrationResource( + body={}, + uri=regr.uri) regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), diff --git a/certbot/main.py b/certbot/main.py index 556722104..2c7ded3e2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -735,8 +735,13 @@ def register(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. acc_contacts = ['mailto:' + email for email in config.email.split(',')] + prev_regr_uri = acc.regr.uri acc.regr = cb_client.acme.update_registration(acc.regr.update( body=acc.regr.body.update(contact=acc_contacts))) + # A v1 account being used as a v2 account will result in changing the uri to + # the v2 uri. Since it's the same object on disk, put it back to the v1 uri + # so that we can also continue to use the account object with acmev1. + acc.regr = acc.regr.update(uri=prev_regr_uri) account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) From 3855cfc08dacccce223eb6c0f3d127143225da2d Mon Sep 17 00:00:00 2001 From: Trinopoty Biswas Date: Wed, 11 Jul 2018 02:21:03 +0530 Subject: [PATCH 284/639] Linode DNS Authenticator (#5302) * Added DNS based authenticator plugin for Linode * Added linode plugin to docs * Added Dockerfile * Added .gitignore and readthedocs.org.requirements.txt * Updated default_propagation_seconds * Updated according to changes requested * Bump version to 0.26.0 * Advertise our packages work on Python 3.7. --- certbot-dns-linode/Dockerfile | 5 + certbot-dns-linode/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-linode/MANIFEST.in | 3 + certbot-dns-linode/README.rst | 1 + .../certbot_dns_linode/__init__.py | 86 ++++++++ .../certbot_dns_linode/dns_linode.py | 72 +++++++ .../certbot_dns_linode/dns_linode_test.py | 47 +++++ certbot-dns-linode/docs/.gitignore | 1 + certbot-dns-linode/docs/Makefile | 20 ++ certbot-dns-linode/docs/api.rst | 8 + certbot-dns-linode/docs/api/dns_linode.rst | 5 + certbot-dns-linode/docs/conf.py | 180 +++++++++++++++++ certbot-dns-linode/docs/index.rst | 28 +++ certbot-dns-linode/docs/make.bat | 36 ++++ .../local-oldest-requirements.txt | 2 + .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-linode/setup.cfg | 2 + certbot-dns-linode/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 4 +- docs/packaging.rst | 2 + docs/using.rst | 1 + tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 29 files changed, 784 insertions(+), 3 deletions(-) create mode 100644 certbot-dns-linode/Dockerfile create mode 100644 certbot-dns-linode/LICENSE.txt create mode 100644 certbot-dns-linode/MANIFEST.in create mode 100644 certbot-dns-linode/README.rst create mode 100644 certbot-dns-linode/certbot_dns_linode/__init__.py create mode 100644 certbot-dns-linode/certbot_dns_linode/dns_linode.py create mode 100644 certbot-dns-linode/certbot_dns_linode/dns_linode_test.py create mode 100644 certbot-dns-linode/docs/.gitignore create mode 100644 certbot-dns-linode/docs/Makefile create mode 100644 certbot-dns-linode/docs/api.rst create mode 100644 certbot-dns-linode/docs/api/dns_linode.rst create mode 100644 certbot-dns-linode/docs/conf.py create mode 100644 certbot-dns-linode/docs/index.rst create mode 100644 certbot-dns-linode/docs/make.bat create mode 100644 certbot-dns-linode/local-oldest-requirements.txt create mode 100644 certbot-dns-linode/readthedocs.org.requirements.txt create mode 100644 certbot-dns-linode/setup.cfg create mode 100644 certbot-dns-linode/setup.py diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile new file mode 100644 index 000000000..2e237b521 --- /dev/null +++ b/certbot-dns-linode/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-linode + +RUN pip install --no-cache-dir --editable src/certbot-dns-linode diff --git a/certbot-dns-linode/LICENSE.txt b/certbot-dns-linode/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-linode/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-linode/MANIFEST.in b/certbot-dns-linode/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-linode/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-linode/README.rst b/certbot-dns-linode/README.rst new file mode 100644 index 000000000..69e1fa056 --- /dev/null +++ b/certbot-dns-linode/README.rst @@ -0,0 +1 @@ +Linode DNS Authenticator plugin for Certbot diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py new file mode 100644 index 000000000..aaed61450 --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -0,0 +1,86 @@ +""" +The `~certbot_dns_linode.dns_linode` plugin automates the process of +completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and +subsequently removing, TXT records using the Linode API. + + +Named Arguments +--------------- + +========================================== =================================== +``--dns-linode-credentials`` Linode credentials_ INI file. + (Required) +``--dns-linode-propagation-seconds`` The number of seconds to wait for + DNS to propagate before asking the + ACME server to verify the DNS + record. + (Default: 960) +========================================== =================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing Linode API +credentials, obtained from your Linode account's `Applications & API +Tokens page `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Linode API credentials used by Certbot + dns_linode_key = 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ64 + +The path to this file can be provided interactively or using the +``--dns-linode-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Linode account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire + new certificates or revoke existing certificates for associated domains, + even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-linode \\ + --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ + --dns-linode-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py new file mode 100644 index 000000000..323c0810a --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -0,0 +1,72 @@ +"""DNS Authenticator for Linode.""" +import logging + +import zope.interface +from lexicon.providers import linode + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +API_KEY_URL = 'https://manager.linode.com/profile/api' + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Linode + + This Authenticator uses the Linode API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using Linode for DNS).' + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960) + add('credentials', help='Linode credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Linode API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Linode credentials INI file', + { + 'key': 'API key for Linode account, obtained from {0}'.format(API_KEY_URL) + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_linode_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_linode_client().del_txt_record(domain, validation_name, validation) + + def _get_linode_client(self): + return _LinodeLexiconClient(self.credentials.conf('key')) + +class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Linode API. + """ + + def __init__(self, api_key): + super(_LinodeLexiconClient, self).__init__() + self.provider = linode.Provider({ + 'auth_token': api_key + }) + + def _handle_general_error(self, e, domain_name): + if not str(e).startswith('Domain not found'): + return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' + .format(domain_name, e)) + diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py b/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py new file mode 100644 index 000000000..2a0ee49f7 --- /dev/null +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py @@ -0,0 +1,47 @@ +"""Tests for certbot_dns_linode.dns_linode.""" + +import os +import unittest + +import mock + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +TOKEN = 'a-token' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_linode.dns_linode import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"linode_key": TOKEN}, path) + + self.config = mock.MagicMock(linode_credentials=path, + linode_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "linode") + + self.mock_client = mock.MagicMock() + # _get_linode_client | pylint: disable=protected-access + self.auth._get_linode_client = mock.MagicMock(return_value=self.mock_client) + +class LinodeLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + + DOMAIN_NOT_FOUND = Exception('Domain not found') + + def setUp(self): + from certbot_dns_linode.dns_linode import _LinodeLexiconClient + + self.client = _LinodeLexiconClient(TOKEN) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-linode/docs/.gitignore b/certbot-dns-linode/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-linode/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-linode/docs/Makefile b/certbot-dns-linode/docs/Makefile new file mode 100644 index 000000000..bcfbfd5b1 --- /dev/null +++ b/certbot-dns-linode/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-linode +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/certbot-dns-linode/docs/api.rst b/certbot-dns-linode/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-linode/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-linode/docs/api/dns_linode.rst b/certbot-dns-linode/docs/api/dns_linode.rst new file mode 100644 index 000000000..6380b3eba --- /dev/null +++ b/certbot-dns-linode/docs/api/dns_linode.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_linode.dns_linode` +------------------------------------------------ + +.. automodule:: certbot_dns_linode.dns_linode + :members: diff --git a/certbot-dns-linode/docs/conf.py b/certbot-dns-linode/docs/conf.py new file mode 100644 index 000000000..1fb721400 --- /dev/null +++ b/certbot-dns-linode/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-linode documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 10:52:06 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-linode' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-linodedoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-linode.tex', u'certbot-dns-linode Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-linode', u'certbot-dns-linode Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-linode', u'certbot-dns-linode Documentation', + author, 'certbot-dns-linode', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-linode/docs/index.rst b/certbot-dns-linode/docs/index.rst new file mode 100644 index 000000000..dd430554b --- /dev/null +++ b/certbot-dns-linode/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-linode documentation master file, created by + sphinx-quickstart on Wed May 10 10:52:06 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-linode's documentation! +==================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_linode + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-linode/docs/make.bat b/certbot-dns-linode/docs/make.bat new file mode 100644 index 000000000..1f2a6867f --- /dev/null +++ b/certbot-dns-linode/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-linode + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-linode/readthedocs.org.requirements.txt b/certbot-dns-linode/readthedocs.org.requirements.txt new file mode 100644 index 000000000..47449454f --- /dev/null +++ b/certbot-dns-linode/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-linode[docs] diff --git a/certbot-dns-linode/setup.cfg b/certbot-dns-linode/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-linode/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py new file mode 100644 index 000000000..2abd19b68 --- /dev/null +++ b/certbot-dns-linode/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + +version = '0.26.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.2.1', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-linode', + version=version, + description="Linode DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-linode = certbot_dns_linode.dns_linode:Authenticator', + ], + }, + test_suite='certbot_dns_linode', +) diff --git a/certbot/cli.py b/certbot/cli.py index 5c4313ea4..244daf546 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1419,6 +1419,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_google"), help=("Obtain certificates using a DNS TXT record (if you are " "using Google Cloud DNS).")) + helpful.add(["plugins", "certonly"], "--dns-linode", action="store_true", + default=flag_default("dns_linode"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using Linode for DNS).")) helpful.add(["plugins", "certonly"], "--dns-luadns", action="store_true", default=flag_default("dns_luadns"), help=("Obtain certificates using a DNS TXT record (if you are " diff --git a/certbot/constants.py b/certbot/constants.py index d31faa71c..750db83b7 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -105,6 +105,7 @@ CLI_DEFAULTS = dict( dns_dnsimple=False, dns_dnsmadeeasy=False, dns_google=False, + dns_linode=False, dns_luadns=False, dns_nsone=False, dns_rfc2136=False, diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index bf9f60e3b..8672ba0ab 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -31,6 +31,7 @@ class PluginEntryPoint(object): "certbot-dns-dnsimple", "certbot-dns-dnsmadeeasy", "certbot-dns-google", + "certbot-dns-linode", "certbot-dns-luadns", "certbot-dns-nsone", "certbot-dns-rfc2136", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 95f123a46..ef98e9cd8 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,7 +165,7 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] + "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -290,6 +290,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-dnsmadeeasy") if config.dns_google: req_auth = set_configurator(req_auth, "dns-google") + if config.dns_linode: + req_auth = set_configurator(req_auth, "dns-linode") if config.dns_luadns: req_auth = set_configurator(req_auth, "dns-luadns") if config.dns_nsone: diff --git a/docs/packaging.rst b/docs/packaging.rst index 3d58ea92e..dc75e34b0 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -17,6 +17,7 @@ We release packages and upload them to PyPI (wheels and source tarballs). - https://pypi.python.org/pypi/certbot-dns-dnsimple - https://pypi.python.org/pypi/certbot-dns-dnsmadeeasy - https://pypi.python.org/pypi/certbot-dns-google +- https://pypi.python.org/pypi/certbot-dns-linode - https://pypi.python.org/pypi/certbot-dns-luadns - https://pypi.python.org/pypi/certbot-dns-nsone - https://pypi.python.org/pypi/certbot-dns-rfc2136 @@ -63,6 +64,7 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-dnsimple - https://www.archlinux.org/packages/community/any/certbot-dns-dnsmadeeasy - https://www.archlinux.org/packages/community/any/certbot-dns-google +- https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 diff --git a/docs/using.rst b/docs/using.rst index 46599a06e..50c27d45e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -203,6 +203,7 @@ Once installed, you can find documentation on how to use each plugin at: * `certbot-dns-dnsimple `_ * `certbot-dns-dnsmadeeasy `_ * `certbot-dns-google `_ +* `certbot-dns-linode `_ * `certbot-dns-luadns `_ * `certbot-dns-nsone `_ * `certbot-dns-rfc2136 `_ diff --git a/tools/release.sh b/tools/release.sh index 02c7ccca5..1286a7b45 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index a623ec529..177bb2714 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -20,6 +20,7 @@ fi -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ -e certbot-dns-google \ + -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-rfc2136 \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 602118004..f18f05374 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -19,6 +19,7 @@ fi -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ -e certbot-dns-google \ + -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-route53 \ diff --git a/tox.cover.sh b/tox.cover.sh index 4167699d6..421771d08 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -33,6 +33,8 @@ cover () { min=99 elif [ "$1" = "certbot_dns_google" ]; then min=99 + elif [ "$1" = "certbot_dns_linode" ]; then + min=98 elif [ "$1" = "certbot_dns_luadns" ]; then min=98 elif [ "$1" = "certbot_dns_nsone" ]; then diff --git a/tox.ini b/tox.ini index ef71b52be..c05627fc1 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ dns_packages = certbot-dns-dnsimple \ certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-linode \ certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ @@ -46,6 +47,7 @@ source_paths = certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-google/certbot_dns_google + certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-rfc2136/certbot_dns_rfc2136 From 0672e63176f51b30b5faaa089e425bbe4aa28dd9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 10 Jul 2018 13:52:58 -0700 Subject: [PATCH 285/639] Remove main components from Alpha. (#6187) acme, certbot, and the Nginx and Apache plugins should no longer be considered alpha-quality. --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot-nginx/setup.py | 2 +- docs/cli-help.txt | 2 +- setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 8332febc1..e6cfcafc6 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -58,7 +58,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aca7f3bce..cbd434f01 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -31,7 +31,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index b80d95613..d31a54c17 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -60,7 +60,7 @@ class NginxConfigurator(common.Installer): """ - description = "Nginx Web Server plugin - Alpha" + description = "Nginx Web Server plugin" DEFAULT_LISTEN_PORT = '80' diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 09192a956..b43684feb 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -31,7 +31,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 594bcfd04..7b8b522c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -635,7 +635,7 @@ manual: Automatically allows public IP logging (default: Ask) nginx: - Nginx Web Server plugin - Alpha + Nginx Web Server plugin --nginx-server-root NGINX_SERVER_ROOT Nginx server root directory. (default: /etc/nginx or diff --git a/setup.py b/setup.py index 5a68ff40e..fc87917fb 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setup( license='Apache License 2.0', python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Environment :: Console :: Curses', 'Intended Audience :: System Administrators', From 8a5abb620338cd804bd7b09f52865c0e2e7658d3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 14:06:45 -0700 Subject: [PATCH 286/639] Always save server value in renewal conf file (#6189) --- certbot/storage.py | 7 ++++++- certbot/tests/storage_test.py | 22 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 5b2293bd1..32d6771c2 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -239,10 +239,15 @@ def relevant_values(all_values): :rtype dict: """ - return dict( + rv = dict( (option, value) for option, value in six.iteritems(all_values) if _relevant(option) and cli.option_was_set(option, value)) + # We always save the server value to help with forward compatibility + # and behavioral consistency when versions of Certbot with different + # server defaults are used. + rv["server"] = all_values["server"] + return rv def lineagename_for_filename(config_filename): """Returns the lineagename for a configuration filename. diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index aa6c52ad4..db6aec1de 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -544,14 +544,22 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(os.path.exists(temp_config_file)) def _test_relevant_values_common(self, values): - option = "rsa_key_size" + defaults = dict((option, cli.flag_default(option)) + for option in ("rsa_key_size", "server",)) mock_parser = mock.Mock(args=["--standalone"], verb="certonly", - defaults={option: cli.flag_default(option)}) + defaults=defaults) + + # make a copy to ensure values isn't modified + values = values.copy() + values.setdefault("server", defaults["server"]) + expected_server = values["server"] from certbot.storage import relevant_values with mock.patch("certbot.cli.helpful_parser", mock_parser): - # make a copy to ensure values isn't modified - return relevant_values(values.copy()) + rv = relevant_values(values) + self.assertIn("server", rv) + self.assertEqual(rv.pop("server"), expected_server) + return rv def test_relevant_values(self): """Test that relevant_values() can reject an irrelevant value.""" @@ -589,6 +597,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual( self._test_relevant_values_common(values), values) + def test_relevant_values_server(self): + self.assertEqual( + # _test_relevant_values_common handles testing the server + # value and removes it + self._test_relevant_values_common({"server": "example.org"}), {}) + @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" From 9314911135ad2ef36dd561eafe4a5e23c4c422ad Mon Sep 17 00:00:00 2001 From: chibiegg Date: Wed, 11 Jul 2018 06:30:37 +0900 Subject: [PATCH 287/639] Sakura Cloud DNS Authenticator (#5701) Implement an Authenticator which can fulfill a dns-01 challenge using the Sakura Cloud DNS API. Applicable only for domains using Sakura Cloud for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-sakuracloud -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to Sakura Cloud account. --- certbot-dns-sakuracloud/Dockerfile | 5 + certbot-dns-sakuracloud/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-sakuracloud/MANIFEST.in | 3 + certbot-dns-sakuracloud/README.rst | 1 + .../certbot_dns_sakuracloud/__init__.py | 86 ++++++++ .../dns_sakuracloud.py | 87 ++++++++ .../dns_sakuracloud_test.py | 55 +++++ certbot-dns-sakuracloud/docs/.gitignore | 1 + certbot-dns-sakuracloud/docs/Makefile | 20 ++ certbot-dns-sakuracloud/docs/api.rst | 8 + .../docs/api/dns_sakuracloud.rst | 5 + certbot-dns-sakuracloud/docs/conf.py | 180 +++++++++++++++++ certbot-dns-sakuracloud/docs/index.rst | 28 +++ certbot-dns-sakuracloud/docs/make.bat | 36 ++++ .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-sakuracloud/setup.cfg | 2 + certbot-dns-sakuracloud/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 3 +- certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 5 +- tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 4 +- 26 files changed, 805 insertions(+), 5 deletions(-) create mode 100644 certbot-dns-sakuracloud/Dockerfile create mode 100644 certbot-dns-sakuracloud/LICENSE.txt create mode 100644 certbot-dns-sakuracloud/MANIFEST.in create mode 100644 certbot-dns-sakuracloud/README.rst create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py create mode 100644 certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py create mode 100644 certbot-dns-sakuracloud/docs/.gitignore create mode 100644 certbot-dns-sakuracloud/docs/Makefile create mode 100644 certbot-dns-sakuracloud/docs/api.rst create mode 100644 certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst create mode 100644 certbot-dns-sakuracloud/docs/conf.py create mode 100644 certbot-dns-sakuracloud/docs/index.rst create mode 100644 certbot-dns-sakuracloud/docs/make.bat create mode 100644 certbot-dns-sakuracloud/readthedocs.org.requirements.txt create mode 100644 certbot-dns-sakuracloud/setup.cfg create mode 100644 certbot-dns-sakuracloud/setup.py diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile new file mode 100644 index 000000000..694773f61 --- /dev/null +++ b/certbot-dns-sakuracloud/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-sakuracloud + +RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud diff --git a/certbot-dns-sakuracloud/LICENSE.txt b/certbot-dns-sakuracloud/LICENSE.txt new file mode 100644 index 000000000..8316b6a0e --- /dev/null +++ b/certbot-dns-sakuracloud/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2018 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-sakuracloud/MANIFEST.in b/certbot-dns-sakuracloud/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-sakuracloud/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-sakuracloud/README.rst b/certbot-dns-sakuracloud/README.rst new file mode 100644 index 000000000..46a082b9c --- /dev/null +++ b/certbot-dns-sakuracloud/README.rst @@ -0,0 +1 @@ +Sakura Cloud DNS Authenticator plugin for Certbot diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py new file mode 100644 index 000000000..f18780c18 --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py @@ -0,0 +1,86 @@ +""" +The `~certbot_dns_sakuracloud.dns_sakuracloud` plugin automates the process of completing +a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently +removing, TXT records using the Sakura Cloud DNS API. + + +Named Arguments +--------------- + +========================================== ====================================== +``--dns-sakuracloud-credentials`` Sakura Cloud credentials_ INI file. + (Required) +``--dns-sakuracloud-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 90) +========================================== ====================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing +Sakura Cloud DNS API credentials, obtained from your Sakura Cloud DNS +`apikey page `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Sakura Cloud API credentials used by Certbot + dns_sakuracloud_api_token = 00000000-0000-0000-0000-000000000000 + dns_sakuracloud_api_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-sakuracloud-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Sakura Cloud account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire new + certificates or revoke existing certificates for associated domains, even if + those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-sakuracloud \\ + --dns-sakuracloud-credentials ~/.secrets/certbot/sakuracloud.ini \\ + --dns-sakuracloud-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py new file mode 100644 index 000000000..6f1c74b68 --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -0,0 +1,87 @@ +"""DNS Authenticator for Sakura Cloud DNS.""" +import logging + +import zope.interface +from lexicon.providers import sakuracloud + +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +APIKEY_URL = "https://secure.sakura.ad.jp/cloud/#!/apikey/top/" + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Sakura Cloud DNS + + This Authenticator uses the Sakura Cloud API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record ' + \ + '(if you are using Sakura Cloud for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments( + add, default_propagation_seconds=90) + add('credentials', help='Sakura Cloud credentials file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Sakura Cloud API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Sakura Cloud credentials file', + { + 'api-token': \ + 'API token for Sakura Cloud API obtained from {0}'.format(APIKEY_URL), + 'api-secret': \ + 'API secret for Sakura Cloud API obtained from {0}'.format(APIKEY_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_sakuracloud_client().add_txt_record( + domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_sakuracloud_client().del_txt_record( + domain, validation_name, validation) + + def _get_sakuracloud_client(self): + return _SakuraCloudLexiconClient( + self.credentials.conf('api-token'), + self.credentials.conf('api-secret'), + self.ttl + ) + + +class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Sakura Cloud via Lexicon. + """ + + def __init__(self, api_token, api_secret, ttl): + super(_SakuraCloudLexiconClient, self).__init__() + + self.provider = sakuracloud.Provider({ + 'auth_token': api_token, + 'auth_secret': api_secret, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): + return # Expected errors when zone name guess is wrong + return super(_SakuraCloudLexiconClient, self)._handle_http_error(e, domain_name) diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py new file mode 100644 index 000000000..84605d06f --- /dev/null +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py @@ -0,0 +1,55 @@ +"""Tests for certbot_dns_sakuracloud.dns_sakuracloud.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_TOKEN = '00000000-0000-0000-0000-000000000000' +API_SECRET = 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_sakuracloud.dns_sakuracloud import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write( + {"sakuracloud_api_token": API_TOKEN, "sakuracloud_api_secret": API_SECRET}, + path + ) + + self.config = mock.MagicMock(sakuracloud_credentials=path, + sakuracloud_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "sakuracloud") + + self.mock_client = mock.MagicMock() + # _get_sakuracloud_client | pylint: disable=protected-access + self.auth._get_sakuracloud_client = mock.MagicMock(return_value=self.mock_client) + + +class NS1LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) + LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) + + def setUp(self): + from certbot_dns_sakuracloud.dns_sakuracloud import _SakuraCloudLexiconClient + + self.client = _SakuraCloudLexiconClient(API_TOKEN, API_SECRET, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-sakuracloud/docs/.gitignore b/certbot-dns-sakuracloud/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-sakuracloud/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-sakuracloud/docs/Makefile b/certbot-dns-sakuracloud/docs/Makefile new file mode 100644 index 000000000..c2969dd98 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-sakuracloud +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-sakuracloud/docs/api.rst b/certbot-dns-sakuracloud/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst b/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst new file mode 100644 index 000000000..74692e15b --- /dev/null +++ b/certbot-dns-sakuracloud/docs/api/dns_sakuracloud.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_sakuracloud.dns_sakuracloud` +---------------------------------------------- + +.. automodule:: certbot_dns_sakuracloud.dns_sakuracloud + :members: diff --git a/certbot-dns-sakuracloud/docs/conf.py b/certbot-dns-sakuracloud/docs/conf.py new file mode 100644 index 000000000..e14fe1d4c --- /dev/null +++ b/certbot-dns-sakuracloud/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-sakuracloud documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 18:30:40 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-sakuracloud' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-sakuraclouddoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-sakuracloud.tex', u'certbot-dns-sakuracloud Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-sakuracloud', u'certbot-dns-sakuracloud Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-sakuracloud', u'certbot-dns-sakuracloud Documentation', + author, 'certbot-dns-sakuracloud', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-sakuracloud/docs/index.rst b/certbot-dns-sakuracloud/docs/index.rst new file mode 100644 index 000000000..715028591 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-sakuracloud documentation master file, created by + sphinx-quickstart on Wed May 10 18:30:40 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-sakuracloud's documentation! +=================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_sakuracloud + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-sakuracloud/docs/make.bat b/certbot-dns-sakuracloud/docs/make.bat new file mode 100644 index 000000000..0d7706bc7 --- /dev/null +++ b/certbot-dns-sakuracloud/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-sakuracloud + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-sakuracloud/readthedocs.org.requirements.txt b/certbot-dns-sakuracloud/readthedocs.org.requirements.txt new file mode 100644 index 000000000..3f46d95ef --- /dev/null +++ b/certbot-dns-sakuracloud/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-sakuracloud[docs] diff --git a/certbot-dns-sakuracloud/setup.cfg b/certbot-dns-sakuracloud/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-sakuracloud/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py new file mode 100644 index 000000000..dd8903fdd --- /dev/null +++ b/certbot-dns-sakuracloud/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.1.23', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-sakuracloud', + version=version, + description="Sakura Cloud DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-sakuracloud = certbot_dns_sakuracloud.dns_sakuracloud:Authenticator', + ], + }, + test_suite='certbot_dns_sakuracloud', +) diff --git a/certbot/cli.py b/certbot/cli.py index 244daf546..c5199dc25 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1438,6 +1438,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_route53"), help=("Obtain certificates using a DNS TXT record (if you are using Route53 for " "DNS).")) + helpful.add(["plugins", "certonly"], "--dns-sakuracloud", action="store_true", + default=flag_default("dns_sakuracloud"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Sakura Cloud for DNS).")) # things should not be reorder past/pre this comment: # plugins_group should be displayed in --help before plugin diff --git a/certbot/constants.py b/certbot/constants.py index 750db83b7..2a752beba 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -109,7 +109,8 @@ CLI_DEFAULTS = dict( dns_luadns=False, dns_nsone=False, dns_rfc2136=False, - dns_route53=False + dns_route53=False, + dns_sakuracloud=False ) STAGING_URI = "https://acme-staging-v02.api.letsencrypt.org/directory" diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 8672ba0ab..d0a20c3ac 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -36,6 +36,7 @@ class PluginEntryPoint(object): "certbot-dns-nsone", "certbot-dns-rfc2136", "certbot-dns-route53", + "certbot-dns-sakuracloud", "certbot-nginx", "certbot-postfix", ] diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index ef98e9cd8..ec0dfbb19 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,7 +165,8 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53"] + "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53", + "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -300,6 +301,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-rfc2136") if config.dns_route53: req_auth = set_configurator(req_auth, "dns-route53") + if config.dns_sakuracloud: + req_auth = set_configurator(req_auth, "dns-sakuracloud") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) return req_auth, req_inst diff --git a/tools/release.sh b/tools/release.sh index 1286a7b45..fdd810497 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 177bb2714..1610d37d3 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -25,6 +25,7 @@ fi -e certbot-dns-nsone \ -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ + -e certbot-dns-sakuracloud \ -e certbot-nginx \ -e certbot-postfix \ -e letshelp-certbot \ diff --git a/tools/venv3.sh b/tools/venv3.sh index f18f05374..1cf5554b1 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,6 +23,7 @@ fi -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-route53 \ + -e certbot-dns-sakuracloud \ -e certbot-nginx \ -e certbot-postfix \ -e letshelp-certbot \ diff --git a/tox.cover.sh b/tox.cover.sh index 421771d08..8c9766d75 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -43,6 +43,8 @@ cover () { min=99 elif [ "$1" = "certbot_dns_route53" ]; then min=92 + elif [ "$1" = "certbot_dns_sakuracloud" ]; then + min=97 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "certbot_postfix" ]; then diff --git a/tox.ini b/tox.ini index c05627fc1..9373b3aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,8 @@ dns_packages = certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ - certbot-dns-route53 + certbot-dns-route53 \ + certbot-dns-sakuracloud all_packages = acme[dev] \ .[dev] \ @@ -52,6 +53,7 @@ source_paths = certbot-dns-nsone/certbot_dns_nsone certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 + certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx certbot-postfix/certbot_postfix letshelp-certbot/letshelp_certbot From 3b39266813e560448c5ea42c9799ee02de0072b0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 14:42:27 -0700 Subject: [PATCH 288/639] Don't use quoted None for plugins in the config (#6195) This stops us from printing messages like: "Could not choose appropriate plugin for updaters: Could not select or initialize the requested installer None." when certbot renew --force-renewal is run with a lineage that doesn't have an installer. * unquote None * Test None values aren't saved in config file. --- certbot/main.py | 2 +- certbot/plugins/selection.py | 4 ++-- certbot/tests/storage_test.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 2c7ded3e2..2cba8745b 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config """ # For user-agent construction - config.installer = config.authenticator = "None" + config.installer = config.authenticator = None if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index ec0dfbb19..2fd84986e 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -170,8 +170,8 @@ noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dn def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." - config.authenticator = plugins.find_init(auth).name if auth else "None" - config.installer = plugins.find_init(inst).name if inst else "None" + config.authenticator = plugins.find_init(auth).name if auth else None + config.installer = plugins.find_init(inst).name if inst else None logger.info("Plugins selected: Authenticator %s, Installer %s", config.authenticator, config.installer) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index db6aec1de..53a976f8d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -545,8 +545,9 @@ class RenewableCertTests(BaseRenewableCertTest): def _test_relevant_values_common(self, values): defaults = dict((option, cli.flag_default(option)) - for option in ("rsa_key_size", "server",)) - mock_parser = mock.Mock(args=["--standalone"], verb="certonly", + for option in ("authenticator", "installer", + "rsa_key_size", "server",)) + mock_parser = mock.Mock(args=[], verb="plugins", defaults=defaults) # make a copy to ensure values isn't modified @@ -588,6 +589,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual( self._test_relevant_values_common(values), values) + def test_relevant_values_plugins_none(self): + self.assertEqual( + self._test_relevant_values_common( + {"authenticator": None, "installer": None}), {}) + @mock.patch("certbot.cli.set_by_cli") @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): From 3f6a90882113fc5954106dd29936f6699487ba0a Mon Sep 17 00:00:00 2001 From: chibiegg Date: Wed, 11 Jul 2018 09:36:20 +0900 Subject: [PATCH 289/639] Gehirn Infrastracture Service DNS Authenticator (#5702) Implement an Authenticator which can fulfill a dns-01 challenge using the Gehirn DNS (Gehirn Infrastructure Service) API. Applicable only for domains using Gehirn DNS for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-gehirn -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to Gehirn DNS account. --- certbot-dns-gehirn/Dockerfile | 5 + certbot-dns-gehirn/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-gehirn/MANIFEST.in | 3 + certbot-dns-gehirn/README.rst | 1 + .../certbot_dns_gehirn/__init__.py | 88 ++++++++ .../certbot_dns_gehirn/dns_gehirn.py | 84 ++++++++ .../certbot_dns_gehirn/dns_gehirn_test.py | 55 +++++ certbot-dns-gehirn/docs/.gitignore | 1 + certbot-dns-gehirn/docs/Makefile | 20 ++ certbot-dns-gehirn/docs/api.rst | 8 + certbot-dns-gehirn/docs/api/dns_gehirn.rst | 5 + certbot-dns-gehirn/docs/conf.py | 180 +++++++++++++++++ certbot-dns-gehirn/docs/index.rst | 28 +++ certbot-dns-gehirn/docs/make.bat | 36 ++++ .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-gehirn/setup.cfg | 2 + certbot-dns-gehirn/setup.py | 66 ++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 8 +- tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 26 files changed, 803 insertions(+), 5 deletions(-) create mode 100644 certbot-dns-gehirn/Dockerfile create mode 100644 certbot-dns-gehirn/LICENSE.txt create mode 100644 certbot-dns-gehirn/MANIFEST.in create mode 100644 certbot-dns-gehirn/README.rst create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/__init__.py create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py create mode 100644 certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py create mode 100644 certbot-dns-gehirn/docs/.gitignore create mode 100644 certbot-dns-gehirn/docs/Makefile create mode 100644 certbot-dns-gehirn/docs/api.rst create mode 100644 certbot-dns-gehirn/docs/api/dns_gehirn.rst create mode 100644 certbot-dns-gehirn/docs/conf.py create mode 100644 certbot-dns-gehirn/docs/index.rst create mode 100644 certbot-dns-gehirn/docs/make.bat create mode 100644 certbot-dns-gehirn/readthedocs.org.requirements.txt create mode 100644 certbot-dns-gehirn/setup.cfg create mode 100644 certbot-dns-gehirn/setup.py diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile new file mode 100644 index 000000000..48ad902b5 --- /dev/null +++ b/certbot-dns-gehirn/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-gehirn + +RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn diff --git a/certbot-dns-gehirn/LICENSE.txt b/certbot-dns-gehirn/LICENSE.txt new file mode 100644 index 000000000..8316b6a0e --- /dev/null +++ b/certbot-dns-gehirn/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2018 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-gehirn/MANIFEST.in b/certbot-dns-gehirn/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-gehirn/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-gehirn/README.rst b/certbot-dns-gehirn/README.rst new file mode 100644 index 000000000..16058eff8 --- /dev/null +++ b/certbot-dns-gehirn/README.rst @@ -0,0 +1 @@ +Gehirn Infrastracture Service DNS Authenticator plugin for Certbot diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py new file mode 100644 index 000000000..db54154ac --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py @@ -0,0 +1,88 @@ +""" +The `~certbot_dns_gehirn.dns_gehirn` plugin automates the process of completing +a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently +removing, TXT records using the Gehirn Infrastracture Service DNS API. + + +Named Arguments +--------------- + +======================================== ===================================== +``--dns-gehirn-credentials`` Gehirn Infrastracture Service + credentials_ INI file. + (Required) +``--dns-gehirn-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 30) +======================================== ===================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing +Gehirn Infrastracture Service DNS API credentials, +obtained from your Gehirn Infrastracture Service +`dashboard `_. + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # Gehirn Infrastracture Service API credentials used by Certbot + dns_gehirn_api_token = 00000000-0000-0000-0000-000000000000 + dns_gehirn_api_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-gehirn-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + Gehirn Infrastracture Service account. Users who can read this file can use + these credentials to issue arbitrary API calls on your behalf. Users who can + cause Certbot to run using these credentials can complete a ``dns-01`` + challenge to acquire new certificates or revoke existing certificates for + associated domains, even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-gehirn \\ + --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\ + --dns-gehirn-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py new file mode 100644 index 000000000..50bfce1ae --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -0,0 +1,84 @@ +"""DNS Authenticator for Gehirn Infrastracture Service DNS.""" +import logging + +import zope.interface +from lexicon.providers import gehirn + +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +DASHBOARD_URL = "https://gis.gehirn.jp/" + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Gehirn Infrastracture Service DNS + + This Authenticator uses the Gehirn Infrastracture Service API to fulfill + a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record ' + \ + '(if you are using Gehirn Infrastracture Service for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='Gehirn Infrastracture Service credentials file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Gehirn Infrastracture Service API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Gehirn Infrastracture Service credentials file', + { + 'api-token': 'API token for Gehirn Infrastracture Service ' + \ + 'API obtained from {0}'.format(DASHBOARD_URL), + 'api-secret': 'API secret for Gehirn Infrastracture Service ' + \ + 'API obtained from {0}'.format(DASHBOARD_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_gehirn_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_gehirn_client().del_txt_record(domain, validation_name, validation) + + def _get_gehirn_client(self): + return _GehirnLexiconClient( + self.credentials.conf('api-token'), + self.credentials.conf('api-secret'), + self.ttl + ) + + +class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the Gehirn Infrastracture Service via Lexicon. + """ + + def __init__(self, api_token, api_secret, ttl): + super(_GehirnLexiconClient, self).__init__() + + self.provider = gehirn.Provider({ + 'auth_token': api_token, + 'auth_secret': api_secret, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): + return # Expected errors when zone name guess is wrong + return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name) diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py new file mode 100644 index 000000000..b771c103e --- /dev/null +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py @@ -0,0 +1,55 @@ +"""Tests for certbot_dns_gehirn.dns_gehirn.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_TOKEN = '00000000-0000-0000-0000-000000000000' +API_SECRET = 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw' + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_gehirn.dns_gehirn import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write( + {"gehirn_api_token": API_TOKEN, "gehirn_api_secret": API_SECRET}, + path + ) + + self.config = mock.MagicMock(gehirn_credentials=path, + gehirn_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "gehirn") + + self.mock_client = mock.MagicMock() + # _get_gehirn_client | pylint: disable=protected-access + self.auth._get_gehirn_client = mock.MagicMock(return_value=self.mock_client) + + +class GehirnLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) + LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) + + def setUp(self): + from certbot_dns_gehirn.dns_gehirn import _GehirnLexiconClient + + self.client = _GehirnLexiconClient(API_TOKEN, API_SECRET, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-gehirn/docs/.gitignore b/certbot-dns-gehirn/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-gehirn/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-gehirn/docs/Makefile b/certbot-dns-gehirn/docs/Makefile new file mode 100644 index 000000000..a363d1b47 --- /dev/null +++ b/certbot-dns-gehirn/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-gehirn +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-gehirn/docs/api.rst b/certbot-dns-gehirn/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-gehirn/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-gehirn/docs/api/dns_gehirn.rst b/certbot-dns-gehirn/docs/api/dns_gehirn.rst new file mode 100644 index 000000000..35a13e9c1 --- /dev/null +++ b/certbot-dns-gehirn/docs/api/dns_gehirn.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_gehirn.dns_gehirn` +------------------------------------ + +.. automodule:: certbot_dns_gehirn.dns_gehirn + :members: diff --git a/certbot-dns-gehirn/docs/conf.py b/certbot-dns-gehirn/docs/conf.py new file mode 100644 index 000000000..a1b2799fb --- /dev/null +++ b/certbot-dns-gehirn/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-gehirn documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 18:30:40 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-gehirn' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-gehirndoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-gehirn.tex', u'certbot-dns-gehirn Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation', + author, 'certbot-dns-gehirn', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-gehirn/docs/index.rst b/certbot-dns-gehirn/docs/index.rst new file mode 100644 index 000000000..77546fa89 --- /dev/null +++ b/certbot-dns-gehirn/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-gehirn documentation master file, created by + sphinx-quickstart on Wed May 10 18:30:40 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-gehirn's documentation! +============================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_gehirn + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-gehirn/docs/make.bat b/certbot-dns-gehirn/docs/make.bat new file mode 100644 index 000000000..905d4ee90 --- /dev/null +++ b/certbot-dns-gehirn/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-gehirn + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-gehirn/readthedocs.org.requirements.txt b/certbot-dns-gehirn/readthedocs.org.requirements.txt new file mode 100644 index 000000000..d9f4f9823 --- /dev/null +++ b/certbot-dns-gehirn/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-gehirn[docs] diff --git a/certbot-dns-gehirn/setup.cfg b/certbot-dns-gehirn/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-gehirn/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py new file mode 100644 index 000000000..cc47da327 --- /dev/null +++ b/certbot-dns-gehirn/setup.py @@ -0,0 +1,66 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.1.22', + 'mock', + 'setuptools', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-gehirn', + version=version, + description="Gehirn Infrastracture Service DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-gehirn = certbot_dns_gehirn.dns_gehirn:Authenticator', + ], + }, + test_suite='certbot_dns_gehirn', +) diff --git a/certbot/cli.py b/certbot/cli.py index c5199dc25..6d262ed72 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1415,6 +1415,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_dnsmadeeasy"), help=("Obtain certificates using a DNS TXT record (if you are" "using DNS Made Easy for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-gehirn", action="store_true", + default=flag_default("dns_gehirn"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Gehirn Infrastracture Service for DNS).")) helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", default=flag_default("dns_google"), help=("Obtain certificates using a DNS TXT record (if you are " diff --git a/certbot/constants.py b/certbot/constants.py index 2a752beba..70249b89b 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -104,6 +104,7 @@ CLI_DEFAULTS = dict( dns_digitalocean=False, dns_dnsimple=False, dns_dnsmadeeasy=False, + dns_gehirn=False, dns_google=False, dns_linode=False, dns_luadns=False, diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index d0a20c3ac..6ed0cf7b7 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -30,6 +30,7 @@ class PluginEntryPoint(object): "certbot-dns-digitalocean", "certbot-dns-dnsimple", "certbot-dns-dnsmadeeasy", + "certbot-dns-gehirn", "certbot-dns-google", "certbot-dns-linode", "certbot-dns-luadns", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 2fd84986e..0073c99fe 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -164,9 +164,9 @@ def choose_plugin(prepared, question): return None noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", - "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-google", - "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", "dns-route53", - "dns-sakuracloud"] + "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-gehirn", + "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", + "dns-route53", "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -289,6 +289,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-dnsimple") if config.dns_dnsmadeeasy: req_auth = set_configurator(req_auth, "dns-dnsmadeeasy") + if config.dns_gehirn: + req_auth = set_configurator(req_auth, "dns-gehirn") if config.dns_google: req_auth = set_configurator(req_auth, "dns-google") if config.dns_linode: diff --git a/tools/release.sh b/tools/release.sh index fdd810497..0d42bc22a 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 1610d37d3..159fc16fb 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -19,6 +19,7 @@ fi -e certbot-dns-digitalocean \ -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ + -e certbot-dns-gehirn \ -e certbot-dns-google \ -e certbot-dns-linode \ -e certbot-dns-luadns \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 1cf5554b1..a1489df22 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -18,6 +18,7 @@ fi -e certbot-dns-digitalocean \ -e certbot-dns-dnsimple \ -e certbot-dns-dnsmadeeasy \ + -e certbot-dns-gehirn \ -e certbot-dns-google \ -e certbot-dns-linode \ -e certbot-dns-luadns \ diff --git a/tox.cover.sh b/tox.cover.sh index 8c9766d75..cccaf6103 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -31,6 +31,8 @@ cover () { min=98 elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then min=99 + elif [ "$1" = "certbot_dns_gehirn" ]; then + min=97 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_dns_linode" ]; then diff --git a/tox.ini b/tox.ini index 9373b3aa7..32020cf14 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ dns_packages = certbot-dns-digitalocean \ certbot-dns-dnsimple \ certbot-dns-dnsmadeeasy \ + certbot-dns-gehirn \ certbot-dns-google \ certbot-dns-linode \ certbot-dns-luadns \ @@ -47,6 +48,7 @@ source_paths = certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy + certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns From b7113a35eb4bf93dfa12b8f16f44043739ace11d Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 10 Jul 2018 18:48:09 -0700 Subject: [PATCH 290/639] Delete empty directories after deleting an account, including symlinks up and down the chain, as appropriate (#6176) --- certbot/account.py | 38 +++++++++++++++++++++++++++++++++++ certbot/tests/account_test.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/certbot/account.py b/certbot/account.py index 2f261f759..f2ed5cfd5 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -250,12 +250,50 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ + # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) shutil.rmtree(account_dir_path) + # Step 2: remove the directory if it's empty, and linked directories + if not os.listdir(self.config.accounts_dir): + self._delete_accounts_dir_for_server_path(self.config.server_path) + + def _delete_accounts_dir_for_server_path(self, server_path): + accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + + # does an appropriate directory link to me? if so, make sure that's gone + reused_servers = {} + for k in constants.LE_REUSE_SERVERS: + reused_servers[constants.LE_REUSE_SERVERS[k]] = k + + # is there a next one up? call that and be done + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) + if os.path.islink(next_accounts_dir_path) \ + and os.readlink(next_accounts_dir_path) == accounts_dir_path: + self._delete_accounts_dir_for_server_path(next_server_path) + return + + # if there's not a next one up to delete, then delete me + # and whatever I link to if applicable + if os.path.islink(accounts_dir_path): + # save my info then delete me + target = os.readlink(accounts_dir_path) + os.unlink(accounts_dir_path) + # then delete whatever I linked to, if appropriate + if server_path in constants.LE_REUSE_SERVERS: + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) + if target == prev_accounts_dir_path: + self._delete_accounts_dir_for_server_path(prev_server_path) + else: + # just delete me + os.rmdir(accounts_dir_path) + def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e7f82a5b8..e0ec3d5f8 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -273,6 +273,40 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_delete_no_account(self): self.assertRaises(errors.AccountNotFound, self.storage.delete, self.acc.id) + def _assert_symlinked_account_removed(self): + # create v1 account + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + # ensure v2 isn't already linked to it + with mock.patch('certbot.constants.LE_REUSE_SERVERS', {}): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + + def _test_delete_folders(self, server_url): + # create symlinked servers + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.storage.find_all() + + # delete starting at given server_url + self._set_server(server_url) + self.storage.delete(self.acc.id) + + # make sure we're gone from both urls + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + + def test_delete_folders_up(self): + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + self._assert_symlinked_account_removed() + + def test_delete_folders_down(self): + self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') + self._assert_symlinked_account_removed() + if __name__ == "__main__": unittest.main() # pragma: no cover From 148d68b99a86b3ef0a1257af8becf481a075ebe0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 10 Jul 2018 18:48:49 -0700 Subject: [PATCH 291/639] Fix broken fedora links (#6196) * fix broken fedora links * Add packaged plugin links --- docs/packaging.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index dc75e34b0..48ea02ae7 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -84,8 +84,21 @@ Fedora In Fedora 23+. -- https://admin.fedoraproject.org/pkgdb/package/certbot/ -- https://admin.fedoraproject.org/pkgdb/package/python-acme/ +- https://apps.fedoraproject.org/packages/python-acme +- https://apps.fedoraproject.org/packages/certbot +- https://apps.fedoraproject.org/packages/python-certbot-apache +- https://apps.fedoraproject.org/packages/python-certbot-dns-cloudflare +- https://apps.fedoraproject.org/packages/python-certbot-dns-cloudxns +- https://apps.fedoraproject.org/packages/python-certbot-dns-digitalocean +- https://apps.fedoraproject.org/packages/python-certbot-dns-dnsimple +- https://apps.fedoraproject.org/packages/python-certbot-dns-dnsmadeeasy +- https://apps.fedoraproject.org/packages/python-certbot-dns-google +- https://apps.fedoraproject.org/packages/python-certbot-dns-linode +- https://apps.fedoraproject.org/packages/python-certbot-dns-luadns +- https://apps.fedoraproject.org/packages/python-certbot-dns-nsone +- https://apps.fedoraproject.org/packages/python-certbot-dns-rfc2136 +- https://apps.fedoraproject.org/packages/python-certbot-dns-route53 +- https://apps.fedoraproject.org/packages/python-certbot-nginx FreeBSD ------- From a2222d5bdff304f8c55b63986d09d3f155ef6893 Mon Sep 17 00:00:00 2001 From: Nicolas Bachschmidt Date: Wed, 11 Jul 2018 05:52:32 +0200 Subject: [PATCH 292/639] OVH DNS Authenticator (#5423) Implement an Authenticator which can fulfill a dns-01 challenge using the OVH DNS API. Applicable only for domains using OVH DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-ovh -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-ovh -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting dnsimple interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Path to credentials file with an invalid application key. * Path to credentials file with an invalid application secret. * Path to credentials file with an invalid consumer key. * Path to credentials file with missing properties. * Domain name not registered to OVH account. --- certbot-dns-ovh/Dockerfile | 5 + certbot-dns-ovh/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-ovh/MANIFEST.in | 3 + certbot-dns-ovh/README.rst | 1 + certbot-dns-ovh/certbot_dns_ovh/__init__.py | 98 +++++++++ certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py | 102 ++++++++++ .../certbot_dns_ovh/dns_ovh_test.py | 62 ++++++ certbot-dns-ovh/docs/.gitignore | 1 + certbot-dns-ovh/docs/Makefile | 20 ++ certbot-dns-ovh/docs/api.rst | 8 + certbot-dns-ovh/docs/api/dns_ovh.rst | 5 + certbot-dns-ovh/docs/conf.py | 180 +++++++++++++++++ certbot-dns-ovh/docs/index.rst | 28 +++ certbot-dns-ovh/docs/make.bat | 36 ++++ certbot-dns-ovh/local-oldest-requirements.txt | 2 + .../readthedocs.org.requirements.txt | 12 ++ certbot-dns-ovh/setup.cfg | 2 + certbot-dns-ovh/setup.py | 69 +++++++ certbot/cli.py | 4 + certbot/constants.py | 1 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 6 +- docs/cli-help.txt | 12 ++ docs/packaging.rst | 2 + docs/using.rst | 1 + tools/release.sh | 2 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 2 + 30 files changed, 857 insertions(+), 4 deletions(-) create mode 100644 certbot-dns-ovh/Dockerfile create mode 100644 certbot-dns-ovh/LICENSE.txt create mode 100644 certbot-dns-ovh/MANIFEST.in create mode 100644 certbot-dns-ovh/README.rst create mode 100644 certbot-dns-ovh/certbot_dns_ovh/__init__.py create mode 100644 certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py create mode 100644 certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py create mode 100644 certbot-dns-ovh/docs/.gitignore create mode 100644 certbot-dns-ovh/docs/Makefile create mode 100644 certbot-dns-ovh/docs/api.rst create mode 100644 certbot-dns-ovh/docs/api/dns_ovh.rst create mode 100644 certbot-dns-ovh/docs/conf.py create mode 100644 certbot-dns-ovh/docs/index.rst create mode 100644 certbot-dns-ovh/docs/make.bat create mode 100644 certbot-dns-ovh/local-oldest-requirements.txt create mode 100644 certbot-dns-ovh/readthedocs.org.requirements.txt create mode 100644 certbot-dns-ovh/setup.cfg create mode 100644 certbot-dns-ovh/setup.py diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile new file mode 100644 index 000000000..e8da96d95 --- /dev/null +++ b/certbot-dns-ovh/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-ovh + +RUN pip install --no-cache-dir --editable src/certbot-dns-ovh diff --git a/certbot-dns-ovh/LICENSE.txt b/certbot-dns-ovh/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-ovh/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-ovh/MANIFEST.in b/certbot-dns-ovh/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-ovh/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-ovh/README.rst b/certbot-dns-ovh/README.rst new file mode 100644 index 000000000..05ffe2a16 --- /dev/null +++ b/certbot-dns-ovh/README.rst @@ -0,0 +1 @@ +OVH DNS Authenticator plugin for Certbot diff --git a/certbot-dns-ovh/certbot_dns_ovh/__init__.py b/certbot-dns-ovh/certbot_dns_ovh/__init__.py new file mode 100644 index 000000000..47f8bda9f --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/__init__.py @@ -0,0 +1,98 @@ +""" +The `~certbot_dns_ovh.dns_ovh` plugin automates the process of +completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and +subsequently removing, TXT records using the OVH API. + + +Named Arguments +--------------- + +=================================== ========================================== +``--dns-ovh-credentials`` OVH credentials_ INI file. + (Required) +``--dns-ovh-propagation-seconds`` The number of seconds to wait for DNS + to propagate before asking the ACME + server to verify the DNS record. + (Default: 30) +=================================== ========================================== + + +Credentials +----------- + +Use of this plugin requires a configuration file containing OVH API +credentials for an account with the following access rules: + +* ``GET /domain/zone/*`` +* ``PUT /domain/zone/*`` +* ``POST /domain/zone/*`` +* ``DELETE /domain/zone/*`` + +These credentials can be obtained there: + +* `OVH Europe `_ (endpoint: ``ovh-eu``) +* `OVH North America `_ (endpoint: + ``ovh-ca``) + +.. code-block:: ini + :name: credentials.ini + :caption: Example credentials file: + + # OVH API credentials used by Certbot + dns_ovh_endpoint = ovh-eu + dns_ovh_application_key = MDAwMDAwMDAwMDAw + dns_ovh_application_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + dns_ovh_consumer_key = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw + +The path to this file can be provided interactively or using the +``--dns-ovh-credentials`` command-line argument. Certbot records the path +to this file for use during renewal, but does not store the file's contents. + +.. caution:: + You should protect these API credentials as you would the password to your + OVH account. Users who can read this file can use these credentials + to issue arbitrary API calls on your behalf. Users who can cause Certbot to + run using these credentials can complete a ``dns-01`` challenge to acquire + new certificates or revoke existing certificates for associated domains, + even if those domains aren't being managed by this server. + +Certbot will emit a warning if it detects that the credentials file can be +accessed by other users on your system. The warning reads "Unsafe permissions +on credentials configuration file", followed by the path to the credentials +file. This warning will be emitted each time Certbot uses the credentials file, +including for renewal, and cannot be silenced except by addressing the issue +(e.g., by using a command like ``chmod 600`` to restrict access to the file). + + +Examples +-------- + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com`` + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ohv.ini \\ + -d example.com + +.. code-block:: bash + :caption: To acquire a single certificate for both ``example.com`` and + ``www.example.com`` + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ovh.ini \\ + -d example.com \\ + -d www.example.com + +.. code-block:: bash + :caption: To acquire a certificate for ``example.com``, waiting 60 seconds + for DNS propagation + + certbot certonly \\ + --dns-ovh \\ + --dns-ovh-credentials ~/.secrets/certbot/ovh.ini \\ + --dns-ovh-propagation-seconds 60 \\ + -d example.com + +""" diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py new file mode 100644 index 000000000..c4ded7748 --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -0,0 +1,102 @@ +"""DNS Authenticator for OVH DNS.""" +import logging + +import zope.interface +from lexicon.providers import ovh + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +TOKEN_URL = 'https://eu.api.ovh.com/createToken/ or https://ca.api.ovh.com/createToken/' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for OVH + + This Authenticator uses the OVH API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certificates using a DNS TXT record (if you are using OVH for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='OVH credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the OVH API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'OVH credentials INI file', + { + 'endpoint': 'OVH API endpoint (ovh-eu or ovh-ca)', + 'application-key': 'Application key for OVH API, obtained from {0}' + .format(TOKEN_URL), + 'application-secret': 'Application secret for OVH API, obtained from {0}' + .format(TOKEN_URL), + 'consumer-key': 'Consumer key for OVH API, obtained from {0}' + .format(TOKEN_URL), + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_ovh_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_ovh_client().del_txt_record(domain, validation_name, validation) + + def _get_ovh_client(self): + return _OVHLexiconClient( + self.credentials.conf('endpoint'), + self.credentials.conf('application-key'), + self.credentials.conf('application-secret'), + self.credentials.conf('consumer-key'), + self.ttl + ) + + +class _OVHLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the OVH API via Lexicon. + """ + + def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): + super(_OVHLexiconClient, self).__init__() + + self.provider = ovh.Provider({ + 'auth_entrypoint': endpoint, + 'auth_application_key': application_key, + 'auth_application_secret': application_secret, + 'auth_consumer_key': consumer_key, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + hint = None + if str(e).startswith('400 Client Error:'): + hint = 'Is your Application Secret value correct?' + if str(e).startswith('403 Client Error:'): + hint = 'Are your Application Key and Consumer Key values correct?' + + return errors.PluginError('Error determining zone identifier for {0}: {1}.{2}' + .format(domain_name, e, ' ({0})'.format(hint) if hint else '')) + + def _handle_general_error(self, e, domain_name): + if domain_name in str(e) and str(e).endswith('not found'): + return + + super(_OVHLexiconClient, self)._handle_general_error(e, domain_name) diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py new file mode 100644 index 000000000..f2a10485d --- /dev/null +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py @@ -0,0 +1,62 @@ +"""Tests for certbot_dns_ovh.dns_ovh.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +ENDPOINT = 'ovh-eu' +APPLICATION_KEY = 'foo' +APPLICATION_SECRET = 'bar' +CONSUMER_KEY = 'spam' + + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_ovh.dns_ovh import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + credentials = { + "ovh_endpoint": ENDPOINT, + "ovh_application_key": APPLICATION_KEY, + "ovh_application_secret": APPLICATION_SECRET, + "ovh_consumer_key": CONSUMER_KEY, + } + dns_test_common.write(credentials, path) + + self.config = mock.MagicMock(ovh_credentials=path, + ovh_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "ovh") + + self.mock_client = mock.MagicMock() + # _get_ovh_client | pylint: disable=protected-access + self.auth._get_ovh_client = mock.MagicMock(return_value=self.mock_client) + + +class OVHLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + DOMAIN_NOT_FOUND = Exception('Domain example.com not found') + LOGIN_ERROR = HTTPError('403 Client Error: Forbidden for url: https://eu.api.ovh.com/1.0/...') + + def setUp(self): + from certbot_dns_ovh.dns_ovh import _OVHLexiconClient + + self.client = _OVHLexiconClient( + ENDPOINT, APPLICATION_KEY, APPLICATION_SECRET, CONSUMER_KEY, 0 + ) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-ovh/docs/.gitignore b/certbot-dns-ovh/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-ovh/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-ovh/docs/Makefile b/certbot-dns-ovh/docs/Makefile new file mode 100644 index 000000000..38f6a9159 --- /dev/null +++ b/certbot-dns-ovh/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-ovh +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-ovh/docs/api.rst b/certbot-dns-ovh/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-ovh/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-ovh/docs/api/dns_ovh.rst b/certbot-dns-ovh/docs/api/dns_ovh.rst new file mode 100644 index 000000000..79863d05f --- /dev/null +++ b/certbot-dns-ovh/docs/api/dns_ovh.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_ovh.dns_ovh` +------------------------------ + +.. automodule:: certbot_dns_ovh.dns_ovh + :members: diff --git a/certbot-dns-ovh/docs/conf.py b/certbot-dns-ovh/docs/conf.py new file mode 100644 index 000000000..57194666e --- /dev/null +++ b/certbot-dns-ovh/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-ovh documentation build configuration file, created by +# sphinx-quickstart on Fri Jan 12 10:14:31 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-ovh' +copyright = u'2018, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-ovhdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-ovh.tex', u'certbot-dns-ovh Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-ovh', u'certbot-dns-ovh Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-ovh', u'certbot-dns-ovh Documentation', + author, 'certbot-dns-ovh', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-ovh/docs/index.rst b/certbot-dns-ovh/docs/index.rst new file mode 100644 index 000000000..ad5860289 --- /dev/null +++ b/certbot-dns-ovh/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-ovh documentation master file, created by + sphinx-quickstart on Fri Jan 12 10:14:31 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-ovh's documentation! +=========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: certbot_dns_ovh + :members: + +.. toctree:: + :maxdepth: 1 + + api + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-ovh/docs/make.bat b/certbot-dns-ovh/docs/make.bat new file mode 100644 index 000000000..78f7dd669 --- /dev/null +++ b/certbot-dns-ovh/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-ovh + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-ovh/readthedocs.org.requirements.txt b/certbot-dns-ovh/readthedocs.org.requirements.txt new file mode 100644 index 000000000..0780e12a1 --- /dev/null +++ b/certbot-dns-ovh/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-ovh[docs] diff --git a/certbot-dns-ovh/setup.cfg b/certbot-dns-ovh/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-ovh/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py new file mode 100644 index 000000000..4e2e664a4 --- /dev/null +++ b/certbot-dns-ovh/setup.py @@ -0,0 +1,69 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.25.0.dev0' + +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. +install_requires = [ + 'acme>=0.21.1', + 'certbot>=0.21.1', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'mock', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-ovh', + version=version, + description="OVH DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-ovh = certbot_dns_ovh.dns_ovh:Authenticator', + ], + }, + test_suite='certbot_dns_ovh', +) diff --git a/certbot/cli.py b/certbot/cli.py index 6d262ed72..2c4aa6530 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1435,6 +1435,10 @@ def _plugins_parsing(helpful, plugins): default=flag_default("dns_nsone"), help=("Obtain certificates using a DNS TXT record (if you are " "using NS1 for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-ovh", action="store_true", + default=flag_default("dns_ovh"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using OVH for DNS).")) helpful.add(["plugins", "certonly"], "--dns-rfc2136", action="store_true", default=flag_default("dns_rfc2136"), help="Obtain certificates using a DNS TXT record (if you are using BIND for DNS).") diff --git a/certbot/constants.py b/certbot/constants.py index 70249b89b..46523ce4d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -109,6 +109,7 @@ CLI_DEFAULTS = dict( dns_linode=False, dns_luadns=False, dns_nsone=False, + dns_ovh=False, dns_rfc2136=False, dns_route53=False, dns_sakuracloud=False diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 6ed0cf7b7..7be320efc 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -35,6 +35,7 @@ class PluginEntryPoint(object): "certbot-dns-linode", "certbot-dns-luadns", "certbot-dns-nsone", + "certbot-dns-ovh", "certbot-dns-rfc2136", "certbot-dns-route53", "certbot-dns-sakuracloud", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 0073c99fe..9c2138247 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -165,8 +165,8 @@ def choose_plugin(prepared, question): noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", "dns-digitalocean", "dns-dnsimple", "dns-dnsmadeeasy", "dns-gehirn", - "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-rfc2136", - "dns-route53", "dns-sakuracloud"] + "dns-google", "dns-linode", "dns-luadns", "dns-nsone", "dns-ovh", + "dns-rfc2136", "dns-route53", "dns-sakuracloud"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -299,6 +299,8 @@ def cli_plugin_requests(config): # pylint: disable=too-many-branches req_auth = set_configurator(req_auth, "dns-luadns") if config.dns_nsone: req_auth = set_configurator(req_auth, "dns-nsone") + if config.dns_ovh: + req_auth = set_configurator(req_auth, "dns-ovh") if config.dns_rfc2136: req_auth = set_configurator(req_auth, "dns-rfc2136") if config.dns_route53: diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 7b8b522c9..8bba718d5 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -454,6 +454,8 @@ plugins: using LuaDNS for DNS). (default: False) --dns-nsone Obtain certificates using a DNS TXT record (if you are using NS1 for DNS). (default: False) + --dns-ovh Obtain certificates using a DNS TXT record (if you are + using OVH for DNS). (default: False) --dns-rfc2136 Obtain certificates using a DNS TXT record (if you are using BIND for DNS). (default: False) --dns-route53 Obtain certificates using a DNS TXT record (if you are @@ -589,6 +591,16 @@ dns-nsone: --dns-nsone-credentials DNS_NSONE_CREDENTIALS NS1 credentials file. (default: None) +dns-ovh: + Obtain certificates using a DNS TXT record (if you are using OVH for DNS). + + --dns-ovh-propagation-seconds DNS_OVH_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 30) + --dns-ovh-credentials DNS_OVH_CREDENTIALS + OVH credentials file. (default: None) + dns-rfc2136: Obtain certificates using a DNS TXT record (if you are using BIND for DNS). diff --git a/docs/packaging.rst b/docs/packaging.rst index 48ea02ae7..a86b770c5 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -20,6 +20,7 @@ We release packages and upload them to PyPI (wheels and source tarballs). - https://pypi.python.org/pypi/certbot-dns-linode - https://pypi.python.org/pypi/certbot-dns-luadns - https://pypi.python.org/pypi/certbot-dns-nsone +- https://pypi.python.org/pypi/certbot-dns-ovh - https://pypi.python.org/pypi/certbot-dns-rfc2136 - https://pypi.python.org/pypi/certbot-dns-route53 @@ -67,6 +68,7 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone +- https://www.archlinux.org/packages/community/any/certbot-dns-ovh - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 - https://www.archlinux.org/packages/community/any/certbot-dns-route53 diff --git a/docs/using.rst b/docs/using.rst index 50c27d45e..946c12bc6 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -206,6 +206,7 @@ Once installed, you can find documentation on how to use each plugin at: * `certbot-dns-linode `_ * `certbot-dns-luadns `_ * `certbot-dns-nsone `_ +* `certbot-dns-ovh `_ * `certbot-dns-rfc2136 `_ * `certbot-dns-route53 `_ diff --git a/tools/release.sh b/tools/release.sh index 0d42bc22a..880563b4b 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -46,7 +46,7 @@ PORT=${PORT:-1234} # subpackages to be released (the way developers think about them) SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" # subpackages to be released (the way the script thinks about them) SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" diff --git a/tools/venv.sh b/tools/venv.sh index 159fc16fb..5692f9ebf 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -24,6 +24,7 @@ fi -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ + -e certbot-dns-ovh \ -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ diff --git a/tools/venv3.sh b/tools/venv3.sh index a1489df22..784fc42e8 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -23,6 +23,7 @@ fi -e certbot-dns-linode \ -e certbot-dns-luadns \ -e certbot-dns-nsone \ + -e certbot-dns-ovh \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ -e certbot-nginx \ diff --git a/tox.cover.sh b/tox.cover.sh index cccaf6103..c713327c5 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else pkgs="$@" fi @@ -41,6 +41,8 @@ cover () { min=98 elif [ "$1" = "certbot_dns_nsone" ]; then min=99 + elif [ "$1" = "certbot_dns_ovh" ]; then + min=97 elif [ "$1" = "certbot_dns_rfc2136" ]; then min=99 elif [ "$1" = "certbot_dns_route53" ]; then diff --git a/tox.ini b/tox.ini index 32020cf14..482c65c36 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,7 @@ dns_packages = certbot-dns-linode \ certbot-dns-luadns \ certbot-dns-nsone \ + certbot-dns-ovh \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-dns-sakuracloud @@ -53,6 +54,7 @@ source_paths = certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone + certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud From c326c021082dede7c3b2bd411cec3aec6dff0ac5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 11 Jul 2018 11:20:36 -0700 Subject: [PATCH 293/639] Update default to ACMEv2 server (#6152) --- certbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/constants.py b/certbot/constants.py index 46523ce4d..a2de2d27a 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -88,7 +88,7 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", - server="https://acme-v01.api.letsencrypt.org/directory", + server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers configurator=None, From 95e271bfcd9efcb6f42d2a712d7dcfc3a21408d6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 14:18:26 -0700 Subject: [PATCH 294/639] Release 0.26.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 4 +- certbot-auto | 83 ++++++++---------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 49 ++++++++++- letsencrypt-auto | 83 ++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 ++-- letsencrypt-auto-source/letsencrypt-auto | 26 +++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++--- 26 files changed, 171 insertions(+), 150 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e6cfcafc6..88967c716 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index cbd434f01..f0a40a2f3 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.25.0', - 'certbot>=0.26.0.dev0', + 'certbot>=0.26.0', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-auto b/certbot-auto index d2cfa672d..17e00929b 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.1" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1103,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c @@ -1208,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 34e39ec5e..c5381278e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9cf26aa52..c7a3418c6 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 8077fe8f5..e5c32af9d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 5877d2d0d..039c49767 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 8c00e6d58..2f3f95a9c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 34772c422..3d1ce16c9 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index cc47da327..be7515274 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ad56e58be..e020412b5 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 2abd19b68..3043aafed 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 796f7a489..3ea232bc3 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d87470c3a..2e63de4f0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 4e2e664a4..fc76b6e0c 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bbfbcbba1..813dc4981 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8836dc6d8..97039dade 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index dd8903fdd..53c65af0f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.25.0.dev0' +version = '0.26.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b43684feb..dd30dae7f 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0.dev0' +version = '0.26.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 3ae0e315b..766be39a3 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0.dev0' +__version__ = '0.26.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 8bba718d5..c044c206a 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.25.1 + "". (default: CertbotACMEClient/0.26.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -196,6 +196,8 @@ security: --strict-permissions Require that all configuration files are owned by the current user; only needed if your config is somewhere unsafe like /tmp/ (default: False) + --auto-hsts Gradually increasing max-age value for HTTP Strict + Transport Security security header (default: False) testing: The following flags are meant for testing and integration purposes only. @@ -249,7 +251,7 @@ paths: --work-dir WORK_DIR Working directory. (default: /var/lib/letsencrypt) --logs-dir LOGS_DIR Logs directory. (default: /var/log/letsencrypt) --server SERVER ACME Directory Resource URI. (default: - https://acme-v01.api.letsencrypt.org/directory) + https://acme-v02.api.letsencrypt.org/directory) manage: Various subcommands and flags are available for managing your @@ -328,6 +330,7 @@ renew: renew", regardless of if the certificate is renewed. This setting does not apply to important TLS configuration updates. (default: False) + --no-autorenew Disable auto renewal of certificates. (default: True) certificates: List certificates managed by Certbot @@ -448,8 +451,13 @@ plugins: using DNSimple for DNS). (default: False) --dns-dnsmadeeasy Obtain certificates using a DNS TXT record (if you areusing DNS Made Easy for DNS). (default: False) + --dns-gehirn Obtain certificates using a DNS TXT record (if you are + using Gehirn Infrastracture Service for DNS). + (default: False) --dns-google Obtain certificates using a DNS TXT record (if you are using Google Cloud DNS). (default: False) + --dns-linode Obtain certificates using a DNS TXT record (if you are + using Linode for DNS). (default: False) --dns-luadns Obtain certificates using a DNS TXT record (if you are using LuaDNS for DNS). (default: False) --dns-nsone Obtain certificates using a DNS TXT record (if you are @@ -460,6 +468,8 @@ plugins: using BIND for DNS). (default: False) --dns-route53 Obtain certificates using a DNS TXT record (if you are using Route53 for DNS). (default: False) + --dns-sakuracloud Obtain certificates using a DNS TXT record (if you are + using Sakura Cloud for DNS). (default: False) apache: Apache Web Server plugin - Beta @@ -553,6 +563,18 @@ dns-dnsmadeeasy: --dns-dnsmadeeasy-credentials DNS_DNSMADEEASY_CREDENTIALS DNS Made Easy credentials INI file. (default: None) +dns-gehirn: + Obtain certificates using a DNS TXT record (if you are using Gehirn + Infrastracture Service for DNS). + + --dns-gehirn-propagation-seconds DNS_GEHIRN_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 30) + --dns-gehirn-credentials DNS_GEHIRN_CREDENTIALS + Gehirn Infrastracture Service credentials file. + (default: None) + dns-google: Obtain certificates using a DNS TXT record (if you are using Google Cloud DNS for DNS). @@ -570,6 +592,16 @@ dns-google: control#permissions_and_roles for information about therequired permissions.) (default: None) +dns-linode: + Obtain certs using a DNS TXT record (if you are using Linode for DNS). + + --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 960) + --dns-linode-credentials DNS_LINODE_CREDENTIALS + Linode credentials INI file. (default: None) + dns-luadns: Obtain certificates using a DNS TXT record (if you are using LuaDNS for DNS). @@ -599,7 +631,7 @@ dns-ovh: before asking the ACME server to verify the DNS record. (default: 30) --dns-ovh-credentials DNS_OVH_CREDENTIALS - OVH credentials file. (default: None) + OVH credentials INI file. (default: None) dns-rfc2136: Obtain certificates using a DNS TXT record (if you are using BIND for @@ -621,6 +653,17 @@ dns-route53: before asking the ACME server to verify the DNS record. (default: 10) +dns-sakuracloud: + Obtain certificates using a DNS TXT record (if you are using Sakura Cloud + for DNS). + + --dns-sakuracloud-propagation-seconds DNS_SAKURACLOUD_PROPAGATION_SECONDS + The number of seconds to wait for DNS to propagate + before asking the ACME server to verify the DNS + record. (default: 90) + --dns-sakuracloud-credentials DNS_SAKURACLOUD_CREDENTIALS + Sakura Cloud credentials file. (default: None) + manual: Authenticate through manual configuration or custom shell scripts. When using shell scripts, an authenticator script must be provided. The diff --git a/letsencrypt-auto b/letsencrypt-auto index d2cfa672d..17e00929b 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.25.1" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.0.2 \ - --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ - --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ - --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \ - --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \ - --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \ - --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \ - --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \ - --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \ - --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \ - --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \ - --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \ - --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \ - --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \ - --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \ - --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \ - --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \ - --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \ - --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \ - --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \ - --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \ - --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \ - --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \ - --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \ - --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \ - --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \ - --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \ - --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \ - --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ - --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ - --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1103,9 +1092,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc +josepy==1.1.0 \ + --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ + --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c @@ -1208,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 67bab66d4..0b7d27be3 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlsgc/cACgkQTRfJlc2X -dfLjBgf/bHZn/q+Dqn34uBXHymRSce7UxQn17izcKAt7hZBl4j4sebQ9+0jjuNur -zrW8b0XJ0PsI10GG9qHR3ajC+04pWfRritnK1g4Ycb/pDcUkWo+8uRwr7skAVcvC -oa8ToBS3iUbd3csFl1mu1BGACUHLvVs2cYdDtMuJj8wjsVZ7KnWBGKULAskwmU4Z -VVUxeUrG9f+2kT35meEJUk91FS+4tmqNIVsVlBzf0Q0ZU1iQnV56dMwTqFRzdDJ2 -DBecE0GwuYnKXo2I7kIYaqACQmk9YFh55Sh0K9PbQxyv7YEZXZtkcdqFqyhxy3Nh -EJ2kurFaM3/VmLljc/rW8QW8B3QNbw== -=pkDz +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAltGdAYACgkQTRfJlc2X +dfKUyQf+NakhD3SfMeuJyT1StexEc9iGaAvspNH+Gf6P5v5dZOZnSOdtraR2kQAi +OQE2L5FAajIhpuELpZTAgCEFU1LZpqTvWOb/1Vb06T8DuLIYierh64LkAn0zJY/M +e8PTWyU5dcM6pY0ITvhuIMDAtomV+TzKeD1qHy2hJVTJGttk/yNtT5p8/NYIuH8Z +OWXkNuo/346xvYpTDp2Xpwv79L9JhQsxfEBpKV4IGObpTf+Mfl2f4taroLYEATGU +vrNM39P0cxu/hEHpog74CHPeK99YlBR6+7tMINQ9bYHkdjq2vLYdyopE8mCN16oy +CwITDfR5POwvs+WjU+oEtgQb73kTug== +=3XNY -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9afa86849..17e00929b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0.dev0" +LE_AUTO_VERSION="0.26.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index f266c93e99a6c6b451a0dd066cf33fc069cac53c..cf87eacfc908dc352e0d17756ecd6bc9d7320e63 100644 GIT binary patch literal 256 zcmV+b0ssD!H-fBQ`@sIYn!8z-`j+lz{Q@N0u$U83m47cn~8z5v4(zv^ufnv-raM*`SHDa?$*L zwjYLFp(zsV>M%UIA)#&*&!|W<43f30`QC>x;6{5kHWfag>;fAdx=2aP?uJ%oq~Mt# zwrh6trQ~7|S0C@}LAcZA0h*o^am2HkeuXz7reK3777FD4>jya36?*RxD`;b-E+2?X Gsx%mai+NN4 literal 256 zcmV+b0ssDtYs3=1`HBT)24SsR1u@$W7ZPSU$S&OOtBrFsR+98n78ySvPt!!G z;8VuT@C4ln966cf7@p1|^2x*#OsORv*dICX|C;T?NkAcYvB#;S?b|v(@U=jc^F-3_=^g!<2^ko94v~{-OR8 zs(jE6oLCW6yS;4jMUbxlVGl~J&StH_A-KuM!mMbFXeL^Jnpkr41{l6#rWYsfUp^(i zzO<}YgQBPwyO%n@J11S;c~Voa9yf?}P0YBYWV5x#$dd)Xb67iJaEyXJESY09t?}#b GsB~g1X@A53 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index f53891ccd..f80ee2cb4 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.25.1 \ - --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \ - --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e -acme==0.25.1 \ - --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \ - --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734 -certbot-apache==0.25.1 \ - --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \ - --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a -certbot-nginx==0.25.1 \ - --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \ - --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c +certbot==0.26.0 \ + --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ + --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d +acme==0.26.0 \ + --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ + --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f +certbot-apache==0.26.0 \ + --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ + --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 +certbot-nginx==0.26.0 \ + --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ + --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 From 0a6d520d26c8466c69078734dfbff096470ae807 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 14:18:44 -0700 Subject: [PATCH 295/639] Bump version to 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88967c716..88592013c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0a40a2f3..f435bb1a9 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c5381278e..e4ccc719a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c7a3418c6..05649d4d0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index e5c32af9d..911f9e052 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 039c49767..9dd318296 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2f3f95a9c..09b11def0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 3d1ce16c9..2ca3213bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index be7515274..e9ead6546 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e020412b5..3c7402f25 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 3043aafed..327224c9c 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 3ea232bc3..1f92f7dce 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2e63de4f0..0b4241afb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fc76b6e0c..258c7f0f1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 813dc4981..bd54ec4c5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 97039dade..5f0b26f6e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 53c65af0f..b7cfc15b5 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index dd30dae7f..4706f17bd 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 766be39a3..3b0b77f6c 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0' +__version__ = '0.27.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 17e00929b..f733c71b4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.27.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 7383fc6bf0702aa7fcbac1b54ce3508e0900cd08 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 17:25:48 -0700 Subject: [PATCH 296/639] move values to pytest.ini --- pytest.ini | 2 ++ tools/install_and_test.sh | 3 ++- tox.cover.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..3ecec1912 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --numprocesses auto --pyargs diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 819f683aa..e84fbc99e 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -14,6 +14,7 @@ fi temp_cwd=$(mktemp -d) trap "rm -rf $temp_cwd" EXIT +cp pytest.ini "$temp_cwd" set -x for requirement in "$@" ; do @@ -24,6 +25,6 @@ for requirement in "$@" ; do pkg="certbot" fi cd "$temp_cwd" - pytest --numprocesses auto --quiet --pyargs $pkg + pytest --quiet $pkg cd - done diff --git a/tox.cover.sh b/tox.cover.sh index c713327c5..7da2d6b1f 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -61,7 +61,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } From 32676f02c322128263b659d7e34aeb1b12d3ac04 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 17:32:48 -0700 Subject: [PATCH 297/639] warnings are errors --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3ecec1912..085c7258e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --numprocesses auto --pyargs +addopts = -Werror --numprocesses auto --pyargs From fdb3c8df4ba181997bb124f7eaf1795ea9f6a1cd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 11 Jul 2018 17:33:04 -0700 Subject: [PATCH 298/639] s/assertEquals/assertEqual --- acme/acme/client_test.py | 2 +- acme/acme/crypto_util_test.py | 6 +++--- .../certbot_apache/tests/autohsts_test.py | 12 ++++++------ .../certbot_apache/tests/centos_test.py | 8 ++++---- .../certbot_apache/tests/configurator_test.py | 18 +++++++++--------- .../certbot_apache/tests/gentoo_test.py | 6 +++--- .../certbot_apache/tests/parser_test.py | 4 ++-- .../certbot_dns_google/dns_google_test.py | 4 ++-- .../certbot_nginx/tests/configurator_test.py | 10 +++++----- .../certbot_postfix/tests/postconf_test.py | 2 +- certbot/plugins/enhancements_test.py | 6 +++--- certbot/plugins/selection_test.py | 2 +- certbot/tests/cert_manager_test.py | 12 ++++++------ certbot/tests/client_test.py | 4 ++-- certbot/tests/display/ops_test.py | 12 ++++++------ certbot/tests/main_test.py | 2 +- certbot/tests/renewupdater_test.py | 4 ++-- 17 files changed, 57 insertions(+), 57 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index cd31c4ac3..b85aba518 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -659,7 +659,7 @@ class ClientTest(ClientTestBase): def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) self.assertTrue('reason' in obj.to_partial_json().keys()) - self.assertEquals(self.rsn, obj.to_partial_json()['reason']) + self.assertEqual(self.rsn, obj.to_partial_json()['reason']) def test_revoke_bad_status_raises_error(self): self.response.status_code = http_client.METHOD_NOT_ALLOWED diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 168489d86..44b245bbe 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -209,8 +209,8 @@ class MakeCSRTest(unittest.TestCase): # have a get_extensions() method, so we skip this test if the method # isn't available. if hasattr(csr, 'get_extensions'): - self.assertEquals(len(csr.get_extensions()), 1) - self.assertEquals(csr.get_extensions()[0].get_data(), + self.assertEqual(len(csr.get_extensions()), 1) + self.assertEqual(csr.get_extensions()[0].get_data(), OpenSSL.crypto.X509Extension( b'subjectAltName', critical=False, @@ -227,7 +227,7 @@ class MakeCSRTest(unittest.TestCase): # have a get_extensions() method, so we skip this test if the method # isn't available. if hasattr(csr, 'get_extensions'): - self.assertEquals(len(csr.get_extensions()), 2) + self.assertEqual(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID, # and the shortname field is just "UNDEF" diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 73da33f15..aa35dbe30 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -64,12 +64,12 @@ class AutoHSTSTest(util.ApacheTest): self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) # Verify initial value - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), initial_val) # Increase self.config.update_autohsts(mock.MagicMock()) # Verify increased value - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), inc_val) self.assertTrue(mock_prepare.called) @@ -80,7 +80,7 @@ class AutoHSTSTest(util.ApacheTest): initial_val = maxage.format(constants.AUTOHSTS_STEPS[0]) self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) # Verify initial value - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), initial_val) self.config.update_autohsts(mock.MagicMock()) @@ -117,11 +117,11 @@ class AutoHSTSTest(util.ApacheTest): self.config.update_autohsts(mock.MagicMock()) # Value should match pre-permanent increment step cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), cur_val) # Make permanent self.config.deploy_autohsts(mock_lineage) - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), max_val) def test_autohsts_update_noop(self): @@ -153,7 +153,7 @@ class AutoHSTSTest(util.ApacheTest): mock_id.return_value = "1234567" self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com", "ocspvhost.com"]) - self.assertEquals(mock_id.call_count, 1) + self.assertEqual(mock_id.call_count, 1) def test_autohsts_remove_orphaned(self): # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 4ee8b5dcf..bc15fc6c7 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -81,9 +81,9 @@ class MultipleVhostsTestCentOS(util.ApacheTest): mock_osi.return_value = ("centos", "7") self.config.parser.update_runtime_variables() - self.assertEquals(mock_get.call_count, 3) - self.assertEquals(len(self.config.parser.modules), 4) - self.assertEquals(len(self.config.parser.variables), 2) + self.assertEqual(mock_get.call_count, 3) + self.assertEqual(len(self.config.parser.modules), 4) + self.assertEqual(len(self.config.parser.variables), 2) self.assertTrue("TEST2" in self.config.parser.variables.keys()) self.assertTrue("mod_another.c" in self.config.parser.modules) @@ -127,7 +127,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): def test_alt_restart_works(self, mock_run_script): mock_run_script.side_effect = [None, errors.SubprocessError, None] self.config.restart() - self.assertEquals(mock_run_script.call_count, 3) + self.assertEqual(mock_run_script.call_count, 3) @mock.patch("certbot_apache.configurator.util.run_script") def test_alt_restart_errors(self, mock_run_script): diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 350262634..ed61f5cf8 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1401,11 +1401,11 @@ class MultipleVhostsTest(util.ApacheTest): vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", create_ssl=True) # Check that the dialog was called with one vh: certbot.demo - self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) - self.assertEquals(len(mock_select_vhs.call_args_list), 1) + self.assertEqual(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEqual(len(mock_select_vhs.call_args_list), 1) # And the actual returned values - self.assertEquals(len(vhs), 1) + self.assertEqual(len(vhs), 1) self.assertTrue(vhs[0].name == "certbot.demo") self.assertTrue(vhs[0].ssl) @@ -1420,7 +1420,7 @@ class MultipleVhostsTest(util.ApacheTest): vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", create_ssl=False) self.assertFalse(mock_makessl.called) - self.assertEquals(vhs[0], self.vh_truth[1]) + self.assertEqual(vhs[0], self.vh_truth[1]) @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") @@ -1433,15 +1433,15 @@ class MultipleVhostsTest(util.ApacheTest): mock_select_vhs.return_value = [self.vh_truth[7]] vhs = self.config._choose_vhosts_wildcard("whatever", create_ssl=True) - self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) - self.assertEquals(len(mock_select_vhs.call_args_list), 1) + self.assertEqual(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEqual(len(mock_select_vhs.call_args_list), 1) # Ensure that make_vhost_ssl was not called, vhost.ssl == true self.assertFalse(mock_makessl.called) # And the actual returned values - self.assertEquals(len(vhs), 1) + self.assertEqual(len(vhs), 1) self.assertTrue(vhs[0].ssl) - self.assertEquals(vhs[0], self.vh_truth[7]) + self.assertEqual(vhs[0], self.vh_truth[7]) def test_deploy_cert_wildcard(self): @@ -1454,7 +1454,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", "/tmp/path", "/tmp/path", "/tmp/path") self.assertTrue(mock_dep.called) - self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(len(mock_dep.call_args_list), 1) self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) @mock.patch("certbot_apache.display_ops.select_vhost_multiple") diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index d32551267..54b4d6532 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -121,15 +121,15 @@ class MultipleVhostsTestGentoo(util.ApacheTest): mock_osi.return_value = ("gentoo", "123") self.config.parser.update_runtime_variables() - self.assertEquals(mock_get.call_count, 1) - self.assertEquals(len(self.config.parser.modules), 4) + self.assertEqual(mock_get.call_count, 1) + self.assertEqual(len(self.config.parser.modules), 4) self.assertTrue("mod_another.c" in self.config.parser.modules) @mock.patch("certbot_apache.configurator.util.run_script") def test_alt_restart_works(self, mock_run_script): mock_run_script.side_effect = [None, errors.SubprocessError, None] self.config.restart() - self.assertEquals(mock_run_script.call_count, 3) + self.assertEqual(mock_run_script.call_count, 3) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index f95f1b346..88eb55d22 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -84,7 +84,7 @@ class BasicParserTest(util.ParserTest): self.assertEqual(self.parser.aug.get(match), str(i + 1)) def test_empty_arg(self): - self.assertEquals(None, + self.assertEqual(None, self.parser.get_arg("/files/whatever/nonexistent")) def test_add_dir_to_ifmodssl(self): @@ -303,7 +303,7 @@ class BasicParserTest(util.ParserTest): from certbot_apache.parser import get_aug_path self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456") comm = self.parser.find_comments("123456") - self.assertEquals(len(comm), 1) + self.assertEqual(len(comm), 1) self.assertTrue(self.parser.loc["name"] in comm[0]) diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index b6f6e08b6..2b081885b 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -276,9 +276,9 @@ class GoogleClientTest(unittest.TestCase): [{'managedZones': [{'id': self.zone}]}]) # Record name mocked in setUp found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") - self.assertEquals(found, ["\"example-txt-contents\""]) + self.assertEqual(found, ["\"example-txt-contents\""]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") - self.assertEquals(not_found, None) + self.assertEqual(not_found, None) @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google.dns_google.open', diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 4d23f3518..3dada2399 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -177,9 +177,9 @@ class NginxConfiguratorTest(util.NginxTest): def test_ipv6only(self): # ipv6_info: (ipv6_active, ipv6only_present) - self.assertEquals((True, False), self.config.ipv6_info("80")) + self.assertEqual((True, False), self.config.ipv6_info("80")) # Port 443 has ipv6only=on because of ipv6ssl.com vhost - self.assertEquals((True, True), self.config.ipv6_info("443")) + self.assertEqual((True, True), self.config.ipv6_info("443")) def test_ipv6only_detection(self): self.config.version = (1, 3, 1) @@ -790,7 +790,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) # And the actual returned values - self.assertEquals(len(vhs), 1) + self.assertEqual(len(vhs), 1) self.assertEqual(vhs[0], vhost) def test_choose_vhosts_wildcard_redirect(self): @@ -806,7 +806,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) # And the actual returned values - self.assertEquals(len(vhs), 1) + self.assertEqual(len(vhs), 1) self.assertEqual(vhs[0], vhost) def test_deploy_cert_wildcard(self): @@ -821,7 +821,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.deploy_cert("*.com", "/tmp/path", "/tmp/path", "/tmp/path", "/tmp/path") self.assertTrue(mock_dep.called) - self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(len(mock_dep.call_args_list), 1) self.assertEqual(vhost, mock_dep.call_args_list[0][0][0]) @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") diff --git a/certbot-postfix/certbot_postfix/tests/postconf_test.py b/certbot-postfix/certbot_postfix/tests/postconf_test.py index 91617d410..01a43773d 100644 --- a/certbot-postfix/certbot_postfix/tests/postconf_test.py +++ b/certbot-postfix/certbot_postfix/tests/postconf_test.py @@ -85,7 +85,7 @@ class PostConfTest(unittest.TestCase): self.config.set('extra_param', 'another_value') self.config.flush() arguments = mock_out.call_args_list[-1][0][0] - self.assertEquals('-e', arguments[0]) + self.assertEqual('-e', arguments[0]) self.assertTrue('default_parameter=new_value' in arguments) self.assertTrue('extra_param=another_value' in arguments) diff --git a/certbot/plugins/enhancements_test.py b/certbot/plugins/enhancements_test.py index b69dc9836..22f6f54e9 100644 --- a/certbot/plugins/enhancements_test.py +++ b/certbot/plugins/enhancements_test.py @@ -37,11 +37,11 @@ class EnhancementTest(test_util.ConfigTestCase): self.assertTrue([i for i in enabled if i["name"] == "somethingelse"]) def test_are_requested(self): - self.assertEquals( + self.assertEqual( len([i for i in enhancements.enabled_enhancements(self.config)]), 0) self.assertFalse(enhancements.are_requested(self.config)) self.config.auto_hsts = True - self.assertEquals( + self.assertEqual( len([i for i in enhancements.enabled_enhancements(self.config)]), 1) self.assertTrue(enhancements.are_requested(self.config)) @@ -57,7 +57,7 @@ class EnhancementTest(test_util.ConfigTestCase): lineage = "lineage" enhancements.enable(lineage, domains, self.mockinstaller, self.config) self.assertTrue(self.mockinstaller.enable_autohsts.called) - self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0], + self.assertEqual(self.mockinstaller.enable_autohsts.call_args[0], (lineage, domains)) diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index 44d64ab8e..5f8e42516 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -197,7 +197,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_no_installer_defined(self): self.config.configurator = None - self.assertEquals(self._call(), None) + self.assertEqual(self._call(), None) def test_no_available_installers(self): self.config.configurator = "apache" diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 6ec1d4f5c..3ef1709f5 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -589,7 +589,7 @@ class GetCertnameTest(unittest.TestCase): from certbot import cert_manager prompt = "Which certificate would you" self.mock_get_utility().menu.return_value = (display_util.OK, 0) - self.assertEquals( + self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=False), ['example.com']) self.assertTrue( @@ -603,11 +603,11 @@ class GetCertnameTest(unittest.TestCase): from certbot import cert_manager prompt = "custom prompt" self.mock_get_utility().menu.return_value = (display_util.OK, 0) - self.assertEquals( + self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=False, custom_prompt=prompt), ['example.com']) - self.assertEquals(self.mock_get_utility().menu.call_args[0][0], + self.assertEqual(self.mock_get_utility().menu.call_args[0][0], prompt) @mock.patch('certbot.storage.renewal_conf_files') @@ -631,7 +631,7 @@ class GetCertnameTest(unittest.TestCase): prompt = "Which certificate(s) would you" self.mock_get_utility().checklist.return_value = (display_util.OK, ['example.com']) - self.assertEquals( + self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=True), ['example.com']) self.assertTrue( @@ -646,11 +646,11 @@ class GetCertnameTest(unittest.TestCase): prompt = "custom prompt" self.mock_get_utility().checklist.return_value = (display_util.OK, ['example.com']) - self.assertEquals( + self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=True, custom_prompt=prompt), ['example.com']) - self.assertEquals( + self.assertEqual( self.mock_get_utility().checklist.call_args[0][0], prompt) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 70ab4c798..a4e70ce35 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -487,7 +487,7 @@ class EnhanceConfigTest(ClientTestCommon): self.config.hsts = True self._test_with_already_existing() self.assertTrue(mock_log.warning.called) - self.assertEquals(mock_log.warning.call_args[0][1], + self.assertEqual(mock_log.warning.call_args[0][1], 'Strict-Transport-Security') @mock.patch("certbot.client.logger") @@ -495,7 +495,7 @@ class EnhanceConfigTest(ClientTestCommon): self.config.redirect = True self._test_with_already_existing() self.assertTrue(mock_log.warning.called) - self.assertEquals(mock_log.warning.call_args[0][1], + self.assertEqual(mock_log.warning.call_args[0][1], 'redirect') def test_no_ask_hsts(self): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 9de8c5e9a..9ad0ce87a 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -502,9 +502,9 @@ class ChooseValuesTest(unittest.TestCase): items = ["first", "second", "third"] mock_util().checklist.return_value = (display_util.OK, [items[2]]) result = self._call(items, None) - self.assertEquals(result, [items[2]]) + self.assertEqual(result, [items[2]]) self.assertTrue(mock_util().checklist.called) - self.assertEquals(mock_util().checklist.call_args[0][0], None) + self.assertEqual(mock_util().checklist.call_args[0][0], None) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_choose_names_success_question(self, mock_util): @@ -512,9 +512,9 @@ class ChooseValuesTest(unittest.TestCase): question = "Which one?" mock_util().checklist.return_value = (display_util.OK, [items[1]]) result = self._call(items, question) - self.assertEquals(result, [items[1]]) + self.assertEqual(result, [items[1]]) self.assertTrue(mock_util().checklist.called) - self.assertEquals(mock_util().checklist.call_args[0][0], question) + self.assertEqual(mock_util().checklist.call_args[0][0], question) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_choose_names_user_cancel(self, mock_util): @@ -522,9 +522,9 @@ class ChooseValuesTest(unittest.TestCase): question = "Want to cancel?" mock_util().checklist.return_value = (display_util.CANCEL, []) result = self._call(items, question) - self.assertEquals(result, []) + self.assertEqual(result, []) self.assertTrue(mock_util().checklist.called) - self.assertEquals(mock_util().checklist.call_args[0][0], question) + self.assertEqual(mock_util().checklist.call_args[0][0], question) if __name__ == "__main__": diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index cc4e6c293..b4e0d5581 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1712,7 +1712,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") self._call(['enhance', '--auto-hsts']) self.assertTrue(self.mockinstaller.enable_autohsts.called) - self.assertEquals(self.mockinstaller.enable_autohsts.call_args[0][1], + self.assertEqual(self.mockinstaller.enable_autohsts.call_args[0][1], ["example.com", "another.tld"]) @mock.patch('certbot.cert_manager.lineage_for_certname') diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index 5a362072c..5fe188c42 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -53,7 +53,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.config.dry_run = True updater.run_generic_updaters(self.config, None, None) self.assertTrue(mock_log.called) - self.assertEquals(mock_log.call_args[0][0], + self.assertEqual(mock_log.call_args[0][0], "Skipping updaters in dry-run mode.") @mock.patch("certbot.updater.logger.debug") @@ -61,7 +61,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.config.dry_run = True updater.run_renewal_deployer(self.config, None, None) self.assertTrue(mock_log.called) - self.assertEquals(mock_log.call_args[0][0], + self.assertEqual(mock_log.call_args[0][0], "Skipping renewal deployer in dry-run mode.") @mock.patch('certbot.plugins.selection.get_unprepared_installer') From 72cf844ecffd2669488aa677d71b8af0bacd1a14 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Thu, 12 Jul 2018 14:13:13 +0200 Subject: [PATCH 299/639] travis: container env are bad at reading cpu count Signed-off-by: Yoan Blanc --- .travis.yml | 2 +- tox.cover.sh | 4 +++- tox.ini | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 937d24610..3e3f7a021 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=cover NUMPROCESSES=2 FYI="this also tests py27" - sudo: required env: TOXENV=nginx_compat services: docker diff --git a/tox.cover.sh b/tox.cover.sh index c713327c5..6440d1b48 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -8,6 +8,8 @@ # # -e makes sure we fail fast and don't submit coveralls submit +NUMPROCESSES=${NUMPROCESSES:=auto} + if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else @@ -61,7 +63,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "$NUMPROCESSES" --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 482c65c36..d113fd450 100644 --- a/tox.ini +++ b/tox.ini @@ -115,6 +115,7 @@ commands = [testenv:cover] basepython = python2.7 +passenv = NUMPROCESSES commands = {[base]install_packages} ./tox.cover.sh From 9b0d2714c1404190a4e70457da732dd08bbe861e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Jul 2018 16:21:24 -0700 Subject: [PATCH 300/639] Handle existing ACMEv1 and ACMEv2 accounts (#6214) Fixes #6207. As noted by Erica: - we no longer need to check if it exists before linking to it, because we delete properly. - the previously excisting check on if server is in `LE_REUSE_SERVERS` before unlinking is nice, but probably not necessary, especially since we don't officially support people doing weird things with symlinks in our directories, and because we rmdir which will fail if it's not empty anyway. * Create single account symlink. * refactor _delete_accounts_dir_for_server_path * add symlinked account dir deletion * add tests --- certbot/account.py | 81 +++++++++++++++++++++++------------ certbot/tests/account_test.py | 20 +++++++++ 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index f2ed5cfd5..59ceb42e0 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -1,5 +1,6 @@ """Creates ACME accounts for server.""" import datetime +import functools import hashlib import logging import os @@ -191,6 +192,11 @@ class AccountFileStorage(interfaces.AccountStorage): def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_account_dir(self, prev_server_path, server_path, account_id): + prev_account_dir = self._account_dir_path_for_server_path(account_id, prev_server_path) + new_account_dir = self._account_dir_path_for_server_path(account_id, server_path) + os.symlink(prev_account_dir, new_account_dir) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): accounts_dir = self.config.accounts_dir_for_server_path(server_path) if os.path.islink(accounts_dir): @@ -207,7 +213,12 @@ class AccountFileStorage(interfaces.AccountStorage): prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) # we didn't error so we found something, so create a symlink to that - self._symlink_to_accounts_dir(prev_server_path, server_path) + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + # If accounts_dir isn't empty, make an account specific symlink + if os.listdir(accounts_dir): + self._symlink_to_account_dir(prev_server_path, server_path, account_id) + else: + self._symlink_to_accounts_dir(prev_server_path, server_path) return prev_loaded_account else: raise errors.AccountNotFound( @@ -250,49 +261,65 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ - # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) - shutil.rmtree(account_dir_path) + # Step 1: Delete account specific links and the directory + self._delete_account_dir_for_server_path(account_id, self.config.server_path) - # Step 2: remove the directory if it's empty, and linked directories + # Step 2: Remove any accounts links and directories that are now empty if not os.listdir(self.config.accounts_dir): self._delete_accounts_dir_for_server_path(self.config.server_path) + def _delete_account_dir_for_server_path(self, account_id, server_path): + link_func = functools.partial(self._account_dir_path_for_server_path, account_id) + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + shutil.rmtree(nonsymlinked_dir) + def _delete_accounts_dir_for_server_path(self, server_path): - accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + link_func = self.config.accounts_dir_for_server_path + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + os.rmdir(nonsymlinked_dir) + + def _delete_links_and_find_target_dir(self, server_path, link_func): + """Delete symlinks and return the nonsymlinked directory path. + + :param str server_path: file path based on server + :param callable link_func: callable that returns possible links + given a server_path + + :returns: the final, non-symlinked target + :rtype: str + + """ + dir_path = link_func(server_path) # does an appropriate directory link to me? if so, make sure that's gone reused_servers = {} for k in constants.LE_REUSE_SERVERS: reused_servers[constants.LE_REUSE_SERVERS[k]] = k - # is there a next one up? call that and be done - if server_path in reused_servers: - next_server_path = reused_servers[server_path] - next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) - if os.path.islink(next_accounts_dir_path) \ - and os.readlink(next_accounts_dir_path) == accounts_dir_path: - self._delete_accounts_dir_for_server_path(next_server_path) - return + # is there a next one up? + possible_next_link = True + while possible_next_link: + possible_next_link = False + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_dir_path = link_func(next_server_path) + if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path: + possible_next_link = True + server_path = next_server_path + dir_path = next_dir_path # if there's not a next one up to delete, then delete me - # and whatever I link to if applicable - if os.path.islink(accounts_dir_path): - # save my info then delete me - target = os.readlink(accounts_dir_path) - os.unlink(accounts_dir_path) - # then delete whatever I linked to, if appropriate - if server_path in constants.LE_REUSE_SERVERS: - prev_server_path = constants.LE_REUSE_SERVERS[server_path] - prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) - if target == prev_accounts_dir_path: - self._delete_accounts_dir_for_server_path(prev_server_path) - else: - # just delete me - os.rmdir(accounts_dir_path) + # and whatever I link to + while os.path.islink(dir_path): + target = os.readlink(dir_path) + os.unlink(dir_path) + dir_path = target + + return dir_path def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e0ec3d5f8..701478336 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -249,6 +249,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + def test_upgrade_load_single_account(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() @@ -307,6 +315,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + def _set_server_and_stop_symlink(self, server_path): + self._set_server(server_path) + with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: + f.write('bar') + + def test_delete_shared_account_up(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + + def test_delete_shared_account_down(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 84287130326406bd30d058668bd5a69ec4f7bdb3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 13 Jul 2018 14:21:23 -0700 Subject: [PATCH 301/639] Remove linode and ovh links which aren't valid yet. (#6198) * Remove linode links which aren't valid yet. * remove ovh references * keep links which are now valid * keep pypi links --- docs/packaging.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index a86b770c5..c13a14af3 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -65,10 +65,8 @@ From our official releases: - https://www.archlinux.org/packages/community/any/certbot-dns-dnsimple - https://www.archlinux.org/packages/community/any/certbot-dns-dnsmadeeasy - https://www.archlinux.org/packages/community/any/certbot-dns-google -- https://www.archlinux.org/packages/community/any/certbot-dns-linode - https://www.archlinux.org/packages/community/any/certbot-dns-luadns - https://www.archlinux.org/packages/community/any/certbot-dns-nsone -- https://www.archlinux.org/packages/community/any/certbot-dns-ovh - https://www.archlinux.org/packages/community/any/certbot-dns-rfc2136 - https://www.archlinux.org/packages/community/any/certbot-dns-route53 @@ -95,7 +93,6 @@ In Fedora 23+. - https://apps.fedoraproject.org/packages/python-certbot-dns-dnsimple - https://apps.fedoraproject.org/packages/python-certbot-dns-dnsmadeeasy - https://apps.fedoraproject.org/packages/python-certbot-dns-google -- https://apps.fedoraproject.org/packages/python-certbot-dns-linode - https://apps.fedoraproject.org/packages/python-certbot-dns-luadns - https://apps.fedoraproject.org/packages/python-certbot-dns-nsone - https://apps.fedoraproject.org/packages/python-certbot-dns-rfc2136 From fa7cb38e97b9cf7a22d377c5982c311f0d189242 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 07:33:38 -0700 Subject: [PATCH 302/639] Add 0.26.0 changelog (#6205) --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88251e48a..4d0808de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,54 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.26.0 - 2018-07-11 + +### Added + +* A new security enhancement which we're calling AutoHSTS has been added to + Certbot's Apache plugin. This enhancement configures your webserver to send a + HTTP Strict Transport Security header with a low max-age value that is slowly + increased over time. The max-age value is not increased to a large value + until you've successfully managed to renew your certificate. This enhancement + can be requested with the --auto-hsts flag. +* New official DNS plugins have been created for Gehirn Infrastracture Service, + Linode, OVH, and Sakura Cloud. These plugins can be found on our Docker Hub + page at https://hub.docker.com/u/certbot and on PyPI. +* The ability to reuse ACME accounts from Let's Encrypt's ACMEv1 endpoint on + Let's Encrypt's ACMEv2 endpoint has been added. +* Certbot and its components now support Python 3.7. +* Certbot's install subcommand now allows you to interactively choose which + certificate to install from the list of certificates managed by Certbot. +* Certbot now accepts the flag `--no-autorenew` which causes any obtained + certificates to not be automatically renewed when it approaches expiration. +* Support for parsing the TLS-ALPN-01 challenge has been added back to the acme + library. + +### Changed + +* Certbot's default ACME server has been changed to Let's Encrypt's ACMEv2 + endpoint. By default, this server will now be used for both new certificate + lineages and renewals. +* The Nginx plugin is no longer marked labeled as an "Alpha" version. +* The `prepare` method of Certbot's plugins is no longer called before running + "Updater" enhancements that are run on every invocation of `certbot renew`. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with functional changes were: + +* acme +* certbot +* certbot-apache +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-ovh +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/55?closed=1 + ## 0.25.1 - 2018-06-13 ### Fixed From 997496388721c9ea1092d60bd6fce7fa641fac9b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 07:54:36 -0700 Subject: [PATCH 303/639] Handle existing ACMEv1 and ACMEv2 accounts (#6214) (#6215) Fixes #6207. As noted by Erica: - we no longer need to check if it exists before linking to it, because we delete properly. - the previously excisting check on if server is in `LE_REUSE_SERVERS` before unlinking is nice, but probably not necessary, especially since we don't officially support people doing weird things with symlinks in our directories, and because we rmdir which will fail if it's not empty anyway. * Create single account symlink. * refactor _delete_accounts_dir_for_server_path * add symlinked account dir deletion * add tests (cherry picked from commit 9b0d2714c1404190a4e70457da732dd08bbe861e) --- certbot/account.py | 81 +++++++++++++++++++++++------------ certbot/tests/account_test.py | 20 +++++++++ 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index f2ed5cfd5..59ceb42e0 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -1,5 +1,6 @@ """Creates ACME accounts for server.""" import datetime +import functools import hashlib import logging import os @@ -191,6 +192,11 @@ class AccountFileStorage(interfaces.AccountStorage): def find_all(self): return self._find_all_for_server_path(self.config.server_path) + def _symlink_to_account_dir(self, prev_server_path, server_path, account_id): + prev_account_dir = self._account_dir_path_for_server_path(account_id, prev_server_path) + new_account_dir = self._account_dir_path_for_server_path(account_id, server_path) + os.symlink(prev_account_dir, new_account_dir) + def _symlink_to_accounts_dir(self, prev_server_path, server_path): accounts_dir = self.config.accounts_dir_for_server_path(server_path) if os.path.islink(accounts_dir): @@ -207,7 +213,12 @@ class AccountFileStorage(interfaces.AccountStorage): prev_server_path = constants.LE_REUSE_SERVERS[server_path] prev_loaded_account = self._load_for_server_path(account_id, prev_server_path) # we didn't error so we found something, so create a symlink to that - self._symlink_to_accounts_dir(prev_server_path, server_path) + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + # If accounts_dir isn't empty, make an account specific symlink + if os.listdir(accounts_dir): + self._symlink_to_account_dir(prev_server_path, server_path, account_id) + else: + self._symlink_to_accounts_dir(prev_server_path, server_path) return prev_loaded_account else: raise errors.AccountNotFound( @@ -250,49 +261,65 @@ class AccountFileStorage(interfaces.AccountStorage): :param account_id: id of account which should be deleted """ - # Step 1: remove the account itself account_dir_path = self._account_dir_path(account_id) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) - shutil.rmtree(account_dir_path) + # Step 1: Delete account specific links and the directory + self._delete_account_dir_for_server_path(account_id, self.config.server_path) - # Step 2: remove the directory if it's empty, and linked directories + # Step 2: Remove any accounts links and directories that are now empty if not os.listdir(self.config.accounts_dir): self._delete_accounts_dir_for_server_path(self.config.server_path) + def _delete_account_dir_for_server_path(self, account_id, server_path): + link_func = functools.partial(self._account_dir_path_for_server_path, account_id) + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + shutil.rmtree(nonsymlinked_dir) + def _delete_accounts_dir_for_server_path(self, server_path): - accounts_dir_path = self.config.accounts_dir_for_server_path(server_path) + link_func = self.config.accounts_dir_for_server_path + nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func) + os.rmdir(nonsymlinked_dir) + + def _delete_links_and_find_target_dir(self, server_path, link_func): + """Delete symlinks and return the nonsymlinked directory path. + + :param str server_path: file path based on server + :param callable link_func: callable that returns possible links + given a server_path + + :returns: the final, non-symlinked target + :rtype: str + + """ + dir_path = link_func(server_path) # does an appropriate directory link to me? if so, make sure that's gone reused_servers = {} for k in constants.LE_REUSE_SERVERS: reused_servers[constants.LE_REUSE_SERVERS[k]] = k - # is there a next one up? call that and be done - if server_path in reused_servers: - next_server_path = reused_servers[server_path] - next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path) - if os.path.islink(next_accounts_dir_path) \ - and os.readlink(next_accounts_dir_path) == accounts_dir_path: - self._delete_accounts_dir_for_server_path(next_server_path) - return + # is there a next one up? + possible_next_link = True + while possible_next_link: + possible_next_link = False + if server_path in reused_servers: + next_server_path = reused_servers[server_path] + next_dir_path = link_func(next_server_path) + if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path: + possible_next_link = True + server_path = next_server_path + dir_path = next_dir_path # if there's not a next one up to delete, then delete me - # and whatever I link to if applicable - if os.path.islink(accounts_dir_path): - # save my info then delete me - target = os.readlink(accounts_dir_path) - os.unlink(accounts_dir_path) - # then delete whatever I linked to, if appropriate - if server_path in constants.LE_REUSE_SERVERS: - prev_server_path = constants.LE_REUSE_SERVERS[server_path] - prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path) - if target == prev_accounts_dir_path: - self._delete_accounts_dir_for_server_path(prev_server_path) - else: - # just delete me - os.rmdir(accounts_dir_path) + # and whatever I link to + while os.path.islink(dir_path): + target = os.readlink(dir_path) + os.unlink(dir_path) + dir_path = target + + return dir_path def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index e0ec3d5f8..701478336 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -249,6 +249,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + def test_upgrade_load_single_account(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + prev_account = self.storage.load(self.acc.id) + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + account = self.storage.load(self.acc.id) + self.assertEqual(prev_account, account) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() @@ -307,6 +315,18 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + def _set_server_and_stop_symlink(self, server_path): + self._set_server(server_path) + with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: + f.write('bar') + + def test_delete_shared_account_up(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + + def test_delete_shared_account_down(self): + self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 72daec4346e3e43eed1ba4f42d5ab6e3346e8798 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Jul 2018 08:38:50 -0700 Subject: [PATCH 304/639] fix account tests (#6216) --- certbot/tests/account_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 701478336..b2be47d0f 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -295,7 +295,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') - self.storage.find_all() + self.storage.load(self.acc.id) # delete starting at given server_url self._set_server(server_url) @@ -326,7 +326,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_delete_shared_account_down(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') - self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') if __name__ == "__main__": unittest.main() # pragma: no cover From 595e77eccbd58131db7a3e07ad9385f436119dc3 Mon Sep 17 00:00:00 2001 From: Trinopoty Biswas Date: Mon, 16 Jul 2018 21:10:16 +0530 Subject: [PATCH 305/639] Fixed linode API settings page URL (#6208) --- certbot-dns-linode/certbot_dns_linode/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index aaed61450..0c445f45d 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -23,7 +23,7 @@ Credentials Use of this plugin requires a configuration file containing Linode API credentials, obtained from your Linode account's `Applications & API -Tokens page `_. +Tokens page `_. .. code-block:: ini :name: credentials.ini From 704101c75b4bf2a7e3f45553fac2ea066bc2108b Mon Sep 17 00:00:00 2001 From: hal869 <36906232+hal869@users.noreply.github.com> Date: Mon, 16 Jul 2018 10:20:53 -0700 Subject: [PATCH 306/639] update venv3.sh to include dns-rfc2136 plugin (#6226) --- tools/venv3.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/venv3.sh b/tools/venv3.sh index 784fc42e8..07512f370 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -24,6 +24,7 @@ fi -e certbot-dns-luadns \ -e certbot-dns-nsone \ -e certbot-dns-ovh \ + -e certbot-dns-rfc2136 \ -e certbot-dns-route53 \ -e certbot-dns-sakuracloud \ -e certbot-nginx \ From a0d68338a2da9ff0d7c751777086dc56b88839b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 16 Jul 2018 16:36:59 -0700 Subject: [PATCH 307/639] Release 0.26.1 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 14 ++++++---- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 86 insertions(+), 84 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88967c716..fe071ad56 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0a40a2f3..7cb15b8ad 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 17e00929b..e097719db 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c5381278e..0741a53af 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c7a3418c6..c036dab8f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index e5c32af9d..a70a65a2e 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 039c49767..69f422c18 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2f3f95a9c..4a0a91e2a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 3d1ce16c9..2daae6a10 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index be7515274..03c3b8bcb 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e020412b5..7e3729e2b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 3043aafed..9da99852d 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 3ea232bc3..259285eef 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2e63de4f0..7e7072fcf 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fc76b6e0c..fb10ebc2f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 813dc4981..4f4e4bc75 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 97039dade..cbe3802d6 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 53c65af0f..f7b134eb8 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index dd30dae7f..0ce8e8147 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.0' +version = '0.26.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 766be39a3..7fa464284 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.0' +__version__ = '0.26.1' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index c044c206a..931ea4c62 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.26.0 + "". (default: CertbotACMEClient/0.26.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -475,9 +475,11 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: None) + Path to the Apache 'a2enmod' binary. (default: + a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: None) + Path to the Apache 'a2dismod' binary. (default: + a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -491,13 +493,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you. (Only Ubuntu/Debian currently) (default: False) + you. (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you. (Only - Ubuntu/Debian currently) (default: False) + Ubuntu/Debian currently) (default: True) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index 17e00929b..e097719db 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 0b7d27be3..9f6706931 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- +Version: GnuPG v2 -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAltGdAYACgkQTRfJlc2X -dfKUyQf+NakhD3SfMeuJyT1StexEc9iGaAvspNH+Gf6P5v5dZOZnSOdtraR2kQAi -OQE2L5FAajIhpuELpZTAgCEFU1LZpqTvWOb/1Vb06T8DuLIYierh64LkAn0zJY/M -e8PTWyU5dcM6pY0ITvhuIMDAtomV+TzKeD1qHy2hJVTJGttk/yNtT5p8/NYIuH8Z -OWXkNuo/346xvYpTDp2Xpwv79L9JhQsxfEBpKV4IGObpTf+Mfl2f4taroLYEATGU -vrNM39P0cxu/hEHpog74CHPeK99YlBR6+7tMINQ9bYHkdjq2vLYdyopE8mCN16oy -CwITDfR5POwvs+WjU+oEtgQb73kTug== -=3XNY +iQEcBAABCAAGBQJbTSv8AAoJEE0XyZXNl3Xy12sH/1FgV3SDVG0T1jgKQOYEUwrq +cmpjdav8YPgFOSQDOcyFZG0DNcRfTskZt45IMkBLLnXq2PuPvkppc1+akP81vMoK +NXHHS+PXDMjnBW4NFkexoM06KRF1SyHnvqsOg13w7UW2CjsAgtazGF5BucNCnjPH +XJTwUf4uhKxeUb0Xkva1OPH++oTWz8+SYgWr/iMggkBrK8y04QUUJ6lyCO6MZgcE +3JcECG7CwMK+hW0gCUkCSNZ0NzOBALCd9wCxNGszgkeJXrrW73oUpZmGC5BxIwYY +o6lcF0qo7Jb92t4B3+7JhulMC5JoVoG4lpiXpKQFFCT0P4pZKotIomKNMATmnB4= +=hzUL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 17e00929b..e097719db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.0" +LE_AUTO_VERSION="0.26.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index cf87eacfc908dc352e0d17756ecd6bc9d7320e63..d1306d03d5cb08580450b34b2cc50a7d5b35bde8 100644 GIT binary patch literal 256 zcmV+b0ssEZf5kI!7rKt%`BTJd3(dQ}1+=^wm$LZwtN4D@Kt?U1ALJ@F2wf=MBK-J| z2C_km8Z46qIOyMvkA(v9J>+vwvIk@Id*X$TMU_IAT;weL#Iofv&kDQ3yBu2YDx`x? zFUBjgYmFBxeWs=PgXF6Oerz8PWP(3kjE-j7Ci&~=g GfTdGDx_-t0 literal 256 zcmV+b0ssD!H-fBQ`@sIYn!8z-`j+lz{Q@N0u$U83m47cn~8z5v4(zv^ufnv-raM*`SHDa?$*L zwjYLFp(zsV>M%UIA)#&*&!|W<43f30`QC>x;6{5kHWfag>;fAdx=2aP?uJ%oq~Mt# zwrh6trQ~7|S0C@}LAcZA0h*o^am2HkeuXz7reK3777FD4>jya36?*RxD`;b-E+2?X Gsx%mai+NN4 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index f80ee2cb4..feb3f1c3a 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.26.0 \ - --hash=sha256:0e171c00fce6ca7f3638602caaa9ca0b5b41ff35013d8a802afbea1d4b77e83a \ - --hash=sha256:5c0a0394c3745fa2d1ef49b9f8d0bd31eec11113b1b127055172fb053dc0946d -acme==0.26.0 \ - --hash=sha256:65ea0b75eba577775afbdc81db576a7ebc5287c87d04c18017d25ee899698956 \ - --hash=sha256:86d5fe89daf45d46dce68711990d6a145b323d84ee7b34322bfe20dc1624e26f -certbot-apache==0.26.0 \ - --hash=sha256:72e147a19c7ab609f6656529f1574327cb08b90d7556974e131f795cab04d18b \ - --hash=sha256:865a08ea38e7911745804de078a386e994888c084823e45710d5cc58ac5824c5 -certbot-nginx==0.26.0 \ - --hash=sha256:4bebf1350765ed3220a163e0c63b23021d19172aee5b7896b12e2341ea129210 \ - --hash=sha256:18d5a9b10aed07a9f0d465e6f08ee57ca112b356e7bc3190ee2ec66347f45cf4 +certbot==0.26.1 \ + --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ + --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 +acme==0.26.1 \ + --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ + --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be +certbot-apache==0.26.1 \ + --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ + --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 +certbot-nginx==0.26.1 \ + --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ + --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 From e9af000b5b452170199ac525b77787844bd486ed Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 16 Jul 2018 16:37:25 -0700 Subject: [PATCH 308/639] Bump version to 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index fe071ad56..88592013c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7cb15b8ad..f435bb1a9 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 0741a53af..e4ccc719a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index c036dab8f..05649d4d0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a70a65a2e..911f9e052 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 69f422c18..9dd318296 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4a0a91e2a..09b11def0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2daae6a10..2ca3213bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 03c3b8bcb..e9ead6546 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 7e3729e2b..3c7402f25 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 9da99852d..327224c9c 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 259285eef..1f92f7dce 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 7e7072fcf..0b4241afb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index fb10ebc2f..258c7f0f1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 4f4e4bc75..bd54ec4c5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index cbe3802d6..5f0b26f6e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index f7b134eb8..b7cfc15b5 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0ce8e8147..4706f17bd 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.26.1' +version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 7fa464284..3b0b77f6c 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.26.1' +__version__ = '0.27.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index e097719db..765072c3f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 6fd312833f8e6cac9fb28f0e1af96d7d8643acb6 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Tue, 17 Jul 2018 15:29:08 -0700 Subject: [PATCH 309/639] Remove setuptools min version for OVH DNS plugin (#6229) This simplifies packaging for EPEL7. See also #5617. --- certbot-dns-ovh/setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 258c7f0f1..e0ce785a1 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'certbot>=0.21.1', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] From 783b6e4746ac95d8aefe3ac9d0616a4176dbd6ca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Jul 2018 17:19:04 -0700 Subject: [PATCH 310/639] Automate EBS cleanup (#6160) * ensure volume cleanup * remove volume cleanup * cleanup function and output --- tests/letstest/multitester.py | 51 +++++++++++++---------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 17740cde8..0ae9636d4 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -128,6 +128,7 @@ def make_instance(instance_name, userdata=""): #userdata contains bash or cloud-init script new_instance = EC2.create_instances( + BlockDeviceMappings=_get_block_device_mappings(ami_id), ImageId=ami_id, SecurityGroups=security_groups, KeyName=keyname, @@ -151,38 +152,21 @@ def make_instance(instance_name, raise return new_instance -def terminate_and_clean(instances): +def _get_block_device_mappings(ami_id): + """Returns the list of block device mappings to ensure cleanup. + + This list sets connected EBS volumes to be deleted when the EC2 + instance is terminated. + """ - Some AMIs specify EBS stores that won't delete on instance termination. - These must be manually deleted after shutdown. - """ - volumes_to_delete = [] - for instance in instances: - for bdmap in instance.block_device_mappings: - if 'Ebs' in bdmap.keys(): - if not bdmap['Ebs']['DeleteOnTermination']: - volumes_to_delete.append(bdmap['Ebs']['VolumeId']) - - for instance in instances: - instance.terminate() - - # can't delete volumes until all attaching instances are terminated - _ids = [instance.id for instance in instances] - all_terminated = False - while not all_terminated: - all_terminated = True - for _id in _ids: - # necessary to reinit object for boto3 to get true state - inst = EC2.Instance(id=_id) - if inst.state['Name'] != 'terminated': - all_terminated = False - time.sleep(5) - - for vol_id in volumes_to_delete: - volume = EC2.Volume(id=vol_id) - volume.delete() - - return volumes_to_delete + # Not all devices use EBS, but the default value for DeleteOnTermination + # when the device does use EBS is true. See: + # * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-blockdev-mapping.html + # * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-blockdev-template.html + return [{'DeviceName': mapping['DeviceName'], + 'Ebs': {'DeleteOnTermination': True}} + for mapping in EC2.Image(ami_id).block_device_mappings + if not mapping.get('Ebs', {}).get('DeleteOnTermination', True)] # Helper Routines @@ -370,10 +354,11 @@ def test_client_process(inqueue, outqueue): def cleanup(cl_args, instances, targetlist): print('Logs in ', LOGDIR) if not cl_args.saveinstances: - print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') + print('Terminating EC2 Instances') if cl_args.killboulder: boulder_server.terminate() - terminate_and_clean(instances) + for instance in instances: + instance.terminate() else: # print login information for the boxes for debugging for ii, target in enumerate(targetlist): From 94cadd33eb4efce19c1023346d5721703e8c88f6 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 17 Jul 2018 20:00:12 -0700 Subject: [PATCH 311/639] test(postfix): env for testing on oldest deps (#6230) Fixes #6124. --- .travis.yml | 2 +- certbot-postfix/certbot_postfix/tests/installer_test.py | 2 +- tox.ini | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e3f7a021..b671c0e8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: - python: "3.5" env: TOXENV=mypy - python: "2.7" - env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' + env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest' sudo: required services: docker - python: "3.4" diff --git a/certbot-postfix/certbot_postfix/tests/installer_test.py b/certbot-postfix/certbot_postfix/tests/installer_test.py index 1bdd2c8b3..37b78bdca 100644 --- a/certbot-postfix/certbot_postfix/tests/installer_test.py +++ b/certbot-postfix/certbot_postfix/tests/installer_test.py @@ -253,7 +253,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase): fake_set.reset_mock() installer.deploy_cert("example.com", "cert_path", "key_path", "chain_path", "fullchain_path") - fake_set.assert_not_called() + self.assertFalse(fake_set.called) @certbot_test_util.patch_get_utility() def test_deploy_already_secure(self, mock_util): diff --git a/tox.ini b/tox.ini index d113fd450..0676a0da4 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,12 @@ commands = setenv = {[testenv:py27-oldest]setenv} +[testenv:py27-postfix-oldest] +commands = + {[base]install_and_test} certbot-postfix +setenv = + {[testenv:py27-oldest]setenv} + [testenv:py27_install] basepython = python2.7 commands = From cdc333491be3cc48dd17c5191f815859035bfdc7 Mon Sep 17 00:00:00 2001 From: R3DDY97 Date: Wed, 18 Jul 2018 21:04:09 +0530 Subject: [PATCH 312/639] Gpg2 doc (#5981) --- docs/install.rst | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index ead59350d..f7504baa5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -40,11 +40,17 @@ supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. + +Additional integrity verification of certbot-auto script can be done by verifying its digital signature. +This requires a local installation of gpg2, which comes packaged in many Linux distributions under name gnupg or gnupg2. + + Installing with ``certbot-auto`` requires 512MB of RAM in order to build some of the dependencies. Installing from pre-built OS packages avoids this requirement. You can also temporarily set a swap file. See "Problems with Python virtual environment" below for details. + Alternate installation methods ================================ @@ -64,12 +70,30 @@ download and run it as follows:: user@webserver:~$ chmod a+x ./certbot-auto user@webserver:~$ ./certbot-auto --help -.. hint:: The certbot-auto download is protected by HTTPS, which is pretty good, but if you'd like to - double check the integrity of the ``certbot-auto`` script, you can use these steps for verification before running it:: +To check the integrity of the ``certbot-auto`` script, +you can use these steps:: + + + user@webserver:~$ wget -N https://dl.eff.org/certbot-auto.asc + user@webserver:~$ gpg2 --keyserver pool.sks-keyservers.net --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 + user@webserver:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc certbot-auto + + + +The output of the last command should look something like:: + + + gpg: Signature made Wed 02 May 2018 05:29:12 AM IST + gpg: using RSA key A2CFB51FA275A7286234E7B24D17C995CD9775F2 + gpg: key 4D17C995CD9775F2 marked as ultimately trusted + gpg: checking the trustdb + gpg: marginals needed: 3 completes needed: 1 trust model: pgp + gpg: depth: 0 valid: 2 signed: 2 trust: 0-, 0q, 0n, 0m, 0f, 2u + gpg: depth: 1 valid: 2 signed: 0 trust: 2-, 0q, 0n, 0m, 0f, 0u + gpg: next trustdb check due at 2027-11-22 + gpg: Good signature from "Let's Encrypt Client Team " [ultimate] + - user@server:~$ wget -N https://dl.eff.org/certbot-auto.asc - user@server:~$ gpg2 --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - user@server:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc certbot-auto The ``certbot-auto`` command updates to the latest client release automatically. Since ``certbot-auto`` is a wrapper to ``certbot``, it accepts exactly From daee6e8eb3ca999840013e3469c9706e3328c55f Mon Sep 17 00:00:00 2001 From: Eli Young Date: Thu, 19 Jul 2018 10:29:34 -0700 Subject: [PATCH 313/639] Fix test naming for certbot-dns-sakuracloud (#6231) --- .../certbot_dns_sakuracloud/dns_sakuracloud_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py index 84605d06f..1d9282f9a 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py @@ -38,7 +38,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, self.auth._get_sakuracloud_client = mock.MagicMock(return_value=self.mock_client) -class NS1LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): +class SakuraCloudLexiconClientTest(unittest.TestCase, + dns_test_common_lexicon.BaseLexiconClientTest): DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN)) LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN)) From c129ab2965f7ec4edf173b1cd900bd5add13cd2a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 23 Jul 2018 11:46:22 -0700 Subject: [PATCH 314/639] Bump the acme version needed for account reuse (#6250) * Bump the acme version needed for account reuse. Fixes https://github.com/certbot/certbot/issues/6155#issuecomment-407122742. * Update nginx oldest requirements. * bump min acme version * update min acme version --- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 2 +- local-oldest-requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index e70ac0c7f..38ed5debe 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.25.0 +acme[dev]==0.26.0 -e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 4706f17bd..02aaa6581 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -7,7 +7,7 @@ version = '0.27.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.25.0', + 'acme>=0.26.0', 'certbot>=0.22.0', 'mock', 'PyOpenSSL', diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 1f449acae..03226fc84 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ -acme[dev]==0.25.0 +acme[dev]==0.26.0 diff --git a/setup.py b/setup.py index fc87917fb..1827c4d42 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.25.0', + 'acme>=0.26.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From dc913574189b780e5b78f28ffaf5db47d3132436 Mon Sep 17 00:00:00 2001 From: dovreshef Date: Fri, 27 Jul 2018 18:50:53 +0300 Subject: [PATCH 315/639] Raise ConflictError on attempts to create an existing account (#6251) * Raise ConflictError on attempts to create an existing account in ACME V2. Fixes issue #6246 * Allow querying an account without calling new_account in ACMEv2 Fixed issue #6258 --- acme/acme/client.py | 17 +++++++++++++++++ acme/acme/client_test.py | 11 +++++++++++ acme/acme/errors.py | 2 ++ 3 files changed, 30 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index a0bfe460d..bd86657b9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -577,16 +577,33 @@ class ClientV2(ClientBase): :param .NewRegistration new_account: + :raises .ConflictError: in case the account already exists + :returns: Registration Resource. :rtype: `.RegistrationResource` """ response = self._post(self.directory['newAccount'], new_account) + # if account already exists + if response.status_code == 200 and 'Location' in response.headers: + raise errors.ConflictError(response.headers.get('Location')) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) self.net.account = regr return regr + def query_registration(self, regr): + """Query server about registration. + + :param messages.RegistrationResource: Existing Registration + Resource. + + """ + self.net.account = regr + updated_regr = super(ClientV2, self).query_registration(regr) + self.net.account = updated_regr + return updated_regr + def update_registration(self, regr, update=None): """Update registration. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index cd31c4ac3..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -134,6 +134,12 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.acme_version, 2) + def test_query_registration_client_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.response.json.return_value = self.regr.body.to_json() + self.assertEqual(self.regr, client.query_registration(self.regr)) + def test_forwarding(self): self.response.json.return_value = DIRECTORY_V1.to_json() client = self._init() @@ -706,6 +712,11 @@ class ClientV2Test(ClientTestBase): self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + def test_new_account_conflict(self): + self.response.status_code = http_client.OK + self.response.headers['Location'] = self.regr.uri + self.assertRaises(errors.ConflictError, self.client.new_account, self.new_reg) + def test_new_order(self): order_response = copy.deepcopy(self.response) order_response.status_code = http_client.CREATED diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 97fa73614..3a0f8c596 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -110,6 +110,8 @@ class ConflictError(ClientError): In the version of ACME implemented by Boulder, this is used to find an account if you only have the private key, but don't know the account URL. + + Also used in V2 of the ACME client for the same purpose. """ def __init__(self, location): self.location = location From ee7d5052fdf06fd364d9dffec4f89c84de84e2f8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 30 Jul 2018 16:32:31 -0700 Subject: [PATCH 316/639] Update changelog for 0.26.1 release (#6237) * Update changelog for 0.26.1 release --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d0808de6..6d384ad30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.26.1 - 2018-07-17 + +### Fixed + +* Fix a bug that was triggered when users who had previously manually set `--server` to get ACMEv2 certs tried to renew ACMEv1 certs. + +Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: + +* certbot + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/58?closed=1 + ## 0.26.0 - 2018-07-11 ### Added @@ -783,7 +796,7 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac ### Added -* When using the standalone plugin while running Certbot interactively +* When using the standalone plugin while running Certbot interactively and a required port is bound by another process, Certbot will give you the option to retry to grab the port rather than immediately exiting. * You are now able to deactivate your account with the Let's Encrypt From d19698251da19f363d94ece9e5ee9dcc425afad2 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 31 Jul 2018 17:08:39 +0300 Subject: [PATCH 317/639] Do not send status or resource fields in newOrder payloads for ACMEv2 --- acme/acme/client.py | 4 ++-- acme/acme/messages.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index bd86657b9..60a67a038 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -646,7 +646,7 @@ class ClientV2(ClientBase): value=name)) order = messages.NewOrder(identifiers=identifiers) response = self._post(self.directory['newOrder'], order) - body = messages.Order.from_json(response.json()) + body = messages.OrderBase.from_json(response.json()) authorizations = [] for url in body.authorizations: authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) @@ -715,7 +715,7 @@ class ClientV2(ClientBase): while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) - body = messages.Order.from_json(response.json()) + body = messages.OrderBase.from_json(response.json()) if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5be458580..86585ccb9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,11 +509,10 @@ class Revocation(jose.JSONObjectWithFields): reason = jose.Field('reason') -class Order(ResourceBody): +class OrderBase(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. - :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -522,8 +521,6 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -534,6 +531,16 @@ class Order(ResourceBody): def identifiers(value): # pylint: disable=missing-docstring,no-self-argument return tuple(Identifier.from_json(identifier) for identifier in value) + +class Order(OrderBase): + """Order Resource Body for ACMEv1 + + :ivar acme.messages.Status status: + """ + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + + class OrderResource(ResourceWithURI): """Order Resource. @@ -549,8 +556,8 @@ class OrderResource(ResourceWithURI): authorizations = jose.Field('authorizations') fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + @Directory.register -class NewOrder(Order): - """New order.""" - resource_type = 'new-order' - resource = fields.Resource(resource_type) +class NewOrder(OrderBase): + """New order for ACMEv2""" + resource_type = "new-order" From 8b3629ebd49b647967dde9e279ad252583f6b881 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 31 Jul 2018 19:55:19 +0300 Subject: [PATCH 318/639] Fix tests --- acme/acme/client_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..3cdcdf041 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -695,9 +695,8 @@ class ClientV2Test(ClientTestBase): self.authzr2 = messages.AuthorizationResource( body=self.authz2, uri=self.authzr_uri2) - self.order = messages.Order( + self.order = messages.OrderBase( identifiers=(self.authz.identifier, self.authz2.identifier), - status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( From 68f9a1b30042cbfc7b42667829a83d23d9f89f6c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 31 Jul 2018 09:56:03 -0700 Subject: [PATCH 319/639] Files with multiple vhosts are fine. (#6273) --- certbot-apache/certbot_apache/display_ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 097b84b96..db1c3cca4 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -113,8 +113,7 @@ def _vhost_menu(domain, vhosts): code, tag = zope.component.getUtility(interfaces.IDisplay).menu( "We were unable to find a vhost with a ServerName " "or Address of {0}.{1}Which virtual host would you " - "like to choose?\n(note: conf files with multiple " - "vhosts are not yet supported)".format(domain, os.linesep), + "like to choose?".format(domain, os.linesep), choices, force_interactive=True) except errors.MissingCommandlineFlag: msg = ( From 62629213153e40054654b1a71a3138a90091f6e7 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Tue, 31 Jul 2018 19:31:36 +0200 Subject: [PATCH 320/639] bump pytest-xdist to 1.22.5 (#6253) Signed-off-by: Yoan Blanc --- .travis.yml | 2 +- tools/dev_constraints.txt | 2 +- tox.cover.sh | 4 +--- tox.ini | 15 +++++++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b671c0e8d..367e00fea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover NUMPROCESSES=2 FYI="this also tests py27" + env: TOXENV=cover FYI="this also tests py27" - sudo: required env: TOXENV=nginx_compat services: docker diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 5a16b8cba..f14169bfc 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -48,7 +48,7 @@ pylint==1.4.2 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 -pytest-xdist==1.20.1 +pytest-xdist==1.22.5 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 diff --git a/tox.cover.sh b/tox.cover.sh index 6440d1b48..bb56ddf18 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -8,8 +8,6 @@ # # -e makes sure we fail fast and don't submit coveralls submit -NUMPROCESSES=${NUMPROCESSES:=auto} - if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" else @@ -63,7 +61,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "$NUMPROCESSES" --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 0676a0da4..9db06f78c 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,7 @@ source_paths = tests/lock_test.py [testenv] +passenv = TRAVIS commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -121,7 +122,6 @@ commands = [testenv:cover] basepython = python2.7 -passenv = NUMPROCESSES commands = {[base]install_packages} ./tox.cover.sh @@ -166,7 +166,9 @@ commands = docker run --rm -it apache-compat -c apache.tar.gz -vvvv whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:nginx_compat] commands = @@ -175,7 +177,9 @@ commands = docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_precise] # At the moment, this tests under Python 2.7 only, as only that version is @@ -185,7 +189,9 @@ commands = docker run --rm -t -i lea whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -198,6 +204,7 @@ whitelist_externals = docker passenv = DOCKER_* + TRAVIS TRAVIS_BRANCH [testenv:le_auto_wheezy] From f2bc876b6eddc248cd4ab021deefd2d7ef85f7b6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 31 Jul 2018 15:56:22 -0700 Subject: [PATCH 321/639] switch to codecov (#6220) --- .travis.yml | 4 ++-- README.rst | 4 ++-- tox.cover.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 367e00fea..b4d702ff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,12 +90,12 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" +install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && coveralls' +after_success: '[ "$TOXENV" == "cover" ] && codecov' notifications: email: false diff --git a/README.rst b/README.rst index a18028aee..0dbe1cdef 100644 --- a/README.rst +++ b/README.rst @@ -107,8 +107,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master - :target: https://coveralls.io/r/certbot/certbot +.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg + :target: https://codecov.io/gh/certbot/certbot :alt: Coverage status .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/ diff --git a/tox.cover.sh b/tox.cover.sh index bb56ddf18..c68e757de 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -6,7 +6,7 @@ # generate separate stats for each package. It should be removed once # those packages are moved to separate repo. # -# -e makes sure we fail fast and don't submit coveralls submit +# -e makes sure we fail fast and don't submit to codecov if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" From c131f4211de987cdd7738a90c091fb25dce54351 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:10:01 +0300 Subject: [PATCH 322/639] Revert "Fix tests" This reverts commit 8b3629ebd49b647967dde9e279ad252583f6b881. --- acme/acme/client_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 3cdcdf041..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -695,8 +695,9 @@ class ClientV2Test(ClientTestBase): self.authzr2 = messages.AuthorizationResource( body=self.authz2, uri=self.authzr_uri2) - self.order = messages.OrderBase( + self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( From b1b46508045d90715e3043ad9e373ba43edf9a8f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:10:55 +0300 Subject: [PATCH 323/639] Revert "Do not send status or resource fields in newOrder payloads for ACMEv2" This reverts commit d19698251da19f363d94ece9e5ee9dcc425afad2. --- acme/acme/client.py | 4 ++-- acme/acme/messages.py | 23 ++++++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 60a67a038..bd86657b9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -646,7 +646,7 @@ class ClientV2(ClientBase): value=name)) order = messages.NewOrder(identifiers=identifiers) response = self._post(self.directory['newOrder'], order) - body = messages.OrderBase.from_json(response.json()) + body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) @@ -715,7 +715,7 @@ class ClientV2(ClientBase): while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) - body = messages.OrderBase.from_json(response.json()) + body = messages.Order.from_json(response.json()) if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 86585ccb9..5be458580 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,10 +509,11 @@ class Revocation(jose.JSONObjectWithFields): reason = jose.Field('reason') -class OrderBase(ResourceBody): +class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -521,6 +522,8 @@ class OrderBase(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -531,16 +534,6 @@ class OrderBase(ResourceBody): def identifiers(value): # pylint: disable=missing-docstring,no-self-argument return tuple(Identifier.from_json(identifier) for identifier in value) - -class Order(OrderBase): - """Order Resource Body for ACMEv1 - - :ivar acme.messages.Status status: - """ - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) - - class OrderResource(ResourceWithURI): """Order Resource. @@ -556,8 +549,8 @@ class OrderResource(ResourceWithURI): authorizations = jose.Field('authorizations') fullchain_pem = jose.Field('fullchain_pem', omitempty=True) - @Directory.register -class NewOrder(OrderBase): - """New order for ACMEv2""" - resource_type = "new-order" +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) From 8943dffe0d804eba8da99a77c5b8b4b72cce8991 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:17:23 +0300 Subject: [PATCH 324/639] Removed status and resource fields from NewOrder object --- acme/acme/client_test.py | 1 - acme/acme/messages.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..965ece55d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -697,7 +697,6 @@ class ClientV2Test(ClientTestBase): self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), - status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5be458580..405fe7d9a 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -513,7 +513,6 @@ class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. - :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -522,8 +521,6 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -553,4 +550,3 @@ class OrderResource(ResourceWithURI): class NewOrder(Order): """New order.""" resource_type = 'new-order' - resource = fields.Resource(resource_type) From f6219ddf196f1de0c4d12e1bb5c1f6d737a6e844 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 22:00:47 +0300 Subject: [PATCH 325/639] Enable Apache VirtualHost for HTTP challenge validation if not enabled already (#6268) If user provides a custom --apache-vhost-root path that's not parsed by Apache per default, Certbot fails the challenge validation. While the VirtualHost on custom path is correctly found, and edited, it's still not seen by Apache. This PR adds a temporary Include directive to the root Apache configuration when writing the challenge tokens to the VirtualHost. --- certbot-apache/certbot_apache/http_01.py | 6 ++++++ .../certbot_apache/tests/http_01_test.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 37545e9cc..22598baca 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -6,6 +6,7 @@ from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-m from certbot import errors from certbot.plugins import common from certbot_apache.obj import VirtualHost # pylint: disable=unused-import +from certbot_apache.parser import get_aug_path logger = logging.getLogger(__name__) @@ -172,4 +173,9 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.parser.add_dir( vhost.path, "Include", self.challenge_conf_post) + if not vhost.enabled: + self.configurator.parser.add_dir( + get_aug_path(self.configurator.parser.loc["default"]), + "Include", vhost.filep) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 98bf412ae..9c729b08c 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -10,6 +10,7 @@ from certbot import achallenges from certbot import errors from certbot.tests import acme_util +from certbot_apache.parser import get_aug_path from certbot_apache.tests import util @@ -134,6 +135,21 @@ class ApacheHttp01Test(util.ApacheTest): def test_perform_3_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=3, minor_version=4) + def test_activate_disabled_vhost(self): + vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="certbot.demo", account_key=self.account_key)] + vhosts[0].enabled = False + self.common_perform_test(achalls, vhosts) + matches = self.config.parser.find_dir( + "Include", vhosts[0].filep, + get_aug_path(self.config.parser.loc["default"])) + self.assertEqual(len(matches), 1) + def combinations_perform_test(self, num_achalls, minor_version): """Test perform with the given achall count and Apache version.""" achalls = self.achalls[:num_achalls] From 7bff0a02e4b86db47015987bbafa3d3a93246829 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 2 Aug 2018 18:17:38 +0300 Subject: [PATCH 326/639] Make Apache control script and binary paths configurable on command line (#6238) This PR adds two new command line parameters, --apache-ctlpath and --apache-binpath both of which are used to construct commands that we shell out for. The way that we previously fetched values either from Certbot configuration object or the dictionary of distribution based constants is now also unified, and the active options are parsed in prepare() to make it easier to override needed values for the distributions needing this behavior. Fixes: #5338 * Added the command line options and parsing * Refactor existing code * Distro override updates * Handle vhost_root from cli * Fix compatibility tests * Add comment about changes to command line arguments * Check None properly * Made help texts consistent * Keep the old defaults * Move to shorter CLI parameter names * No need for specific bin path, nor apache_cmd anymore * Make sure that we use user provided vhost-root value * Fix alt restart commands in overrides * Fix version_cmd defaults in overrides * Fix comparison * Remove cruft, and use configuration object for parser parameter --- certbot-apache/certbot_apache/configurator.py | 115 +++++++++++------- .../certbot_apache/override_arch.py | 4 +- .../certbot_apache/override_centos.py | 14 ++- .../certbot_apache/override_darwin.py | 6 +- .../certbot_apache/override_debian.py | 10 +- .../certbot_apache/override_gentoo.py | 18 ++- .../certbot_apache/override_suse.py | 2 +- certbot-apache/certbot_apache/parser.py | 8 +- .../certbot_apache/tests/autohsts_test.py | 3 + .../certbot_apache/tests/centos_test.py | 2 + .../certbot_apache/tests/configurator_test.py | 33 ++--- .../certbot_apache/tests/debian_test.py | 2 +- .../certbot_apache/tests/gentoo_test.py | 2 +- .../certbot_apache/tests/parser_test.py | 6 +- certbot-apache/certbot_apache/tests/util.py | 49 ++++---- .../configurators/apache/common.py | 3 - 16 files changed, 150 insertions(+), 127 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb77e2e41..da632dc81 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import copy import fnmatch import logging import os @@ -97,48 +98,72 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) - def constant(self, key): - """Get constant for OS_DEFAULTS""" - return self.OS_DEFAULTS.get(key) + def option(self, key): + """Get a value from options""" + return self.options.get(key) + + def _prepare_options(self): + """ + Set the values possibly changed by command line parameters to + OS_DEFAULTS constant dictionary + """ + opts = ["enmod", "dismod", "le_vhost_ext", "server_root", "vhost_root", + "logs_root", "challenge_location", "handle_modules", "handle_sites", + "ctl"] + for o in opts: + # Config options use dashes instead of underscores + if self.conf(o.replace("_", "-")) is not None: + self.options[o] = self.conf(o.replace("_", "-")) + else: + self.options[o] = self.OS_DEFAULTS[o] + + # Special cases + self.options["version_cmd"][0] = self.option("ctl") + self.options["restart_cmd"][0] = self.option("ctl") + self.options["conftest_cmd"][0] = self.option("ctl") @classmethod def add_parser_arguments(cls, add): + # When adding, modifying or deleting command line arguments, be sure to + # include the changes in the list used in method _prepare_options() to + # ensure consistent behavior. add("enmod", default=cls.OS_DEFAULTS["enmod"], - help="Path to the Apache 'a2enmod' binary.") + help="Path to the Apache 'a2enmod' binary") add("dismod", default=cls.OS_DEFAULTS["dismod"], - help="Path to the Apache 'a2dismod' binary.") + help="Path to the Apache 'a2dismod' binary") add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], - help="SSL vhost configuration extension.") + help="SSL vhost configuration extension") add("server-root", default=cls.OS_DEFAULTS["server_root"], - help="Apache server root directory.") + help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") add("logs-root", default=cls.OS_DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", default=cls.OS_DEFAULTS["challenge_location"], - help="Directory path for challenge configuration.") - add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you. " + + help="Directory path for challenge configuration") + add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you. " + + help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - util.add_deprecated_argument(add, argument_name="ctl", nargs=1) + add("ctl", default=cls.OS_DEFAULTS["ctl"], + help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) @@ -169,7 +194,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser = None self.version = version self.vhosts = None - self.vhostroot = None + self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} @@ -201,12 +226,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except ImportError: raise errors.NoInstallationError("Problem in Augeas installation") + self._prepare_options() + # Verify Apache is installed - restart_cmd = self.constant("restart_cmd")[0] - if not util.exe_exists(restart_cmd): - if not path_surgery(restart_cmd): - raise errors.NoInstallationError( - 'Cannot find Apache control command {0}'.format(restart_cmd)) + self._verify_exe_availability(self.option("ctl")) # Make sure configuration is valid self.config_test() @@ -226,12 +249,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - # Parse vhost-root if defined on cli - if not self.conf("vhost-root"): - self.vhostroot = self.constant("vhost_root") - else: - self.vhostroot = os.path.abspath(self.conf("vhost-root")) - self.parser = self.get_parser() # Check for errors in parsing files with Augeas @@ -245,13 +262,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.conf("server-root")) + util.lock_dir_until_exit(self.option("server_root")) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( - "Unable to lock %s", self.conf("server-root")) + "Unable to lock %s", self.option("server_root")) self._prepared = True + def _verify_exe_availability(self, exe): + """Checks availability of Apache executable""" + if not util.exe_exists(exe): + if not path_surgery(exe): + raise errors.NoInstallationError( + 'Cannot find Apache executable {0}'.format(exe)) + def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. If augeas version is recent enough, it will support case insensitive @@ -269,8 +293,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def get_parser(self): """Initializes the ApacheParser""" + # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.conf("vhost-root"), self.version, configurator=self) def _wildcard_domain(self, domain): @@ -1037,7 +1062,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param boolean temp: If the change is temporary """ - if self.conf("handle-modules"): + if self.option("handle_modules"): if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1066,7 +1091,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.constant("le_vhost_ext")`` + ``self.option("le_vhost_ext")`` .. note:: This function saves the configuration @@ -1165,18 +1190,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - # Defined by user on CLI - - fp = os.path.join(os.path.realpath(self.vhostroot), + fp = os.path.join(os.path.realpath(self.option("vhost_root")), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = os.path.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.conf("le_vhost_ext") + return fp[:-(len(".conf"))] + self.option("le_vhost_ext") else: - return fp + self.conf("le_vhost_ext") + return fp + self.option("le_vhost_ext") def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2025,7 +2048,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.conf("logs-root"))) + self.option("logs_root"))) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2037,7 +2060,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.vhostroot, + redirect_filepath = os.path.join(self.option("vhost_root"), redirect_filename) # Register the new file that will be created @@ -2158,18 +2181,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ error = "" try: - util.run_script(self.constant("restart_cmd")) + util.run_script(self.option("restart_cmd")) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.constant("restart_cmd")) - alt_restart = self.constant("restart_cmd_alt") + self.option("restart_cmd")) + alt_restart = self.option("restart_cmd_alt") if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.constant( + util.run_script(self.option( "restart_cmd_alt")) return except errors.SubprocessError as secerr: @@ -2185,7 +2208,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - util.run_script(self.constant("conftest_cmd")) + util.run_script(self.option("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2201,11 +2224,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = util.run_script(self.constant("version_cmd")) + stdout, _ = util.run_script(self.option("version_cmd")) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.constant("version_cmd")) + self.option("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) @@ -2295,7 +2318,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # certbot for unprivileged users via setuid), this function will need # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/override_arch.py b/certbot-apache/certbot_apache/override_arch.py index ea5155a3c..c5620e9f9 100644 --- a/certbot-apache/certbot_apache/override_arch.py +++ b/certbot-apache/certbot_apache/override_arch.py @@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index 0b6b12b96..a4f1b84ec 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -18,25 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "centos-options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in CentOS. + """ + super(CentOSConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) diff --git a/certbot-apache/certbot_apache/override_darwin.py b/certbot-apache/certbot_apache/override_darwin.py index 53741d504..4e2a6acac 100644 --- a/certbot-apache/certbot_apache/override_darwin.py +++ b/certbot-apache/certbot_apache/override_darwin.py @@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/other", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/httpd', '-v'], - apache_cmd="/usr/sbin/httpd", + ctl="apachectl", + version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 02dffc3f7..0caa619d2 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -23,14 +23,14 @@ class DebianConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=True, + handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( @@ -134,11 +134,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.conf("dismod")): + if not util.exe_exists(self.option("dismod")): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), "-f", mod_name]) - util.run_script([self.conf("enmod"), mod_name]) + temp, [self.option("dismod"), "-f", mod_name]) + util.run_script([self.option("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 165e44c96..556e3225e 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -18,25 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/apache2', '-v'], - apache_cmd="apache2ctl", + ctl="apache2ctl", + version_cmd=['apache2ctl', '-v'], restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in Gentoo. + """ + super(GentooConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) @@ -61,7 +69,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + mod_cmd = [self.configurator.option("ctl"), "modules"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py index a67054b5b..83079b92c 100644 --- a/certbot-apache/certbot_apache/override_suse.py +++ b/certbot-apache/certbot_apache/override_suse.py @@ -16,8 +16,8 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 02337f8d4..148f052d0 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -69,7 +69,7 @@ class ApacheParser(object): # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.constant("vhost_files")) + self.configurator.option("vhost_files")) # check to see if there were unparsed define statements if version < (2, 4): @@ -152,7 +152,7 @@ class ApacheParser(object): """Get Defines from httpd process""" variables = dict() - define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + define_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_RUN_CFG"] matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") try: @@ -179,7 +179,7 @@ class ApacheParser(object): # configuration files _ = self.find_dir("Include") - inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + inc_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_INCLUDES"] matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)") if matches: @@ -190,7 +190,7 @@ class ApacheParser(object): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + mod_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_MODULES"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 73da33f15..c5d720dd3 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -119,6 +119,9 @@ class AutoHSTSTest(util.ApacheTest): cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), cur_val) + # Ensure that the value is raised to max + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + maxage.format(constants.AUTOHSTS_STEPS[-1])) # Make permanent self.config.deploy_autohsts(mock_lineage) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 4ee8b5dcf..46b857c3e 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -135,5 +135,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): errors.SubprocessError, errors.SubprocessError] self.assertRaises(errors.MisconfigurationError, self.config.restart) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 350262634..6f1c358c2 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -116,8 +116,9 @@ class MultipleVhostsTest(util.ApacheTest): ApacheConfigurator.add_parser_arguments(mock.MagicMock()) def test_constant(self): - self.assertEqual(self.config.constant("server_root"), "/etc/apache2") - self.assertEqual(self.config.constant("nonexistent"), None) + self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in + self.config.option("server_root")) + self.assertEqual(self.config.option("nonexistent"), None) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -651,22 +652,10 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost_slink.name, "nonsym.link") def test_make_vhost_ssl_nonexistent_vhost_path(self): - def conf_side_effect(arg): - """ Mock function for ApacheConfigurator.conf """ - confvars = { - "vhost-root": "/tmp/nonexistent", - "le_vhost_ext": "-le-ssl.conf", - "handle-sites": True} - return confvars[arg] - - with mock.patch( - "certbot_apache.configurator.ApacheConfigurator.conf" - ) as mock_conf: - mock_conf.side_effect = conf_side_effect - ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) - self.assertEqual(os.path.dirname(ssl_vhost.filep), - os.path.dirname(os.path.realpath( - self.vh_truth[1].filep))) + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) + self.assertEqual(os.path.dirname(ssl_vhost.filep), + os.path.dirname(os.path.realpath( + self.vh_truth[1].filep))) def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -1583,7 +1572,7 @@ class AugeasVhostsTest(util.ApacheTest): broken_vhost) class MultiVhostsTest(util.ApacheTest): - """Test vhosts with illegal names dependent on augeas version.""" + """Test configuration with multiple virtualhosts in a single file.""" # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ @@ -1703,7 +1692,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1739,7 +1728,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1755,7 +1744,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py index fde8d4c35..bb1d64278 100644 --- a/certbot-apache/certbot_apache/tests/debian_test.py +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ super(MultipleVhostsTestDebian, self).setUp() self.config = util.get_apache_configurator( - self.config_path, None, self.config_dir, self.work_dir, + self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth(self.temp_dir, diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index d32551267..0681e30b5 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.modules = set() with mock.patch("certbot.util.get_os_info") as mock_osi: - # Make sure we have the have the CentOS httpd constants + # Make sure we have the have the Gentoo httpd constants mock_osi.return_value = ("gentoo", "123") self.config.parser.update_runtime_variables() diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index f95f1b346..d62fd54e8 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -282,11 +282,11 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): mock_popen.side_effect = OSError - mock_const.return_value = "nonexistent" + mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 6d3cfa109..9329ccb20 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -97,9 +97,10 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc backups = os.path.join(work_dir, "backups") mock_le_config = mock.MagicMock( apache_server_root=config_path, - apache_vhost_root=conf_vhost_path, + apache_vhost_root=None, apache_le_vhost_ext="-le-ssl.conf", apache_challenge_location=config_path, + apache_enmod=None, backup_dir=backups, config_dir=config_dir, http01_port=80, @@ -107,33 +108,25 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) - orig_os_constant = configurator.ApacheConfigurator(mock_le_config, - name="apache", - version=version).constant - - def mock_os_constant(key, vhost_path=vhost_path): - """Mock default vhost path""" - if key == "vhost_root": - return vhost_path - else: - return orig_os_constant(key) - - with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons: - mock_cons.side_effect = mock_os_constant - with mock.patch("certbot_apache.configurator.util.run_script"): - with mock.patch("certbot_apache.configurator.util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - with mock.patch("certbot_apache.parser.ApacheParser." - "update_runtime_variables"): - try: - config_class = entrypoint.OVERRIDE_CLASSES[os_info] - except KeyError: - config_class = configurator.ApacheConfigurator - config = config_class(config=mock_le_config, name="apache", - version=version) - - config.prepare() + with mock.patch("certbot_apache.configurator.util.run_script"): + with mock.patch("certbot_apache.configurator.util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + with mock.patch("certbot_apache.parser.ApacheParser." + "update_runtime_variables"): + try: + config_class = entrypoint.OVERRIDE_CLASSES[os_info] + except KeyError: + config_class = configurator.ApacheConfigurator + config = config_class(config=mock_le_config, name="apache", + version=version) + if not conf_vhost_path: + config_class.OS_DEFAULTS["vhost_root"] = vhost_path + else: + # Custom virtualhost path was requested + config.config.apache_vhost_root = conf_vhost_path + config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.prepare() return config diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 1d2cfdeca..82195264b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy): setattr(self.le_config, "apache_" + k, entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) - # An alias - self.le_config.apache_handle_modules = self.le_config.apache_handle_mods - self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), name="apache") From 4989668e0aaec8ff7d03661846d3340ca6b245af Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 3 Aug 2018 17:55:12 -0400 Subject: [PATCH 327/639] Clarify that configurations can be invalid (#6277) --- certbot/cert_manager.py | 2 +- certbot/renewal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d1205835a..771ca8caf 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -353,7 +353,7 @@ def _describe_certs(config, parsed_certs, parse_failures): notify("Found the following {0}certs:".format(match)) notify(_report_human_readable(config, parsed_certs)) if parse_failures: - notify("\nThe following renewal configuration files " + notify("\nThe following renewal configurations " "were invalid:") notify(_report_lines(parse_failures)) diff --git a/certbot/renewal.py b/certbot/renewal.py index f50131028..ecc8b1f2f 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -359,7 +359,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, notify_error(report(renew_failures, "failure")) if parse_failures: - notify("\nAdditionally, the following renewal configuration files " + notify("\nAdditionally, the following renewal configurations " "were invalid: ") notify(report(parse_failures, "parsefail")) From d8057f0e17dc757fae662dad91a6fedc96ad6a2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 6 Aug 2018 09:45:56 -0700 Subject: [PATCH 328/639] Fix Sphinx (#6070) Fixes #4686. In Sphinx 1.6, they changed how they handle images in latex and PDF files. You can learn more about this by reading the linked issue (or I can answer any questions), but the shortish version is we now need to use the extension sphinx.ext.imgconverter. This is only available in Sphinx 1.6+. I also updated our pinned versions to use the latest Sphinx and a new dependency it pulled in called sphinxcontrib-websupport. To build the latex and PDF docs, you must first run: apt-get install imagemagick latexmk texlive texlive-latex-extra Afterwards, if you create the normal Certbot dev environment using this branch, activate the virtual environment, and from the root of the repo run make -C docs clean latex latexpdf, you'll successfully build the PDF docs. * fix #4686 * bump minimum Sphinx req --- docs/conf.py | 1 + setup.py | 4 ++-- tools/dev_constraints.txt | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 09bb44285..2e6c5a9b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ needs_sphinx = '1.0' # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.imgconverter', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', diff --git a/setup.py b/setup.py index 1827c4d42..a13b7cdb9 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,8 @@ dev3_extras = [ docs_extras = [ 'repoze.sphinx.autointerface', - # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 - 'Sphinx >=1.0,<=1.5.6', + # sphinx.ext.imgconverter + 'Sphinx >=1.6', 'sphinx_rtd_theme', ] diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f14169bfc..ef7804328 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -60,8 +60,9 @@ s3transfer==0.1.11 scandir==1.6 simplegeneric==0.8.1 snowballstemmer==1.2.1 -Sphinx==1.5.6 +Sphinx==1.7.5 sphinx-rtd-theme==0.2.4 +sphinxcontrib-websupport==1.0.1 tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 From b1003b7250fe0b53a683d1a48b732130f0d5aa99 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 16 Aug 2018 21:28:25 +0200 Subject: [PATCH 329/639] Fail fast during tests if python executable is not in the PATH (#6306) --- tests/boulder-integration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2f9130489..fbe8d26aa 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -10,6 +10,9 @@ set -eux +# Check that python executable is available in the PATH. Fail immediatly if not. +command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) + . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx From 0e9dd5e3d21d3907733cac7ba9bdf3f7a27105db Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 21 Aug 2018 08:59:56 -0700 Subject: [PATCH 330/639] Remove visible warnings about missing apachectl. (#6307) --- certbot/plugins/util.py | 4 ++-- certbot/plugins/util_test.py | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..3f03bf375 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -51,6 +51,6 @@ def path_surgery(cmd): return True else: expanded = " expanded" if any(added) else "" - logger.warning("Failed to find executable %s in%s PATH: %s", cmd, - expanded, path) + logger.debug("Failed to find executable %s in%s PATH: %s", cmd, + expanded, path) return False diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..9757d8de7 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -16,9 +16,8 @@ class GetPrefixTest(unittest.TestCase): class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" - @mock.patch("certbot.plugins.util.logger.warning") @mock.patch("certbot.plugins.util.logger.debug") - def test_path_surgery(self, mock_debug, mock_warn): + def test_path_surgery(self, mock_debug): from certbot.plugins.util import path_surgery all_path = {"PATH": "/usr/local/bin:/bin/:/usr/sbin/:/usr/local/sbin/"} with mock.patch.dict('os.environ', all_path): @@ -26,14 +25,12 @@ class PathSurgeryTest(unittest.TestCase): mock_exists.return_value = True self.assertEqual(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) - self.assertEqual(mock_warn.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) no_path = {"PATH": "/tmp/"} with mock.patch.dict('os.environ', no_path): path_surgery("thingy") - self.assertEqual(mock_debug.call_count, 1) - self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("Failed to find" in mock_warn.call_args[0][0]) + self.assertEqual(mock_debug.call_count, 2) + self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) self.assertTrue("/usr/local/bin" in os.environ["PATH"]) self.assertTrue("/tmp" in os.environ["PATH"]) From 6e23b81dba7566267c98e4b946bbc90f755b562d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 29 Aug 2018 14:11:13 -0700 Subject: [PATCH 331/639] Separate integration (#5814) Main piece of #5810. * Rename Certbot integration tests * Remove nginx from certbot tests * allow for running individual integration tests * fail under 65 * Add set -e * Track Nginx coverage and omit it from report later. * Use INTEGRATION_TEST in script * add INTEGRATION_TEST=all * update min certbot percentage --- .travis.yml | 4 +- tests/boulder-integration.sh | 505 +-------------------------- tests/certbot-boulder-integration.sh | 492 ++++++++++++++++++++++++++ 3 files changed, 505 insertions(+), 496 deletions(-) create mode 100755 tests/certbot-boulder-integration.sh diff --git a/.travis.yml b/.travis.yml index b4d702ff1..acdf365bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v2 + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index fbe8d26aa..3e16fcbbc 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -1,499 +1,16 @@ #!/bin/bash -# Simple integration test. Make sure to activate virtualenv beforehand -# (source venv/bin/activate) and that you are running Boulder test -# instance (see ./boulder-fetch.sh). -# -# Environment variables: -# SERVER: Passed as "certbot --server" argument. -# -# Note: this script is called by Boulder integration test suite! -set -eux +set -e -# Check that python executable is available in the PATH. Fail immediatly if not. -command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) - -. ./tests/integration/_common.sh -export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx - -cleanup_and_exit() { - EXIT_STATUS=$? - if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` - then - echo Kill server subprocess, left running by abnormal exit - kill $SERVER_STILL_RUNNING - fi - if [ -f "$HOOK_DIRS_TEST" ]; then - rm -f "$HOOK_DIRS_TEST" - fi - exit $EXIT_STATUS -} - -trap cleanup_and_exit EXIT - -export HOOK_DIRS_TEST="$(mktemp)" -renewal_hooks_root="$config_dir/renewal-hooks" -renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) -renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" -renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" -renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" - -# Creates hooks in Certbot's renewal hook directory that write to a file -CreateDirHooks() { - for hook_dir in $renewal_hooks_dirs; do - mkdir -p $hook_dir - hook_path="$hook_dir/hook.sh" - cat << EOF > "$hook_path" -#!/bin/bash -xe -if [ "\$0" = "$renewal_dir_deploy_hook" ]; then - if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then - echo "Environment variables not properly set!" >&2 - exit 1 +if [ "$INTEGRATION_TEST" = "certbot" ]; then + tests/certbot-boulder-integration.sh +elif [ "$INTEGRATION_TEST" = "nginx" ]; then + certbot-nginx/tests/boulder-integration.sh +else + tests/certbot-boulder-integration.sh + # Most CI systems set this variable to true. + # If the tests are running as part of CI, Nginx should be available. + if ${CI:-false} || type nginx; then + certbot-nginx/tests/boulder-integration.sh fi fi -echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" -EOF - chmod +x "$hook_path" - done -} - -# Asserts that the hooks created by CreateDirHooks have been run once and -# resets the file. -# -# Arguments: -# The number of times the deploy hook should have been run. (It should run -# once for each certificate that was issued in that run of Certbot.) -CheckDirHooks() { - expected="pre\n" - for ((i=0; i<$1; i++)); do - expected=$expected"deploy\n" - done - expected=$expected"post" - - if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then - echo "Unexpected directory hook output!" >&2 - echo "Expected:" >&2 - echo -e "$expected" >&2 - echo "Got:" >&2 - cat "$HOOK_DIRS_TEST" >&2 - exit 1 - fi - - rm -f "$HOOK_DIRS_TEST" - export HOOK_DIRS_TEST="$(mktemp)" -} - -common_no_force_renew() { - certbot_test_no_force_renew \ - --authenticator standalone \ - --installer null \ - "$@" -} - -common() { - common_no_force_renew \ - --renew-by-default \ - "$@" -} - -export HOOK_TEST="/tmp/hook$$" -CheckHooks() { - if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then - expected="wtf.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf2.pre\n" - else - expected=$expected"wtf2.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" - else - expected="wtf2.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf.pre\n" - else - expected=$expected"wtf.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" - fi - - if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then - echo Hooks did not run as expected\; got >&2 - cat "$HOOK_TEST" >&2 - echo -e "Expected\n$expected" >&2 - rm "$HOOK_TEST" - exit 1 - fi - rm "$HOOK_TEST" -} - -# Checks if deploy is in the hook output and deletes the file -DeployInHookOutput() { - CONTENTS=$(cat "$HOOK_TEST") - rm "$HOOK_TEST" - grep deploy <(echo "$CONTENTS") -} - -# Asserts that there is a saved renew_hook for a lineage. -# -# Arguments: -# Name of lineage to check -CheckSavedRenewHook() { - if ! grep renew_hook "$config_dir/renewal/$1.conf"; then - echo "Hook wasn't saved as renew_hook" >&2 - exit 1 - fi -} - -# Asserts the deploy hook was properly run and saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -CheckDeployHook() { - if ! DeployInHookOutput; then - echo "The deploy hook wasn't run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Asserts the renew hook wasn't run but was saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -# Asserts the deploy hook wasn't run and deletes the hook file -CheckRenewHook() { - if DeployInHookOutput; then - echo "The renew hook was incorrectly run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Return success only if input contains exactly $1 lines of text, of -# which $2 different values occur in the first field. -TotalAndDistinctLines() { - total=$1 - distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' -} - -# Cleanup coverage data -coverage erase - -# test for regressions of #4719 -get_num_tmp_files() { - ls -1 /tmp | wc -l -} -num_tmp_files=$(get_num_tmp_files) -common --csr / && echo expected error && exit 1 || true -common --help -common --help all -common --version -if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then - echo "New files or directories created in /tmp!" - exit 1 -fi -CreateDirHooks - -common register -for dir in $renewal_hooks_dirs; do - if [ ! -d "$dir" ]; then - echo "Hook directory not created by Certbot!" >&2 - exit 1 - fi -done - -common unregister - -common register --email ex1@domain.org,ex2@domain.org - -common register --update-registration --email ex1@domain.org - -common register --update-registration --email ex1@domain.org,ex2@domain.org - -common plugins --init --prepare | grep webroot - -# We start a server listening on the port for the -# unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & -python_server_pid=$! - -certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! -certname="le2.wtf" -common --domains le2.wtf --preferred-challenges http-01 run \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -certname="le.wtf" -common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ - --manual-auth-hook ./tests/manual-http-auth.sh \ - --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ - --cert-name $certname \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -common certonly --cert-name newname -d newname.le.wtf - -export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ - OPENSSL_CNF=examples/openssl.cnf -./examples/generate-csr.sh le3.wtf -common auth --csr "$CSR_PATH" \ - --cert-path "${root}/csr/cert.pem" \ - --chain-path "${root}/csr/chain.pem" -openssl x509 -in "${root}/csr/cert.pem" -text -openssl x509 -in "${root}/csr/chain.pem" -text - -common --domains le3.wtf install \ - --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/key.pem" - -CheckCertCount() { - CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` - if [ "$CERTCOUNT" -ne "$2" ] ; then - echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` - exit 1 - fi -} - -CheckCertCount "le.wtf" 1 -# This won't renew (because it's not time yet) -common_no_force_renew renew -CheckCertCount "le.wtf" 1 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed for non-renewal!" >&2; - exit 1 -fi - -rm -rf "$renewal_hooks_root" -# renew using HTTP manual auth hooks -common renew --cert-name le.wtf --authenticator manual -CheckCertCount "le.wtf" 2 - -# test renewal with no executables in hook directories -for hook_dir in $renewal_hooks_dirs; do - touch "$hook_dir/file" - mkdir "$hook_dir/dir" -done -# renew using DNS manual auth hooks -common renew --cert-name dns.le.wtf --authenticator manual -CheckCertCount "dns.le.wtf" 2 - -# test with disabled directory hooks -rm -rf "$renewal_hooks_root" -CreateDirHooks -# This will renew because the expiry is less than 10 years from now -sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" -common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks -CheckCertCount "le.wtf" 3 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed with --no-directory-hooks!" >&2 - exit 1 -fi - -# The 4096 bit setting should persist to the first renewal, but be overridden in the second - -size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` -size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` -size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` -# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes -if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then - echo key sizes violate assumptions: - ls -l "${root}/conf/archive/le.wtf/privkey"* - exit 1 -fi - -# --renew-by-default is used, so renewal should occur -[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" -common renew -CheckCertCount "le.wtf" 4 -CheckHooks -CheckDirHooks 5 - -# test with overlapping directory hooks on the command line -common renew --cert-name le2.wtf \ - --pre-hook "$renewal_dir_pre_hook" \ - --deploy-hook "$renewal_dir_deploy_hook" \ - --post-hook "$renewal_dir_post_hook" -CheckDirHooks 1 - -# test with overlapping directory hooks in the renewal conf files -common renew --cert-name le2.wtf -CheckDirHooks 1 - -# manual-dns-auth.sh will skip completing the challenge for domains that begin -# with fail. -common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ - --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh - -if common certificates | grep "fail\.dns1\.le\.wtf"; then - echo "certificate should not have been issued for domain!" >&2 - exit 1 -fi - -# reuse-key -common --domains reusekey.le.wtf --reuse-key -common renew --cert-name reusekey.le.wtf -CheckCertCount "reusekey.le.wtf" 2 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# The final awk command here exits successfully if its input consists of -# exactly two lines with identical first fields, and unsuccessfully otherwise. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 - -# don't reuse key (just by forcing reissuance without --reuse-key) -common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal -CheckCertCount "reusekey.le.wtf" 3 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# Exactly three lines, of which exactly two identical first fields. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 - -# Nonetheless, all three certificates are different even though two of them -# share the same subject key. -sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 - -# ECDSA -openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" -SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ - -config "${OPENSSL_CNF:-openssl.cnf}" \ - -key "${root}/privkey-p384.pem" \ - -subj "/" \ - -reqexts san \ - -outform der \ - -out "${root}/csr-p384.der" -common auth --csr "${root}/csr-p384.der" \ - --cert-path "${root}/csr/cert-p384.pem" \ - --chain-path "${root}/csr/chain-p384.pem" -openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' - -# OCSP Must Staple -common auth --must-staple --domains "must-staple.le.wtf" -openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' - -# revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke -# revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke -if [ ! -d "$root/conf/live/le1.wtf" ]; then - echo "cert deleted when --no-delete-after-revoke was used!" - exit 1 -fi -common delete --cert-name le1.wtf -# revoke by cert key -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" - -# Get new certs to test revoke with a reason, by account and by cert key -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ - --reason cessationOfOperation -common --domains le2.wtf -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" \ - --reason keyCompromise - -common unregister - -out=$(common certificates) -subdomains="le dns.le newname.le must-staple.le" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if ! echo $out | grep "$domain"; then - echo "$domain not in certificates output!" - exit 1; - fi -done - -# Testing that revocation also deletes by default -subdomains="le1 le2" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if echo $out | grep "$domain"; then - echo "Revoked $domain in certificates output! Should not be!" - exit 1; - fi -done - -# Test that revocation raises correct error if --cert-name and --cert-path don't match -common --domains le1.wtf -common --domains le2.wtf -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true -if ! echo $out | grep "or both must point to the same certificate lineages."; then - echo "Non-interactive revoking with mismatched --cert-name and --cert-path " - echo "did not raise the correct error!" - exit 1 -fi - -# Revoking by matching --cert-name and --cert-path deletes -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" -out=$(common certificates) -if echo $out | grep "le1.wtf"; then - echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" - exit 1 -fi - -# Test that revocation doesn't delete if multiple lineages share an archive dir -common --domains le1.wtf -common --domains le2.wtf -sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" -#common update_symlinks # not needed, but a bit more context for what this test is about -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") -if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then - echo "Deleted a cert that had an overlapping archive dir with another lineage!" - exit 1 -fi - -cert_name="must-staple.le.wtf" -common delete --cert-name $cert_name -archive="$root/conf/archive/$cert_name" -conf="$root/conf/renewal/$cert_name.conf" -live="$root/conf/live/$cert_name" -for path in $archive $conf $live; do - if [ -e $path ]; then - echo "Lineage not properly deleted!" - exit 1 - fi -done - -# Test ACMEv2-only features -if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then - common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh -fi - -coverage report --fail-under 65 --include 'certbot/*' --show-missing - -# Most CI systems set this variable to true. -# If the tests are running as part of CI, Nginx should be available. -if ${CI:-false} || type nginx; -then - . ./certbot-nginx/tests/boulder-integration.sh -fi diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh new file mode 100755 index 000000000..8b8b931e5 --- /dev/null +++ b/tests/certbot-boulder-integration.sh @@ -0,0 +1,492 @@ +#!/bin/bash +# Simple integration test. Make sure to activate virtualenv beforehand +# (source venv/bin/activate) and that you are running Boulder test +# instance (see ./boulder-fetch.sh). +# +# Environment variables: +# SERVER: Passed as "certbot --server" argument. +# +# Note: this script is called by Boulder integration test suite! + +set -eux + +# Check that python executable is available in the PATH. Fail immediatly if not. +command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) + +. ./tests/integration/_common.sh +export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx + +cleanup_and_exit() { + EXIT_STATUS=$? + if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` + then + echo Kill server subprocess, left running by abnormal exit + kill $SERVER_STILL_RUNNING + fi + if [ -f "$HOOK_DIRS_TEST" ]; then + rm -f "$HOOK_DIRS_TEST" + fi + exit $EXIT_STATUS +} + +trap cleanup_and_exit EXIT + +export HOOK_DIRS_TEST="$(mktemp)" +renewal_hooks_root="$config_dir/renewal-hooks" +renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) +renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" +renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" +renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" + +# Creates hooks in Certbot's renewal hook directory that write to a file +CreateDirHooks() { + for hook_dir in $renewal_hooks_dirs; do + mkdir -p $hook_dir + hook_path="$hook_dir/hook.sh" + cat << EOF > "$hook_path" +#!/bin/bash -xe +if [ "\$0" = "$renewal_dir_deploy_hook" ]; then + if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then + echo "Environment variables not properly set!" >&2 + exit 1 + fi +fi +echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" +EOF + chmod +x "$hook_path" + done +} + +# Asserts that the hooks created by CreateDirHooks have been run once and +# resets the file. +# +# Arguments: +# The number of times the deploy hook should have been run. (It should run +# once for each certificate that was issued in that run of Certbot.) +CheckDirHooks() { + expected="pre\n" + for ((i=0; i<$1; i++)); do + expected=$expected"deploy\n" + done + expected=$expected"post" + + if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then + echo "Unexpected directory hook output!" >&2 + echo "Expected:" >&2 + echo -e "$expected" >&2 + echo "Got:" >&2 + cat "$HOOK_DIRS_TEST" >&2 + exit 1 + fi + + rm -f "$HOOK_DIRS_TEST" + export HOOK_DIRS_TEST="$(mktemp)" +} + +common_no_force_renew() { + certbot_test_no_force_renew \ + --authenticator standalone \ + --installer null \ + "$@" +} + +common() { + common_no_force_renew \ + --renew-by-default \ + "$@" +} + +export HOOK_TEST="/tmp/hook$$" +CheckHooks() { + if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then + expected="wtf.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf2.pre\n" + else + expected=$expected"wtf2.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" + else + expected="wtf2.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf.pre\n" + else + expected=$expected"wtf.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" + fi + + if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then + echo Hooks did not run as expected\; got >&2 + cat "$HOOK_TEST" >&2 + echo -e "Expected\n$expected" >&2 + rm "$HOOK_TEST" + exit 1 + fi + rm "$HOOK_TEST" +} + +# Checks if deploy is in the hook output and deletes the file +DeployInHookOutput() { + CONTENTS=$(cat "$HOOK_TEST") + rm "$HOOK_TEST" + grep deploy <(echo "$CONTENTS") +} + +# Asserts that there is a saved renew_hook for a lineage. +# +# Arguments: +# Name of lineage to check +CheckSavedRenewHook() { + if ! grep renew_hook "$config_dir/renewal/$1.conf"; then + echo "Hook wasn't saved as renew_hook" >&2 + exit 1 + fi +} + +# Asserts the deploy hook was properly run and saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +CheckDeployHook() { + if ! DeployInHookOutput; then + echo "The deploy hook wasn't run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Asserts the renew hook wasn't run but was saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +# Asserts the deploy hook wasn't run and deletes the hook file +CheckRenewHook() { + if DeployInHookOutput; then + echo "The renew hook was incorrectly run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + +# Cleanup coverage data +coverage erase + +# test for regressions of #4719 +get_num_tmp_files() { + ls -1 /tmp | wc -l +} +num_tmp_files=$(get_num_tmp_files) +common --csr / && echo expected error && exit 1 || true +common --help +common --help all +common --version +if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then + echo "New files or directories created in /tmp!" + exit 1 +fi +CreateDirHooks + +common register +for dir in $renewal_hooks_dirs; do + if [ ! -d "$dir" ]; then + echo "Hook directory not created by Certbot!" >&2 + exit 1 + fi +done + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org + +common plugins --init --prepare | grep webroot + +# We start a server listening on the port for the +# unrequested challenge to prevent regressions in #3601. +python ./tests/run_http_server.py $http_01_port & +python_server_pid=$! + +certname="le1.wtf" +common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +python ./tests/run_http_server.py $tls_sni_01_port & +python_server_pid=$! +certname="le2.wtf" +common --domains le2.wtf --preferred-challenges http-01 run \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +certname="le.wtf" +common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ + --manual-auth-hook ./tests/manual-http-auth.sh \ + --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +certname="dns.le.wtf" +common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ + --cert-name $certname \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +common certonly --cert-name newname -d newname.le.wtf + +export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ + OPENSSL_CNF=examples/openssl.cnf +./examples/generate-csr.sh le3.wtf +common auth --csr "$CSR_PATH" \ + --cert-path "${root}/csr/cert.pem" \ + --chain-path "${root}/csr/chain.pem" +openssl x509 -in "${root}/csr/cert.pem" -text +openssl x509 -in "${root}/csr/chain.pem" -text + +common --domains le3.wtf install \ + --cert-path "${root}/csr/cert.pem" \ + --key-path "${root}/key.pem" + +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` + if [ "$CERTCOUNT" -ne "$2" ] ; then + echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` + exit 1 + fi +} + +CheckCertCount "le.wtf" 1 +# This won't renew (because it's not time yet) +common_no_force_renew renew +CheckCertCount "le.wtf" 1 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed for non-renewal!" >&2; + exit 1 +fi + +rm -rf "$renewal_hooks_root" +# renew using HTTP manual auth hooks +common renew --cert-name le.wtf --authenticator manual +CheckCertCount "le.wtf" 2 + +# test renewal with no executables in hook directories +for hook_dir in $renewal_hooks_dirs; do + touch "$hook_dir/file" + mkdir "$hook_dir/dir" +done +# renew using DNS manual auth hooks +common renew --cert-name dns.le.wtf --authenticator manual +CheckCertCount "dns.le.wtf" 2 + +# test with disabled directory hooks +rm -rf "$renewal_hooks_root" +CreateDirHooks +# This will renew because the expiry is less than 10 years from now +sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" +common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks +CheckCertCount "le.wtf" 3 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed with --no-directory-hooks!" >&2 + exit 1 +fi + +# The 4096 bit setting should persist to the first renewal, but be overridden in the second + +size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes +if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then + echo key sizes violate assumptions: + ls -l "${root}/conf/archive/le.wtf/privkey"* + exit 1 +fi + +# --renew-by-default is used, so renewal should occur +[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" +common renew +CheckCertCount "le.wtf" 4 +CheckHooks +CheckDirHooks 5 + +# test with overlapping directory hooks on the command line +common renew --cert-name le2.wtf \ + --pre-hook "$renewal_dir_pre_hook" \ + --deploy-hook "$renewal_dir_deploy_hook" \ + --post-hook "$renewal_dir_post_hook" +CheckDirHooks 1 + +# test with overlapping directory hooks in the renewal conf files +common renew --cert-name le2.wtf +CheckDirHooks 1 + +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + +# ECDSA +openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" +SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ + -config "${OPENSSL_CNF:-openssl.cnf}" \ + -key "${root}/privkey-p384.pem" \ + -subj "/" \ + -reqexts san \ + -outform der \ + -out "${root}/csr-p384.der" +common auth --csr "${root}/csr-p384.der" \ + --cert-path "${root}/csr/cert-p384.pem" \ + --chain-path "${root}/csr/chain-p384.pem" +openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' + +# OCSP Must Staple +common auth --must-staple --domains "must-staple.le.wtf" +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' + +# revoke by account key +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke +# revoke renewed +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke +if [ ! -d "$root/conf/live/le1.wtf" ]; then + echo "cert deleted when --no-delete-after-revoke was used!" + exit 1 +fi +common delete --cert-name le1.wtf +# revoke by cert key +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" + +# Get new certs to test revoke with a reason, by account and by cert key +common --domains le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ + --reason cessationOfOperation +common --domains le2.wtf +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" \ + --reason keyCompromise + +common unregister + +out=$(common certificates) +subdomains="le dns.le newname.le must-staple.le" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if ! echo $out | grep "$domain"; then + echo "$domain not in certificates output!" + exit 1; + fi +done + +# Testing that revocation also deletes by default +subdomains="le1 le2" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if echo $out | grep "$domain"; then + echo "Revoked $domain in certificates output! Should not be!" + exit 1; + fi +done + +# Test that revocation raises correct error if --cert-name and --cert-path don't match +common --domains le1.wtf +common --domains le2.wtf +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true +if ! echo $out | grep "or both must point to the same certificate lineages."; then + echo "Non-interactive revoking with mismatched --cert-name and --cert-path " + echo "did not raise the correct error!" + exit 1 +fi + +# Revoking by matching --cert-name and --cert-path deletes +common --domains le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" +out=$(common certificates) +if echo $out | grep "le1.wtf"; then + echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" + exit 1 +fi + +# Test that revocation doesn't delete if multiple lineages share an archive dir +common --domains le1.wtf +common --domains le2.wtf +sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" +#common update_symlinks # not needed, but a bit more context for what this test is about +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") +if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then + echo "Deleted a cert that had an overlapping archive dir with another lineage!" + exit 1 +fi + +cert_name="must-staple.le.wtf" +common delete --cert-name $cert_name +archive="$root/conf/archive/$cert_name" +conf="$root/conf/renewal/$cert_name.conf" +live="$root/conf/live/$cert_name" +for path in $archive $conf $live; do + if [ -e $path ]; then + echo "Lineage not properly deleted!" + exit 1 + fi +done + +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh +fi + +coverage report --fail-under 64 --include 'certbot/*' --show-missing From 405a8b426422fbc0a885e04a8b4b36d1b608258f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 29 Aug 2018 15:15:57 -0700 Subject: [PATCH 332/639] Pin the real oldest requirement for nginx tests. (#6327) --- certbot-nginx/local-oldest-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 38ed5debe..bcd02d197 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.26.0 --e .[dev] +certbot[dev]==0.22.0 From cd2edeff1b0f86cc01759e94546b0155f6a8549f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 13:12:06 -0700 Subject: [PATCH 333/639] Fix test farm tests (#6335) * update CentOS AMI ids * Remove assumption of usable default subnet --- tests/letstest/multitester.py | 60 ++++++++++++++++++++++++++++------- tests/letstest/targets.yaml | 4 +-- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 0ae9636d4..320328d9e 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -103,13 +103,32 @@ LOGDIR = "" #points to logging / working directory # boto3/AWS api globals AWS_SESSION = None EC2 = None +SECURITY_GROUP_NAME = 'certbot-security-group' +SUBNET_NAME = 'certbot-subnet' # Boto3/AWS automation functions #------------------------------------------------------------------------------- -def make_security_group(): +def should_use_subnet(subnet): + """Should we use the given subnet for these tests? + + We should if it is the default subnet for the availability zone or the + subnet is named "certbot-subnet". + + """ + if not subnet.map_public_ip_on_launch: + return False + if subnet.default_for_az: + return True + for tag in subnet.tags: + if tag['Key'] == 'Name' and tag['Value'] == SUBNET_NAME: + return True + return False + +def make_security_group(vpc): + """Creates a security group in the given VPC.""" # will fail if security group of GroupName already exists # cannot have duplicate SGs of the same name - mysg = EC2.create_security_group(GroupName="letsencrypt_test", + mysg = vpc.create_security_group(GroupName=SECURITY_GROUP_NAME, Description='security group for automated testing') mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) @@ -123,14 +142,16 @@ def make_security_group(): def make_instance(instance_name, ami_id, keyname, + security_group_id, + subnet_id, machine_type='t2.micro', - security_groups=['letsencrypt_test'], userdata=""): #userdata contains bash or cloud-init script new_instance = EC2.create_instances( BlockDeviceMappings=_get_block_device_mappings(ami_id), ImageId=ami_id, - SecurityGroups=security_groups, + SecurityGroupIds=[security_group_id], + SubnetId=subnet_id, KeyName=keyname, MinCount=1, MaxCount=1, @@ -294,7 +315,7 @@ def grab_certbot_log(): sudo('if [ -f ./certbot.log ]; then \ cat ./certbot.log; else echo "[nolocallog]"; fi') -def create_client_instances(targetlist): +def create_client_instances(targetlist, security_group_id, subnet_id): "Create a fleet of client instances" instances = [] print("Creating instances: ", end="") @@ -314,6 +335,8 @@ def create_client_instances(targetlist): target['ami'], KEYNAME, machine_type=machine_type, + security_group_id=security_group_id, + subnet_id=subnet_id, userdata=userdata)) print() return instances @@ -418,14 +441,28 @@ print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, AWS_SESSION = boto3.session.Session(profile_name=PROFILE) EC2 = AWS_SESSION.resource('ec2') +print("Determining Subnet") +for subnet in EC2.subnets.all(): + if should_use_subnet(subnet): + subnet_id = subnet.id + vpc_id = subnet.vpc.id + break +else: + print("No usable subnet exists!") + print("Please create a VPC with a subnet named {0}".format(SUBNET_NAME)) + print("that maps public IPv4 addresses to instances launched in the subnet.") + sys.exit(1) + print("Making Security Group") +vpc = EC2.Vpc(vpc_id) sg_exists = False -for sg in EC2.security_groups.all(): - if sg.group_name == 'letsencrypt_test': +for sg in vpc.security_groups.all(): + if sg.group_name == SECURITY_GROUP_NAME: + security_group_id = sg.id sg_exists = True - print(" %s already exists"%'letsencrypt_test') + print(" %s already exists"%SECURITY_GROUP_NAME) if not sg_exists: - make_security_group() + security_group_id = make_security_group(vpc).id time.sleep(30) boulder_preexists = False @@ -446,11 +483,12 @@ else: KEYNAME, machine_type='t2.micro', #machine_type='t2.medium', - security_groups=['letsencrypt_test']) + security_group_id=security_group_id, + subnet_id=subnet_id) try: if not cl_args.boulderonly: - instances = create_client_instances(targetlist) + instances = create_client_instances(targetlist, security_group_id, subnet_id) # Configure and launch boulder server #------------------------------------------------------------------------------- diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 766b4ea09..57ce4811a 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -48,13 +48,13 @@ targets: # CentOS # These Marketplace AMIs must, irritatingly, have their terms manually # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-61bbf104 + - ami: ami-9887c6e7 name: centos7 type: centos virt: hvm user: centos # centos6 requires EPEL repo added - - ami: ami-57cd8732 + - ami: ami-1585c46a name: centos6 type: centos virt: hvm From e178bbfdf54e7ac5160de7a4656c6c19fa8ee4d2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 14:10:05 -0700 Subject: [PATCH 334/639] Release script improvements (#6337) * Add error checking and automatic logging. * Ignore release dir and logs * Don't always require PGP card and fix script cmd. * keep track of default GPG key * Add PGP card sanity check after offline signature * fix typo * I'm tired of pressing y. * Automate running tools/offline-sigrequest.sh. * Update comment and make output more readable. --- .gitignore | 3 +- tools/_release.sh | 247 ++++++++++++++++++++++++++++++++++++++++++++ tools/release.sh | 257 +++++----------------------------------------- 3 files changed, 276 insertions(+), 231 deletions(-) create mode 100755 tools/_release.sh diff --git a/.gitignore b/.gitignore index e744a82a2..9ef645593 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ dist*/ /venv*/ /kgs/ /.tox/ -/releases/ +/releases*/ +/log* letsencrypt.log certbot.log letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 diff --git a/tools/_release.sh b/tools/_release.sh new file mode 100755 index 000000000..ec9bd7461 --- /dev/null +++ b/tools/_release.sh @@ -0,0 +1,247 @@ +#!/bin/bash -xe +# Release packages to PyPI + +if [ "$RELEASE_DIR" = "" ]; then + echo Please run this script through the tools/release.sh wrapper script or set the environment + echo variable RELEASE_DIR to the directory where the release should be built. + exit 1 +fi + +version="$1" +echo Releasing production version "$version"... +nextversion="$2" +RELEASE_BRANCH="candidate-$version" + +if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then + RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" +fi +DEFAULT_GPG_KEY="A2CFB51FA275A7286234E7B24D17C995CD9775F2" +RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-"$DEFAULT_GPG_KEY"} +# Needed to fix problems with git signatures and pinentry +export GPG_TTY=$(tty) + +# port for a local Python Package Index (used in testing) +PORT=${PORT:-1234} + +# subpackages to be released (the way developers think about them) +SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" + +# subpackages to be released (the way the script thinks about them) +SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" +SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" +SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" +# certbot_compatibility_test is not packaged because: +# - it is not meant to be used by anyone else than Certbot devs +# - it causes problems when running pytest - the latter tries to +# run everything that matches test*, while there are no unittests +# there + +tag="v$version" +mv "dist.$version" "dist.$version.$(date +%s).bak" || true +git tag --delete "$tag" || true + +tmpvenv=$(mktemp -d) +virtualenv --no-site-packages -p python2 $tmpvenv +. $tmpvenv/bin/activate +# update setuptools/pip just like in other places in the repo +pip install -U setuptools +pip install -U pip # latest pip => no --pre for dev releases +pip install -U wheel # setup.py bdist_wheel + +# newer versions of virtualenv inherit setuptools/pip/wheel versions +# from current env when creating a child env +pip install -U virtualenv + +root_without_le="$version.$$" +root="$RELEASE_DIR/le.$root_without_le" + +echo "Cloning into fresh copy at $root" # clean repo = no artifacts +git clone . $root +git rev-parse HEAD +cd $root +if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then + git branch -f "$RELEASE_BRANCH" +fi +git checkout "$RELEASE_BRANCH" + +for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +do + sed -i 's/\.dev0//' "$pkg_dir/setup.py" + git add "$pkg_dir/setup.py" +done + +SetVersion() { + ver="$1" + # bumping Certbot's version number is done differently + for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test + do + setup_file="$pkg_dir/setup.py" + if [ $(grep -c '^version' "$setup_file") != 1 ]; then + echo "Unexpected count of version variables in $setup_file" + exit 1 + fi + sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py + done + init_file="certbot/__init__.py" + if [ $(grep -c '^__version' "$init_file") != 1 ]; then + echo "Unexpected count of __version variables in $init_file" + exit 1 + fi + sed -i "s/^__version.*/__version__ = '$ver'/" "$init_file" + + git add $SUBPKGS certbot-compatibility-test +} + +SetVersion "$version" + +echo "Preparing sdists and wheels" +for pkg_dir in . $SUBPKGS_NO_CERTBOT +do + cd $pkg_dir + + python setup.py clean + rm -rf build dist + python setup.py sdist + python setup.py bdist_wheel + + echo "Signing ($pkg_dir)" + for x in dist/*.tar.gz dist/*.whl + do + gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x + done + + cd - +done + + +mkdir "dist.$version" +mv dist "dist.$version/certbot" +for pkg_dir in $SUBPKGS_NO_CERTBOT +do + mv $pkg_dir/dist "dist.$version/$pkg_dir/" +done + +echo "Testing packages" +cd "dist.$version" +# start local PyPI +python -m SimpleHTTPServer $PORT & +# cd .. is NOT done on purpose: we make sure that all subpackages are +# installed from local PyPI rather than current directory (repo root) +virtualenv --no-site-packages ../venv +. ../venv/bin/activate +pip install -U setuptools +pip install -U pip +# Now, use our local PyPI. Disable cache so we get the correct KGS even if we +# (or our dependencies) have conditional dependencies implemented with if +# statements in setup.py and we have cached wheels lying around that would +# cause those ifs to not be evaluated. +pip install \ + --no-cache-dir \ + --extra-index-url http://localhost:$PORT \ + $SUBPKGS +# stop local PyPI +kill $! +cd ~- + +# get a snapshot of the CLI help for the docs +# We set CERTBOT_DOCS to use dummy values in example user-agent string. +CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt +jws --help > acme/docs/jws-help.txt + +cd .. +# freeze before installing anything else, so that we know end-user KGS +# make sure "twine upload" doesn't catch "kgs" +if [ -d kgs ] ; then + echo Deleting old kgs... + rm -rf kgs +fi +mkdir kgs +kgs="kgs/$version" +pip freeze | tee $kgs +pip install pytest +for module in $subpkgs_modules ; do + echo testing $module + pytest --pyargs $module +done +cd ~- + +# pin pip hashes of the things we just built +for pkg in $SUBPKGS_IN_AUTO ; do + echo $pkg==$version \\ + pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' +done > letsencrypt-auto-source/pieces/certbot-requirements.txt +deactivate + +# there should be one requirement specifier and two hashes for each subpackage +expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then + echo Unexpected pip hash output + exit 1 +fi + +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +tools/offline-sigrequest.sh +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + echo "The signature on letsencrypt-auto is not correct." + read -p "Would you like this script to try and sign it again [Y/n]?" response + case $response in + [yY][eE][sS]|[yY]|"") + tools/offline-sigrequest.sh;; + *) + ;; + esac +done + +if [ "$RELEASE_GPG_KEY" = "$DEFAULT_GPG_KEY" ]; then + while ! gpg2 --card-status >/dev/null 2>&1; do + echo gpg cannot find your OpenPGP card + read -p "Please take the card out and put it back in again." + done +fi + +# This signature is not quite as strong, but easier for people to verify out of band +gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto +# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, +# but we can use the right name for certbot-auto.asc from day one +mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc + +# copy leauto to the root, overwriting the previous release version +cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto +cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto + +git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt +git diff --cached +git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" + +cd .. +echo Now in $PWD +name=${root_without_le%.*} +ext="${root_without_le##*.}" +rev="$(git rev-parse --short HEAD)" +echo tar cJvf $name.$rev.tar.xz $name.$rev +echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz +cd ~- + +echo "New root: $root" +echo "Test commands (in the letstest repo):" +echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' +echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' +echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' +echo "In order to upload packages run the following command:" +echo twine upload "$root/dist.$version/*/*" + +if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then + SetVersion "$nextversion".dev0 + letsencrypt-auto-source/build.py + git add letsencrypt-auto-source/letsencrypt-auto + git diff + git commit -m "Bump version to $nextversion" +fi diff --git a/tools/release.sh b/tools/release.sh index 880563b4b..ae3e78dc1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -1,11 +1,5 @@ -#!/bin/bash -xe -# Release dev packages to PyPI - -Usage() { - echo Usage: - echo "$0 [ --production ]" - exit 1 -} +#!/bin/bash -e +# Release packages to PyPI if [ "`dirname $0`" != "tools" ] ; then echo Please run this script from the repo root @@ -13,235 +7,38 @@ if [ "`dirname $0`" != "tools" ] ; then fi CheckVersion() { - # Args: - if ! echo "$2" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then + # Args: + if ! echo "$1" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then echo "$1 doesn't look like 1.2.3" + echo "Usage:" + echo "$0 RELEASE_VERSION NEXT_VERSION" exit 1 fi } -if [ "$1" = "--production" ] ; then - version="$2" - CheckVersion Version "$version" - echo Releasing production version "$version"... - nextversion="$3" - CheckVersion "Next version" "$nextversion" - RELEASE_BRANCH="candidate-$version" -else - version=`grep "__version__" certbot/__init__.py | cut -d\' -f2 | sed s/\.dev0//` - version="$version.dev$(date +%Y%m%d)1" - RELEASE_BRANCH="dev-release" - echo Releasing developer version "$version"... -fi +CheckVersion "$1" +CheckVersion "$2" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi -RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} -# Needed to fix problems with git signatures and pinentry -export GPG_TTY=$(tty) - -# port for a local Python Package Index (used in testing) -PORT=${PORT:-1234} - -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - -# subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" -subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" -# certbot_compatibility_test is not packaged because: -# - it is not meant to be used by anyone else than Certbot devs -# - it causes problems when running pytest - the latter tries to -# run everything that matches test*, while there are no unittests -# there - -tag="v$version" -mv "dist.$version" "dist.$version.$(date +%s).bak" || true -git tag --delete "$tag" || true - -tmpvenv=$(mktemp -d) -virtualenv --no-site-packages -p python2 $tmpvenv -. $tmpvenv/bin/activate -# update setuptools/pip just like in other places in the repo -pip install -U setuptools -pip install -U pip # latest pip => no --pre for dev releases -pip install -U wheel # setup.py bdist_wheel - -# newer versions of virtualenv inherit setuptools/pip/wheel versions -# from current env when creating a child env -pip install -U virtualenv - -root_without_le="$version.$$" -root="./releases/le.$root_without_le" - -echo "Cloning into fresh copy at $root" # clean repo = no artifacts -git clone . $root -git rev-parse HEAD -cd $root -if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then - git branch -f "$RELEASE_BRANCH" -fi -git checkout "$RELEASE_BRANCH" - -for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . -do - sed -i 's/\.dev0//' "$pkg_dir/setup.py" -done -# We only add Certbot's setup.py here because the other files are added in the -# call to SetVersion below. -git add -p setup.py - -SetVersion() { - ver="$1" - # bumping Certbot's version number is done differently - for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test - do - sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py - done - sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py - - # interactive user input - git add -p $SUBPKGS certbot-compatibility-test - -} - -SetVersion "$version" - -echo "Preparing sdists and wheels" -for pkg_dir in . $SUBPKGS_NO_CERTBOT -do - cd $pkg_dir - - python setup.py clean - rm -rf build dist - python setup.py sdist - python setup.py bdist_wheel - - echo "Signing ($pkg_dir)" - for x in dist/*.tar.gz dist/*.whl - do - gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x - done - - cd - -done - - -mkdir "dist.$version" -mv dist "dist.$version/certbot" -for pkg_dir in $SUBPKGS_NO_CERTBOT -do - mv $pkg_dir/dist "dist.$version/$pkg_dir/" -done - -echo "Testing packages" -cd "dist.$version" -# start local PyPI -python -m SimpleHTTPServer $PORT & -# cd .. is NOT done on purpose: we make sure that all subpackages are -# installed from local PyPI rather than current directory (repo root) -virtualenv --no-site-packages ../venv -. ../venv/bin/activate -pip install -U setuptools -pip install -U pip -# Now, use our local PyPI. Disable cache so we get the correct KGS even if we -# (or our dependencies) have conditional dependencies implemented with if -# statements in setup.py and we have cached wheels lying around that would -# cause those ifs to not be evaluated. -pip install \ - --no-cache-dir \ - --extra-index-url http://localhost:$PORT \ - $SUBPKGS -# stop local PyPI -kill $! -cd ~- - -# get a snapshot of the CLI help for the docs -# We set CERTBOT_DOCS to use dummy values in example user-agent string. -CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt -jws --help > acme/docs/jws-help.txt - -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -pip install pytest -for module in $subpkgs_modules ; do - echo testing $module - pytest --pyargs $module -done -cd ~- - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt -deactivate - -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output +if [ "$RELEASE_GPG_KEY" = "" ] && ! gpg2 --card-status >/dev/null 2>&1; then + echo OpenPGP card not found! + echo Please insert your PGP card and run this script again. exit 1 fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# and that it's signed correctly -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" -done - -# This signature is not quite as strong, but easier for people to verify out of band -gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt -git diff --cached -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" - -cd .. -echo Now in $PWD -name=${root_without_le%.*} -ext="${root_without_le##*.}" -rev="$(git rev-parse --short HEAD)" -echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz -cd ~- - -echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' -echo "In order to upload packages run the following command:" -echo twine upload "$root/dist.$version/*/*" - -if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then - SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto - git diff - git commit -m "Bump version to $nextversion" +if ! command -v script >/dev/null 2>&1; then + echo The command script was not found. + echo Please install it. + exit 1 +fi + +export RELEASE_DIR="./releases" +mv "$RELEASE_DIR" "$RELEASE_DIR.$(date +%s).bak" || true +LOG_PATH="log" +mv "$LOG_PATH" "$LOG_PATH.$(date +%s).bak" || true + +# Work with both Linux and macOS versions of script +if script --help | grep -q -- '--command'; then + script --command "tools/_release.sh $1 $2" "$LOG_PATH" +else + script "$LOG_PATH" tools/_release.sh "$1" "$2" fi From 19149a0d578249487e2a137b6a69514fdbd395e8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Sep 2018 15:41:59 -0700 Subject: [PATCH 335/639] Release 0.27.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++---------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 22 +++++++++------- letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/letsencrypt-auto.sig | 5 ++-- .../pieces/certbot-requirements.txt | 24 ++++++++--------- 26 files changed, 92 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 88592013c..7d11449a0 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f435bb1a9..3b5cb7313 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index e097719db..a04bdaf25 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e4ccc719a..5ebfc9f10 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 05649d4d0..fc8fde114 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 911f9e052..9aa9fed33 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9dd318296..24721960b 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 09b11def0..551369f79 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2ca3213bf..ad5ae08af 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index e9ead6546..746040f09 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 3c7402f25..21f9f99ce 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 327224c9c..a241bdcfb 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 1f92f7dce..6860975cc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 0b4241afb..a98419b73 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index e0ce785a1..6342bb11f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bd54ec4c5..f82e1508c 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 5f0b26f6e..2fe473337 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b7cfc15b5..5ebab483a 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 02aaa6581..6cd2af84e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.27.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 3b0b77f6c..d6fee2e83 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.27.0.dev0' +__version__ = '0.27.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 931ea4c62..4fea2d4d1 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.26.1 + "". (default: CertbotACMEClient/0.27.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -475,16 +475,15 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary (default: a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: + Path to the Apache 'a2dismod' binary (default: a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT - SSL vhost configuration extension. (default: -le- + SSL vhost configuration extension (default: -le- ssl.conf) --apache-server-root APACHE_SERVER_ROOT - Apache server root directory. (default: /etc/apache2) + Apache server root directory (default: /etc/apache2) --apache-vhost-root APACHE_VHOST_ROOT Apache server VirtualHost configuration root (default: None) @@ -492,14 +491,17 @@ apache: Apache server logs directory (default: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION - Directory path for challenge configuration. (default: + Directory path for challenge configuration (default: /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES - Let installer handle enabling required modules for - you. (Only Ubuntu/Debian currently) (default: True) + Let installer handle enabling required modules for you + (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES - Let installer handle enabling sites for you. (Only + Let installer handle enabling sites for you (Only Ubuntu/Debian currently) (default: True) + --apache-ctl APACHE_CTL + Full path to Apache control script (default: + apache2ctl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index e097719db..a04bdaf25 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9f6706931..5a6fb41f6 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJbTSv8AAoJEE0XyZXNl3Xy12sH/1FgV3SDVG0T1jgKQOYEUwrq -cmpjdav8YPgFOSQDOcyFZG0DNcRfTskZt45IMkBLLnXq2PuPvkppc1+akP81vMoK -NXHHS+PXDMjnBW4NFkexoM06KRF1SyHnvqsOg13w7UW2CjsAgtazGF5BucNCnjPH -XJTwUf4uhKxeUb0Xkva1OPH++oTWz8+SYgWr/iMggkBrK8y04QUUJ6lyCO6MZgcE -3JcECG7CwMK+hW0gCUkCSNZ0NzOBALCd9wCxNGszgkeJXrrW73oUpZmGC5BxIwYY -o6lcF0qo7Jb92t4B3+7JhulMC5JoVoG4lpiXpKQFFCT0P4pZKotIomKNMATmnB4= -=hzUL +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluQW1IACgkQTRfJlc2X +dfI4zAf+NhtBZkuJuyxOAOiHGwic/Wjdu8g+D2gNlqVD+aUco99mrbyYL5/mB/N6 +q27jav+WjJGK+gCBFrddIbPjJXdrWErf93MmkuVKwpBMyQVwuj9hfWfXnB6K7Kse +8mOOPxWKusq5ykLmujPaYG7Fcfxf5DkmxD/hlwFW7zxOQsIARnfUuW2UST58s6q8 +muG/I92DasOihyRLBxZkd930/CocXl9SJVdC2Aus9zc/2ClQsSPFkPGqC3P9ijC4 +DdU4jJF04LTHtbgXHbN/VIH/Hlu4s++uRyRpoWikbOFwlERrkmsc+1/zi3V2l6a2 +nQQzZE3qwk6fL6NXrJPftcw5Ge1uTA== +=yCtI -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 765072c3f..a04bdaf25 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0.dev0" +LE_AUTO_VERSION="0.27.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index d1306d03d..4de632983 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,3 +1,2 @@ -ÍÅ3pºŽàùSÄj Í»¾´¼—²øö«ø~Ô@F-¢ä*6](Ý"üø²A‹,“8èß…ñ=äsO²cô{â…ŽE•B—\ä,üIJå1Ï -»Â»Zï*¤ƒO/Æ+³k+}¦¥ùƒä«~ld‚?]ŒŽfÚ(r?JÐ2Ô”ò †4=¿ K+K¾è#Ó@š_›[)4ºV8;Õ Ž-øKöWørÝýp…ý¤¤wߊ2»Û®x,g§Ú§w·áê×ù8½0;£uC^Ì“ÚÓB¯§€yIÚÄ Ã¹X/ b£øºìÏ -¸®s‹Z¢ö€¥S \ No newline at end of file +‡ÀÓ•°¦«m’" $`áêÊ$¯Ãí€uÞCÎ? »ß]e鹌N]ûùmCÜT*Tg¦ß3”5Î@{¼ƒÔ5Œ«1AE‰Ù +â,ê¼ßeÉâAÞMçT¡/€¡òÈÐú·ÝÙSKyUعÔln€L.†Ñ‹iT;ÜM°gÁÊ!ЊtfC, UáÙ‚GžÂš¹Îûë]¸V\:X§ýA¢-§AКèänN RÿHL8…`»(çÔŒ?m€GÉ…•Ví}•i’ààÿ¢XícÎcÑÆšª‰Š‚ÙˆqëRÎÎIð=©/µs-»>ZI²à#ÑNw( §òº¦”.¼Jey \ No newline at end of file diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index feb3f1c3a..273649da9 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.0 \ + --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ + --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 +acme==0.27.0 \ + --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ + --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 +certbot-apache==0.27.0 \ + --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ + --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca +certbot-nginx==0.27.0 \ + --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ + --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb From e28f3da9749643b41e5ab564829629c6f6e66ae9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Sep 2018 15:42:01 -0700 Subject: [PATCH 336/639] Bump version to 0.28.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7d11449a0..85492d9a3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3b5cb7313..aa5908f9e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 5ebfc9f10..8dac3e047 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index fc8fde114..b823cf98f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9aa9fed33..3daf933cb 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 24721960b..feb2a0d58 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 551369f79..b6efcf360 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ad5ae08af..c268eaa8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 746040f09..fc147f85c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 21f9f99ce..86d36bcb3 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index a241bdcfb..1d196403f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 6860975cc..a5c06d90e 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a98419b73..474093a5b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 6342bb11f..15fe3d6d7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index f82e1508c..c009ef032 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2fe473337..2bae0c3d0 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 5ebab483a..9f8bfbbdb 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6cd2af84e..3c8a66ee5 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index d6fee2e83..ab23926c9 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.27.0' +__version__ = '0.28.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index a04bdaf25..4b55022db 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.28.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 2708d28157622f62a19efc53d4fc1fb6530d06b1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Sep 2018 17:13:30 -0700 Subject: [PATCH 337/639] Update changelog for 0.27.0 (#6338) --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d384ad30..6ab875941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.27.0 - 2018-09-05 + +### Added + +* The Apache plugin now accepts the parameter --apache-ctl which can be + used to configure the path to the Apache control script. + +### Changed + +* When using `acme.client.ClientV2` (or + `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a + newer version of the ACME protocol), an `acme.errors.ConflictError` will be + raised if you try to create an ACME account with a key that has already been + used. Previously, a JSON parsing error was raised in this scenario when using + the library with Let's Encrypt's ACMEv2 endpoint. + +### Fixed + +* When Apache is not installed, Certbot's Apache plugin no longer prints + messages about being unable to find apachectl to the terminal when the plugin + is not selected. +* If you're using the Apache plugin with the --apache-vhost-root flag set to a + directory containing a disabled virtual host for the domain you're requesting + a certificate for, the virtual host will now be temporarily enabled if + necessary to pass the HTTP challenge. +* The documentation for the Certbot package can now be built using Sphinx 1.6+. +* You can now call `query_registration` without having to first call + `new_account` on `acme.client.ClientV2` objects. +* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`. +* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura + Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-ovh +* certbot-dns-sakuracloud + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/57?closed=1 + ## 0.26.1 - 2018-07-17 ### Fixed From 0c66de47cf38b83b5a11804c011e54bfca309dd8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 5 Sep 2018 18:05:42 -0700 Subject: [PATCH 338/639] Remind people to modify changelog when submitting PRs (#6341) * Remind people to modify changelog when submitting PRs * Update pull_request_template.md --- pull_request_template.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 pull_request_template.md diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 000000000..c071d4135 --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1 @@ +Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. From 05ad5392551f027a365c7ad70d7d3ab901d8f4a7 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 5 Sep 2018 18:05:48 -0700 Subject: [PATCH 339/639] git ignore pytest cache (#6340) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9ef645593..54545e883 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ tests/letstest/venv/ # pytest cache .cache .mypy_cache/ +.pytest_cache/ # docker files .docker From d39a354a659cbc3ee7ff177ea6bd42724b16c71e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 10:17:51 -0700 Subject: [PATCH 340/639] Create master section for incremental changes (#6342) --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ab875941..561ee49a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.28.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + ## 0.27.0 - 2018-09-05 ### Added From 4e2faffe8936004429c7ee4622e788671dd6afe2 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 6 Sep 2018 15:00:20 -0700 Subject: [PATCH 341/639] fix(apache): s/handle_mods/handle_modules (#6347) fixes #6344 * fix(apache): s/handle_mods/handle_modules * test(apache): ensure all keys defined in OS_DEFAULTS overrides * changelog udpate --- CHANGELOG.md | 3 ++- certbot-apache/certbot_apache/override_suse.py | 2 +- .../certbot_apache/tests/configurator_test.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 561ee49a6..8d2ff6323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* +* Fixed parameter name in OpenSUSE overrides for default parameters in the + Apache plugin. Certbot on OpenSUSE works again. ## 0.27.0 - 2018-09-05 diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py index 83079b92c..3d0043afe 100644 --- a/certbot-apache/certbot_apache/override_suse.py +++ b/certbot-apache/certbot_apache/override_suse.py @@ -23,7 +23,7 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 6f1c358c2..0fb89c95a 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,6 +115,18 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use + from certbot_apache.entrypoint import OVERRIDE_CLASSES + for cls in OVERRIDE_CLASSES.values(): + cls.add_parser_arguments(mock.MagicMock()) + + def test_all_configurators_defaults_defined(self): + from certbot_apache.entrypoint import OVERRIDE_CLASSES + from certbot_apache.configurator import ApacheConfigurator + parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + for cls in OVERRIDE_CLASSES.values(): + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + def test_constant(self): self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in self.config.option("server_root")) From 101eae4e05022c058bb7e9bd46bdbea6e3f573f3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 17:21:31 -0700 Subject: [PATCH 342/639] Update CHANGELOG.md for 0.27.1 release (#6350) --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d2ff6323..6d80fe730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,23 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed +* + +## 0.27.1 - 2018-09-06 + +### Fixed + * Fixed parameter name in OpenSUSE overrides for default parameters in the Apache plugin. Certbot on OpenSUSE works again. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* certbot-apache + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/60?closed=1 ## 0.27.0 - 2018-09-05 From b50abddb5f144d0570ce57dfa87804e781037175 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 6 Sep 2018 17:49:24 -0700 Subject: [PATCH 343/639] Candidate 0.27.1 (#6351) * fix(apache): s/handle_mods/handle_modules (#6347) (#6349) fixes #6344 * fix(apache): s/handle_mods/handle_modules * test(apache): ensure all keys defined in OS_DEFAULTS overrides * changelog udpate (cherry picked from commit 4e2faffe8936004429c7ee4622e788671dd6afe2) * Release 0.27.1 * Bump version to 0.28.0 --- certbot-auto | 26 +++++++++--------- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/certbot-auto b/certbot-auto index a04bdaf25..076c45e39 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4fea2d4d1..4ed9f0731 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.0 + "". (default: CertbotACMEClient/0.27.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the diff --git a/letsencrypt-auto b/letsencrypt-auto index a04bdaf25..076c45e39 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 5a6fb41f6..747d98e2d 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluQW1IACgkQTRfJlc2X -dfI4zAf+NhtBZkuJuyxOAOiHGwic/Wjdu8g+D2gNlqVD+aUco99mrbyYL5/mB/N6 -q27jav+WjJGK+gCBFrddIbPjJXdrWErf93MmkuVKwpBMyQVwuj9hfWfXnB6K7Kse -8mOOPxWKusq5ykLmujPaYG7Fcfxf5DkmxD/hlwFW7zxOQsIARnfUuW2UST58s6q8 -muG/I92DasOihyRLBxZkd930/CocXl9SJVdC2Aus9zc/2ClQsSPFkPGqC3P9ijC4 -DdU4jJF04LTHtbgXHbN/VIH/Hlu4s++uRyRpoWikbOFwlERrkmsc+1/zi3V2l6a2 -nQQzZE3qwk6fL6NXrJPftcw5Ge1uTA== -=yCtI +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X +dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ +nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn +ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa +I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk +AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm +8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== +=QRnC -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 4b55022db..09e728aa4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4de632983e5a48c69b74f50d28f12b787fe3d374..b717e359b2a2c65f6cb588a007f5024572c1e4ff 100644 GIT binary patch literal 256 zcmV+b0ssEcvxB0LY3lA!OPssV3H<~y?AaAqjuWWrmMIj>00(?0Ek&mFUj?a=WHkII z-RNTHeD@0(kwW#&HKEPltP&F}Il@J^#^yCaE=%~OTz&S0VZ;xAiFO*AjYgLt@?f^0 z5%fBEr@1)1d9JccQ6BE>@C=b&m*{iB+J6X1CVdnJgQ4FUmIMh?O(XLrLz zjb^9RY>s^Yo_H0;@vfC^I1)~)4x~+{{pTYk(%Y}AY2=R!zDwo_S`Vp=%04-BF^8$F zx6n%y(2yohnk6h!P4Oc-;Nb&}2~R5^lWYM-Sp*8Vav}^+f?3H>xwNIK(r-rHpKXnf GZUdUtC3&a- literal 256 zcmV+b0ssDoz|)nmrmJm|B7YzxVBzY@B(KBmfOXzO&Oac#-(6+txr|O-`}u7{+*B%5 zXQtmXln6D>KzqD{)HNH7tA8;;21N&n*$Uz;>b&1&$>KrYP3KghFMy%)$QvBc`nTQL zQ%iYO*truD)NF2mOo1+jjepULX;eGhO|WOd$|2B-bY?>=43Aae*@8!&!kW3x`|Ac> zxK>;`Sf~9#qAjOE(3R_%S2X_6h_;QtSz zSnXrZW6{Q%s)>q%*@$uLQXI|#&Pnh+sV}v2ExSHi84pRa;3LsacPI^~^17y!F1$)* G4|x*$B7(XA diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 273649da9..b9cd42694 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.0 \ - --hash=sha256:f3a641a16fa846886f1d28fcf22c84999c5a2b541468c93c9c4fd1efaf27e4a2 \ - --hash=sha256:492b4680855eddc65bee396be80e653ee212cd495d1b796e570dc97c3008f695 -acme==0.27.0 \ - --hash=sha256:7ee77fce381dc78cb2228945be8dd6a4010f7378cd2d7a4173f27f231b84444a \ - --hash=sha256:e6158df21b887c24ab4a39d16ac6a8c644d543079ebefd580966e563be4102f3 -certbot-apache==0.27.0 \ - --hash=sha256:e7428b61de428710e07331485b1a41b108ca34a78cfcdaa91741c030b2445215 \ - --hash=sha256:8bde490a9982a4c0fe59e1bf46a68cc0b2ddc942eea8e00d5b01e78ac7a815ca -certbot-nginx==0.27.0 \ - --hash=sha256:52a081c6b9c840ea3d6ddebc300d0e185213a511a4ac2da4a041d526d666483c \ - --hash=sha256:9bd1aaaab413db399112eb46730d5a63a8363b37febd957d18447bd53ade82fb +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f From 5d1c6d28d50c539cf616d79634cd579252ca9a3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 7 Sep 2018 12:18:59 -0700 Subject: [PATCH 344/639] Update DNS plugin docs. (#6358) --- docs/using.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 946c12bc6..76db22c84 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -190,10 +190,11 @@ If you'd like to obtain a wildcard certificate from Let's Encrypt or run ``certbot`` on a machine other than your target webserver, you can use one of Certbot's DNS plugins. -These plugins are still in the process of being packaged -by many distributions and cannot currently be installed with ``certbot-auto``. -If, however, you are comfortable installing the certificates yourself, -you can run these plugins with :ref:`Docker `. +These plugins are not included in a default Certbot installation and must be +installed separately. While the DNS plugins cannot currently be used with +``certbot-auto``, they are available in many OS package managers and as Docker +images. Visit https://certbot.eff.org to learn the best way to use the DNS +plugins on your system. Once installed, you can find documentation on how to use each plugin at: From 85a859d63fb0617166aae58f362a27c709bc85eb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 8 Sep 2018 16:34:27 +0200 Subject: [PATCH 345/639] Make Certbot runnable on Windows (#6296) * Add and use a compatibility layer to allow certbot to be run on windows. * Fix path comparison * Corrections on compat and util for tests * Less intrusive way to parse prefix in webroot plugin working for both linux and windows. * Disable pylint import-error for some optional imports in compat.py * Ensure path is normalized before prefixes are generated in webroot plugin * Same prefixes in linux and windows, in fact root path is not needed in webroot plugin * Check that user has administrative rights before continuing on windows (necessary for symlink creation) * More straightforward way to test administrative rights on windows * Try to resolve import error in travis ci * OK. We go for full introspection to trick the ci. * Move the administrative rights control to the certbot entrypoint * Add comment for a really non trivial code. * Allow some commands to be run on a shell without admin rights * Avoid races conditions on windows for lock files * Add sphinx doc to the compat functions. * Remove irrelevant Windows error in the lock mechanism. * Some corrections on compat --- certbot/account.py | 5 +- certbot/cert_manager.py | 5 +- certbot/client.py | 3 +- certbot/compat.py | 140 ++++++++++++++++++++++++++++ certbot/crypto_util.py | 5 +- certbot/display/util.py | 9 +- certbot/lock.py | 23 +---- certbot/log.py | 3 +- certbot/main.py | 12 ++- certbot/plugins/util.py | 7 +- certbot/plugins/util_test.py | 6 +- certbot/plugins/webroot.py | 6 +- certbot/reverter.py | 7 +- certbot/tests/configuration_test.py | 19 ++-- certbot/tests/display/util_test.py | 8 +- certbot/tests/lock_test.py | 2 +- certbot/tests/main_test.py | 5 +- certbot/tests/util.py | 1 - certbot/tests/util_test.py | 5 +- 19 files changed, 204 insertions(+), 67 deletions(-) create mode 100644 certbot/compat.py diff --git a/certbot/account.py b/certbot/account.py index 59ceb42e0..76135c3aa 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -17,6 +17,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -140,7 +141,7 @@ class AccountFileStorage(interfaces.AccountStorage): """ def __init__(self, config): self.config = config - util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(), self.config.strict_permissions) def _account_dir_path(self, account_id): @@ -323,7 +324,7 @@ class AccountFileStorage(interfaces.AccountStorage): def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) - util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), + util.make_or_verify_dir(account_dir_path, 0o700, compat.os_geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 771ca8caf..2a67f8765 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -8,6 +8,7 @@ import traceback import zope.component from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -104,7 +105,7 @@ def lineage_for_certname(cli_config, certname): """Find a lineage object with name certname.""" configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) try: renewal_file = storage.renewal_file_for_certname(cli_config, certname) except errors.CertStorageError: @@ -374,7 +375,7 @@ def _search_lineages(cli_config, func, initial_rv, *args): """ configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) rv = initial_rv for renewal_file in storage.renewal_conf_files(cli_config): diff --git a/certbot/client.py b/certbot/client.py index 4d4915a27..e634b6bd9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -24,6 +24,7 @@ import certbot from certbot import account from certbot import auth_handler from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import eff @@ -447,7 +448,7 @@ class Client(object): """ for path in cert_path, chain_path, fullchain_path: util.make_or_verify_dir( - os.path.dirname(path), 0o755, os.geteuid(), + os.path.dirname(path), 0o755, compat.os_geteuid(), self.config.strict_permissions) diff --git a/certbot/compat.py b/certbot/compat.py new file mode 100644 index 000000000..1dc89dfd8 --- /dev/null +++ b/certbot/compat.py @@ -0,0 +1,140 @@ +""" +Compatibility layer to run certbot both on Linux and Windows. + +The approach used here is similar to Modernizr for Web browsers. +We do not check the plateform type to determine if a particular logic is supported. +Instead, we apply a logic, and then fallback to another logic if first logic +is not supported at runtime. + +Then logic chains are abstracted into single functions to be exposed to certbot. +""" +import os +import select +import sys +import errno +import ctypes + +from certbot import errors + +try: + # Linux specific + import fcntl # pylint: disable=import-error +except ImportError: + # Windows specific + import msvcrt # pylint: disable=import-error + +UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ + 'certificates', 'enhance', 'revoke', 'delete', + 'register', 'unregister', 'config_changes', 'plugins'] +def raise_for_non_administrative_windows_rights(subcommand): + """ + On Windows, raise if current shell does not have the administrative rights. + Do nothing on Linux. + + :param str subcommand: The subcommand (like 'certonly') passed to the certbot client. + + :raises .errors.Error: If the provided subcommand must be run on a shell with + administrative rights, and current shell does not have these rights. + + """ + # Why not simply try ctypes.windll.shell32.IsUserAnAdmin() and catch AttributeError ? + # Because windll exists only on a Windows runtime, and static code analysis engines + # do not like at all non existent objects when run from Linux (even if we handle properly + # all the cases in the code). + # So we access windll only by reflection to trick theses engines. + if hasattr(ctypes, 'windll') and subcommand not in UNPRIVILEGED_SUBCOMMANDS_ALLOWED: + windll = getattr(ctypes, 'windll') + if windll.shell32.IsUserAnAdmin() == 0: + raise errors.Error( + 'Error, "{0}" subcommand must be run on a shell with administrative rights.' + .format(subcommand)) + +def os_geteuid(): + """ + Get current user uid + + :returns: The current user uid. + :rtype: int + + """ + try: + # Linux specific + return os.geteuid() + except AttributeError: + # Windows specific + return 0 + +def readline_with_timeout(timeout, prompt): + """ + Read user input to return the first line entered, or raise after specified timeout. + + :param float timeout: The timeout in seconds given to the user. + :param str prompt: The prompt message to display to the user. + + :returns: The first line entered by the user. + :rtype: str + + """ + try: + # Linux specific + # + # Call to select can only be done like this on UNIX + rlist, _, _ = select.select([sys.stdin], [], [], timeout) + if not rlist: + raise errors.Error( + "Timed out waiting for answer to prompt '{0}'".format(prompt)) + return rlist[0].readline() + except OSError: + # Windows specific + # + # No way with select to make a timeout to the user input on Windows, + # as select only supports socket in this case. + # So no timeout on Windows for now. + return sys.stdin.readline() + +def lock_file(fd): + """ + Lock the file linked to the specified file descriptor. + + :param int fd: The file descriptor of the file to lock. + + """ + if 'fcntl' in sys.modules: + # Linux specific + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + # Windows specific + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + +def release_locked_file(fd, path): + """ + Remove, close, and release a lock file specified by its file descriptor and its path. + + :param int fd: The file descriptor of the lock file. + :param str path: The path of the lock file. + + """ + # Linux specific + # + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(path) + except OSError as err: + if err.errno == errno.EACCES: + # Windows specific + # We will not be able to remove a file before closing it. + # To avoid race conditions described for Linux, we will not delete the lockfile, + # just close it to be reused on the next Certbot call. + pass + else: + raise + finally: + os.close(fd) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 943e2c87f..6193a8fbf 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -25,6 +25,7 @@ from OpenSSL import SSL # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import errors from certbot import interfaces from certbot import util @@ -60,7 +61,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): config = zope.component.getUtility(interfaces.IConfig) # Save file - util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(key_dir, 0o700, compat.os_geteuid(), config.strict_permissions) key_f, key_path = util.unique_file( os.path.join(key_dir, keyname), 0o600, "wb") @@ -91,7 +92,7 @@ def init_save_csr(privkey, names, path): privkey.pem, names, must_staple=config.must_staple) # Save CSR - util.make_or_verify_dir(path, 0o755, os.geteuid(), + util.make_or_verify_dir(path, 0o755, compat.os_geteuid(), config.strict_permissions) csr_f, csr_filename = util.unique_file( os.path.join(path, "csr-certbot.pem"), 0o644, "wb") diff --git a/certbot/display/util.py b/certbot/display/util.py index e157a1123..9a813d4b7 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -1,12 +1,12 @@ """Certbot display.""" import logging import os -import select import sys import textwrap import zope.interface +from certbot import compat from certbot import constants from certbot import interfaces from certbot import errors @@ -79,13 +79,8 @@ def input_with_timeout(prompt=None, timeout=36000.0): sys.stdout.write(prompt) sys.stdout.flush() - # select can only be used like this on UNIX - rlist, _, _ = select.select([sys.stdin], [], [], timeout) - if not rlist: - raise errors.Error( - "Timed out waiting for answer to prompt '{0}'".format(prompt)) + line = compat.readline_with_timeout(timeout, prompt) - line = rlist[0].readline() if not line: raise EOFError return line.rstrip('\n') diff --git a/certbot/lock.py b/certbot/lock.py index 5f59cc090..3ff46518d 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,9 +1,9 @@ """Implements file locks for locking files and directories in UNIX.""" import errno -import fcntl import logging import os +from certbot import compat from certbot import errors logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ class LockFile(object): """ try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + compat.lock_file(fd) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): logger.debug( @@ -118,22 +118,7 @@ class LockFile(object): def release(self): """Remove, close, and release the lock file.""" - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - # - # Calling os.remove on a file that's in use doesn't work on - # Windows, but neither does locking with fcntl. try: - os.remove(self._path) + compat.release_locked_file(self._fd, self._path) finally: - try: - os.close(self._fd) - finally: - self._fd = None + self._fd = None diff --git a/certbot/log.py b/certbot/log.py index 89626af99..b883936f3 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -23,6 +23,7 @@ import traceback from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import util @@ -133,7 +134,7 @@ def setup_log_file_handler(config, logfile, fmt): # TODO: logs might contain sensitive data such as contents of the # private key! #525 util.set_up_core_dir( - config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) + config.logs_dir, 0o700, compat.os_geteuid(), config.strict_permissions) log_file_path = os.path.join(config.logs_dir, logfile) try: handler = logging.handlers.RotatingFileHandler( diff --git a/certbot/main.py b/certbot/main.py index 2cba8745b..214378da8 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -19,6 +19,7 @@ from certbot import account from certbot import cert_manager from certbot import cli from certbot import client +from certbot import compat from certbot import configuration from certbot import constants from certbot import crypto_util @@ -1289,16 +1290,16 @@ def make_or_verify_needed_dirs(config): """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) hook_dirs = (config.renewal_pre_hooks_dir, config.renewal_deploy_hooks_dir, config.renewal_post_hooks_dir,) for hook_dir in hook_dirs: util.make_or_verify_dir(hook_dir, - uid=os.geteuid(), + uid=compat.os_geteuid(), strict=config.strict_permissions) @@ -1333,6 +1334,7 @@ def main(cli_args=sys.argv[1:]): :raises errors.Error: error if plugin command is not supported """ + log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() @@ -1346,6 +1348,10 @@ def main(cli_args=sys.argv[1:]): config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) + # On windows, shell without administrative right cannot create symlinks required by certbot. + # So we check the rights before continuing. + compat.raise_for_non_administrative_windows_rights(config.verb) + try: log.post_arg_parse_setup(config) make_or_verify_needed_dirs(config) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index 3f03bf375..5c682c3ff 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -9,18 +9,19 @@ logger = logging.getLogger(__name__) def get_prefixes(path): """Retrieves all possible path prefixes of a path, in descending order of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + (linux) /a/b/c returns ['/a/b/c', '/a/b', '/a', '/'] + (windows) C:\\a\\b\\c returns ['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:'] :param str path: the path to break into prefixes :returns: all possible path prefixes of given path in descending order :rtype: `list` of `str` """ - prefix = path + prefix = os.path.normpath(path) prefixes = [] while len(prefix) > 0: prefixes.append(prefix) prefix, _ = os.path.split(prefix) - # break once we hit '/' + # break once we hit the root path if prefix == prefixes[-1]: break return prefixes diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 9757d8de7..b2f9c79ea 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -9,9 +9,9 @@ class GetPrefixTest(unittest.TestCase): """Tests for certbot.plugins.get_prefixes.""" def test_get_prefix(self): from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) + self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes('/'), ['/']) + self.assertEqual(get_prefixes('a'), ['a']) class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 5d0d7d586..529094705 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -170,7 +170,9 @@ to serve all files under specified web root ({0}).""" old_umask = os.umask(0o022) try: stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + # We ignore the last prefix in the next iteration, + # as it does not correspond to a folder path ('/' or 'C:') + for prefix in sorted(util.get_prefixes(self.full_roots[name])[:-1], key=len): try: # This is coupled with the "umask" call above because # os.mkdir's "mode" parameter may not always work: @@ -180,7 +182,7 @@ to serve all files under specified web root ({0}).""" # Set owner as parent directory if possible try: os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: + except (OSError, AttributeError) as exception: logger.info("Unable to change owner and uid of webroot directory") logger.debug("Error was: %s", exception) except OSError as exception: diff --git a/certbot/reverter.py b/certbot/reverter.py index 683c0cc32..5d56615fd 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -10,6 +10,7 @@ import traceback import six import zope.component +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -65,7 +66,7 @@ class Reverter(object): self.config = config util.make_or_verify_dir( - config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + config.backup_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) def revert_temporary_config(self): @@ -219,7 +220,7 @@ class Reverter(object): """ util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) op_fd, existing_filepaths = self._read_and_append( @@ -433,7 +434,7 @@ class Reverter(object): cp_dir = self.config.in_progress_dir util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) return cp_dir diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 59fb2cea9..eb8bcff89 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -48,18 +48,23 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_constants.TEMP_CHECKPOINT_DIR = 't' self.assertEqual( - self.config.accounts_dir, os.path.join( - self.config.config_dir, 'acc/acme-server.org:443/new')) + os.path.normpath(self.config.accounts_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'acc/acme-server.org:443/new'))) self.assertEqual( - self.config.backup_dir, os.path.join(self.config.work_dir, 'backups')) + os.path.normpath(self.config.backup_dir), + os.path.normpath(os.path.join(self.config.work_dir, 'backups'))) self.assertEqual( - self.config.csr_dir, os.path.join(self.config.config_dir, 'csr')) + os.path.normpath(self.config.csr_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'csr'))) self.assertEqual( - self.config.in_progress_dir, os.path.join(self.config.work_dir, '../p')) + os.path.normpath(self.config.in_progress_dir), + os.path.normpath(os.path.join(self.config.work_dir, '../p'))) self.assertEqual( - self.config.key_dir, os.path.join(self.config.config_dir, 'keys')) + os.path.normpath(self.config.key_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'keys'))) self.assertEqual( - self.config.temp_checkpoint_dir, os.path.join(self.config.work_dir, 't')) + os.path.normpath(self.config.temp_checkpoint_dir), + os.path.normpath(os.path.join(self.config.work_dir, 't'))) def test_absolute_paths(self): from certbot.configuration import NamespaceConfig diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 1dfc21c30..f5cf29047 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -34,7 +34,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" stdin = six.StringIO(expected + "\n") - with mock.patch("certbot.display.util.select.select") as mock_select: + with mock.patch("certbot.compat.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -321,11 +321,7 @@ class FileOutputDisplayTest(unittest.TestCase): class NoninteractiveDisplayTest(unittest.TestCase): - """Test non-interactive display. - - These tests are pretty easy! - - """ + """Test non-interactive display. These tests are pretty easy!""" def setUp(self): super(NoninteractiveDisplayTest, self).setUp() self.mock_stdout = mock.MagicMock() diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index e1a4f8c8a..51469e8c1 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -89,7 +89,7 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.lock.fcntl.lockf') + @mock.patch('certbot.compat.fcntl.lockf') def test_unexpected_lockf_err(self, mock_lockf): msg = 'hi there' mock_lockf.side_effect = IOError(msg) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index cc4e6c293..47ca235f1 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -19,6 +19,7 @@ from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli +from certbot import compat from certbot import constants from certbot import configuration from certbot import crypto_util @@ -1577,7 +1578,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for core_dir in (self.config.config_dir, self.config.work_dir,): mock_util.set_up_core_dir.assert_any_call( core_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), self.config.strict_permissions + compat.os_geteuid(), self.config.strict_permissions ) hook_dirs = (self.config.renewal_pre_hooks_dir, @@ -1586,7 +1587,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for hook_dir in hook_dirs: # default mode of 755 is used mock_util.make_or_verify_dir.assert_any_call( - hook_dir, uid=os.geteuid(), + hook_dir, uid=compat.os_geteuid(), strict=self.config.strict_permissions) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8434d11de..d505ea76c 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -362,7 +362,6 @@ def lock_and_call(func, lock_path): child.join() assert child.exitcode == 0 - def hold_lock(cv, lock_path): # pragma: no cover """Acquire a file lock at lock_path and wait to release it. diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 0e280f3ab..689b4108d 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -10,6 +10,7 @@ import mock import six from six.moves import reload_module # pylint: disable=import-error +from certbot import compat from certbot import errors import certbot.tests.util as test_util @@ -116,7 +117,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): @mock.patch('certbot.util.lock_dir_until_exit') def test_success(self, mock_lock): new_dir = os.path.join(self.tempdir, 'new') - self._call(new_dir, 0o700, os.geteuid(), False) + self._call(new_dir, 0o700, compat.os_geteuid(), False) self.assertTrue(os.path.exists(new_dir)) self.assertEqual(mock_lock.call_count, 1) @@ -124,7 +125,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, - self.tempdir, 0o700, os.geteuid(), False) + self.tempdir, 0o700, compat.os_geteuid(), False) class MakeOrVerifyDirTest(test_util.TempDirTestCase): From 251355cade365b77c79582a8b7f14454c601a10a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 11 Sep 2018 15:44:26 -0700 Subject: [PATCH 346/639] Add better error handling around release signatures (#6353) * Better error handling around sig after offline-sig * Add error handling around first sig with git. * Don't fail if offline-sig fails. --- tools/_release.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index ec9bd7461..dab4eec3a 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -15,8 +15,7 @@ RELEASE_BRANCH="candidate-$version" if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" fi -DEFAULT_GPG_KEY="A2CFB51FA275A7286234E7B24D17C995CD9775F2" -RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-"$DEFAULT_GPG_KEY"} +RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -185,7 +184,7 @@ fi letsencrypt-auto-source/build.py # and that it's signed correctly -tools/offline-sigrequest.sh +tools/offline-sigrequest.sh || true while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ letsencrypt-auto-source/letsencrypt-auto.sig \ letsencrypt-auto-source/letsencrypt-auto ; do @@ -193,21 +192,19 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Would you like this script to try and sign it again [Y/n]?" response case $response in [yY][eE][sS]|[yY]|"") - tools/offline-sigrequest.sh;; + tools/offline-sigrequest.sh || true;; *) ;; esac done -if [ "$RELEASE_GPG_KEY" = "$DEFAULT_GPG_KEY" ]; then - while ! gpg2 --card-status >/dev/null 2>&1; do - echo gpg cannot find your OpenPGP card - read -p "Please take the card out and put it back in again." - done -fi - # This signature is not quite as strong, but easier for people to verify out of band -gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto +while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do + echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." + echo "Make sure your OpenPGP card is in your computer if you are using one." + echo "You may need to take the card out and put it back in again." + read -p "Press enter to try signing again." +done # We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, # but we can use the right name for certbot-auto.asc from day one mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc @@ -218,7 +215,12 @@ cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt git diff --cached -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do + echo "Unable to sign the release commit using git." + echo "You may have to configure git to use gpg2 by running:" + echo 'git config --global gpg.program $(command -v gpg2)' + read -p "Press enter to try signing again." +done git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" cd .. From 8f7209de145735565b21d66b5796035a3bce7615 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 12 Sep 2018 16:35:43 -0700 Subject: [PATCH 347/639] Silence spammy integration test cases. (#5934) --- tests/certbot-boulder-integration.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 8b8b931e5..8a8e60798 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -185,10 +185,10 @@ get_num_tmp_files() { ls -1 /tmp | wc -l } num_tmp_files=$(get_num_tmp_files) -common --csr / && echo expected error && exit 1 || true -common --help -common --help all -common --version +common --csr / > /dev/null && echo expected error && exit 1 || true +common --help > /dev/null +common --help all > /dev/null +common --version > /dev/null if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then echo "New files or directories created in /tmp!" exit 1 From a3b858db34a2512d184f8d90220fd02d3c310c7e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 13 Sep 2018 00:38:37 +0100 Subject: [PATCH 348/639] Exclude one-time use parameters. Fixes #6118 (#6223) * Exclude one-time use parameters. Fixes #6118 * Fix error. * Delete items inplace, rather than creating new list. * Fix stupid mistake. * Use .index() for stability. * Try previous idea while resetting the index. * Shorter comment for pylint. * More readable approach * Fix whitespace --- certbot-nginx/certbot_nginx/parser.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 7d1da2e73..a7ca2a735 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -395,12 +395,17 @@ class NginxParser(object): addr.ipv6only = False for directive in enclosing_block[new_vhost.path[-1]][1]: if len(directive) > 0 and directive[0] == 'listen': - if 'default_server' in directive: - del directive[directive.index('default_server')] - if 'default' in directive: - del directive[directive.index('default')] - if 'ipv6only=on' in directive: - del directive[directive.index('ipv6only=on')] + # Exclude one-time use parameters which will cause an error if repeated. + # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen + exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', + 'ipv6only', 'reuseport', 'so_keepalive')) + + for param in exclude: + # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 + keys = [x.split('=')[0] for x in directive] + if param in keys: + del directive[keys.index(param)] return new_vhost From b32ec6ed301bd591124a1a60abc7710b3b95b5d7 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Wed, 12 Sep 2018 16:40:10 -0700 Subject: [PATCH 349/639] Remove CHANGES.rst (#6162) The change log is now being tracked in CHANGELOG.md, so CHANGES.rst is no longer necessary. --- CHANGES.rst | 8 -------- Dockerfile | 2 +- Dockerfile-old | 2 +- MANIFEST.in | 2 +- certbot-compatibility-test/Dockerfile | 2 +- setup.py | 3 +-- 6 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 CHANGES.rst diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 3b92c125f..000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,8 +0,0 @@ -ChangeLog -========= - -To see the changes in a given release, view the issues closed in a given -release's GitHub milestone: - - - `Past releases `_ - - `Upcoming releases `_ diff --git a/Dockerfile b/Dockerfile index 28cd6b323..d1296b30f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot -COPY CHANGES.rst README.rst setup.py src/ +COPY CHANGELOG.md README.rst setup.py src/ COPY acme src/acme COPY certbot src/certbot diff --git a/Dockerfile-old b/Dockerfile-old index 7bce82e0c..da48c7fc8 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ # all above files are necessary for setup.py and venv setup, however, # package source code directory has to be copied separately to a diff --git a/MANIFEST.in b/MANIFEST.in index 434a156b7..7f529c7a7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.rst -include CHANGES.rst +include CHANGELOG.md include CONTRIBUTING.md include LICENSE.txt include linter_plugin.py diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index fe55a68a6..803b4a1b9 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... diff --git a/setup.py b/setup.py index a13b7cdb9..f8f5feadc 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ init_fn = os.path.join(here, 'certbot', '__init__.py') meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) -changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] # This package relies on PyOpenSSL, requests, and six, however, it isn't @@ -79,7 +78,7 @@ setup( name='certbot', version=version, description="ACME client", - long_description=readme, # later: + '\n\n' + changes + long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", author_email='client-dev@letsencrypt.org', From 38b1d9d6bab6bef9f9f8499a9a6f8dcd921133ba Mon Sep 17 00:00:00 2001 From: David Beitey Date: Wed, 12 Sep 2018 23:48:50 +0000 Subject: [PATCH 350/639] More detailed error logging for nginx plugin (#6175) This makes errors more useful when Nginx can't be found or when Nginx's configuration can't be found. Previously, a generic `NoInstallationError` isn't descriptive enough to explain _what_ wasn't installed or what failed without going digging into the source code. --- certbot-nginx/certbot_nginx/configurator.py | 4 +++- certbot-nginx/certbot_nginx/parser.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index d31a54c17..d526381a2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -136,7 +136,9 @@ class NginxConfigurator(common.Installer): """ # Verify Nginx is installed if not util.exe_exists(self.conf('ctl')): - raise errors.NoInstallationError + raise errors.NoInstallationError( + "Could not find a usable 'nginx' binary. Ensure nginx exists, " + "the binary is executable, and your PATH is set correctly.") # Make sure configuration is valid self.config_test() diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index a7ca2a735..a5cf2892e 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -222,7 +222,7 @@ class NginxParser(object): return os.path.join(self.root, name) raise errors.NoInstallationError( - "Could not find configuration root") + "Could not find Nginx root configuration file (nginx.conf)") def filedump(self, ext='tmp', lazy=True): """Dumps parsed configurations into files. From e501322ff1e4f0349bdd6ecc3dd526613cd7f00d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 14 Sep 2018 00:20:22 +0200 Subject: [PATCH 351/639] Connect AppVeyor to the certbot git repository (#6361) --- appveyor.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..67ad67c16 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,12 @@ +# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2 +branches: + only: + - master + - /^\d+\.\d+\.x$/ # version branches like X.X.X + - /^test-.*$/ + +build: off + +test_script: + - ps: Write-Host "Hello, world!" + \ No newline at end of file From 3ef43e4d88783fe23c0ff44e5edfc75ebe4891c8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 18 Sep 2018 12:52:11 -0700 Subject: [PATCH 352/639] Update parser to match new Nginx functionality (#6381) Previously, Nginx did not allow `${` to start a variable name. Now it's allowed to. This means we'll be more permissible than Nginx when people are on older versions of Nginx, but it's unlikely anyone was relying on this to fail in the first place, so that's probably ok. --- CHANGELOG.md | 6 +++--- certbot-nginx/certbot_nginx/nginxparser.py | 2 +- .../certbot_nginx/tests/nginxparser_test.py | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d80fe730..9abe047c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,15 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* - +* Match Nginx parser update in allowing variable names to start with `${`. + ## 0.27.1 - 2018-09-06 ### Fixed * Fixed parameter name in OpenSUSE overrides for default parameters in the Apache plugin. Certbot on OpenSUSE works again. - + Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 8818bc040..bfb75adcc 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -26,7 +26,7 @@ class RawNginxParser(object): dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\') squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\') quoted = dquoted | squoted - head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space) + head_tokenchars = Regex(r"(\$\{)|[^{};\s'\"]") # if (last_space) tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars)) paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars)) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index dd31ebac8..7fc4576c3 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -271,6 +271,8 @@ class TestRawNginxParser(unittest.TestCase): location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ { alias /data/w3/images/$1; } + + proxy_set_header X-Origin-URI ${scheme}://${http_host}/$request_uri; """ parsed = loads(test) self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'], @@ -281,7 +283,8 @@ class TestRawNginxParser(unittest.TestCase): [['return', '403']]], [['if', '($args', '~', 'post=140)'], [['rewrite', '^', 'http://example.com/']]], [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'], - [['alias', '/data/w3/images/$1']]]] + [['alias', '/data/w3/images/$1']]], + ['proxy_set_header', 'X-Origin-URI', '${scheme}://${http_host}/$request_uri']] ) def test_edge_cases(self): @@ -289,10 +292,6 @@ class TestRawNginxParser(unittest.TestCase): parsed = loads(r'"hello\""; # blah "heh heh"') self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']]) - # empty var as block - parsed = loads(r"${}") - self.assertEqual(parsed, [[['$'], []]]) - # if with comment parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah ) }""") @@ -342,10 +341,9 @@ class TestRawNginxParser(unittest.TestCase): ]) # variable weirdness - parsed = loads("directive $var;") - self.assertEqual(parsed, [['directive', '$var']]) + parsed = loads("directive $var ${var} $ ${};") + self.assertEqual(parsed, [['directive', '$var', '${var}', '$', '${}']]) self.assertRaises(ParseException, loads, "server {server_name test.com};") - self.assertRaises(ParseException, loads, "directive ${var};") self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']]) self.assertRaises(ParseException, loads, "blag${dfgdf{g};") From efd2ed1bdb38cba36058be46a0208677012389c4 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 19 Sep 2018 02:35:28 +0200 Subject: [PATCH 353/639] Correct OVH integration tests on machines without internet access (#6380) * Correct OVH integration tests on machines without internet. * Update changelog --- CHANGELOG.md | 1 + certbot-dns-ovh/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9abe047c5..25aff0a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed * Match Nginx parser update in allowing variable names to start with `${`. +* Correct OVH integration tests on machines without internet access. ## 0.27.1 - 2018-09-06 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 15fe3d6d7..1f3acbf62 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -11,7 +11,7 @@ version = '0.28.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=2.7.3', # Correct OVH integration tests 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index ef7804328..00ecee03e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.2.1 +dns-lexicon==2.7.3 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 From cb4b1897c98799136187cc86f538828431382a6d Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 19:51:27 -0700 Subject: [PATCH 354/639] Make it clear that we don't need both --cert-path and --cert-name --- certbot/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 3f3a599e8..99bf33180 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -389,7 +389,8 @@ VERB_HELP = [ ("revoke", { "short": "Revoke a certificate specified with --cert-path or --cert-name", "opts": "Options for revocation of certificates", - "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem --cert-name example.com [options]\n\n" + "usage": "\n\n certbot revoke [--cert-path /path/to/fullchain.pem | " + "--cert-name example.com] [options]\n\n" }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", From f4d615371ed2514545373d6f3dce3d816fe17e34 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 19:52:52 -0700 Subject: [PATCH 355/639] Don't let users select both --cert-name and --cert-path when revoking --- certbot/main.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 9bdcb8d0a..86327a836 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -560,42 +560,7 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b if not (config.certname or config.cert_path): raise errors.Error('At least one of --cert-path or --cert-name must be specified.') - if config.certname and config.cert_path: - # first, check if certname and cert_path imply the same certs - implied_cert_name = cert_manager.cert_path_to_lineage(config) - - if implied_cert_name != config.certname: - cert_path_implied_cert_name = cert_manager.cert_path_to_lineage(config) - cert_path_implied_conf = storage.renewal_file_for_certname(config, - cert_path_implied_cert_name) - cert_path_cert = storage.RenewableCert(cert_path_implied_conf, config) - cert_path_info = cert_manager.human_readable_cert_info(config, cert_path_cert, - skip_filter_checks=True) - - cert_name_implied_conf = storage.renewal_file_for_certname(config, config.certname) - cert_name_cert = storage.RenewableCert(cert_name_implied_conf, config) - cert_name_info = cert_manager.human_readable_cert_info(config, cert_name_cert) - - msg = ("You specified conflicting values for --cert-path and --cert-name. " - "Which did you mean to select?") - choices = [cert_path_info, cert_name_info] - try: - code, index = display.menu(msg, - choices, ok_label="Select", force_interactive=True) - except errors.MissingCommandlineFlag: - error_msg = ('To run in non-interactive mode, you must either specify only one of ' - '--cert-path or --cert-name, or both must point to the same certificate lineages.') - raise errors.Error(error_msg) - - if code != display_util.OK or not index in range(0, len(choices)): - raise errors.Error("User ended interaction.") - - if index == 0: - config.certname = cert_path_implied_cert_name - else: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) - - elif config.cert_path: + if config.cert_path: config.certname = cert_manager.cert_path_to_lineage(config) else: # if only config.certname was specified From b42283f0b3acd79973dc4f77ed832c3b7ea6183c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 19:57:48 -0700 Subject: [PATCH 356/639] update boulder integration test to check for new behavior --- tests/certbot-boulder-integration.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 8a8e60798..e250e591b 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -440,25 +440,15 @@ for subdomain in $subdomains; do fi done -# Test that revocation raises correct error if --cert-name and --cert-path don't match +# Test that revocation raises correct error when both --cert-name and --cert-path specified common --domains le1.wtf -common --domains le2.wtf -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true -if ! echo $out | grep "or both must point to the same certificate lineages."; then - echo "Non-interactive revoking with mismatched --cert-name and --cert-path " +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" 2>&1) || true +if ! echo $out | grep "Exactly one of --cert-path or --cert-name must be specified"; then + echo "Non-interactive revoking with both --cert-name and --cert-path " echo "did not raise the correct error!" exit 1 fi -# Revoking by matching --cert-name and --cert-path deletes -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" -out=$(common certificates) -if echo $out | grep "le1.wtf"; then - echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" - exit 1 -fi - # Test that revocation doesn't delete if multiple lineages share an archive dir common --domains le1.wtf common --domains le2.wtf From 7ccd6ec98e03193880ced7f6d7ee9b0a631eaa9e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 20:00:13 -0700 Subject: [PATCH 357/639] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25aff0a82..def17accc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added -* +* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`. ### Changed From faa44070f5b5cde91afa6970e4547ff21ec71c44 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 20:09:47 -0700 Subject: [PATCH 358/639] remove inappropriate logic and tests --- certbot/main.py | 13 ++--- certbot/tests/main_test.py | 102 ------------------------------------- 2 files changed, 5 insertions(+), 110 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 86327a836..6cd2bbfac 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -532,8 +532,7 @@ def _determine_account(config): def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-branches """Does the user want to delete their now-revoked certs? If run in non-interactive mode, - deleting happens automatically, unless if both `--cert-name` and `--cert-path` were - specified with conflicting values. + deleting happens automatically. :param config: parsed command line arguments :type config: interfaces.IConfig @@ -557,15 +556,13 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) return - if not (config.certname or config.cert_path): - raise errors.Error('At least one of --cert-path or --cert-name must be specified.') + # config.cert_path must have been set + # config.certname may have been set + assert config.cert_path - if config.cert_path: + if not config.certname: config.certname = cert_manager.cert_path_to_lineage(config) - else: # if only config.certname was specified - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) - # don't delete if the archive_dir is used by some other lineage archive_dir = storage.full_archive_path( configobj.ConfigObj(storage.renewal_file_for_certname(config, config.certname)), diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index a65723ccf..e1f76223c 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -375,25 +375,6 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase): self._call(config) mock_delete.assert_not_called() - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.storage.cert_path_for_cert_name') - @test_util.patch_get_utility() - def test_cert_name_only(self, mock_get_utility, - mock_cert_path_for_cert_name, mock_delete, mock_archive, - mock_overlapping_archive_dirs, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "" - mock_cert_path_for_cert_name.return_value = "/some/reasonable/path" - mock_overlapping_archive_dirs.return_value = False - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') @@ -456,89 +437,6 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase): self.assertEqual(mock_delete.call_count, 1) self.assertFalse(mock_get_utility().yesno.called) - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_certname_and_cert_path_match(self, mock_get_utility, - mock_cert_path_to_lineage, mock_delete, mock_archive, - mock_overlapping_archive_dirs, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage.return_value = config.certname - mock_overlapping_archive_dirs.return_value = False - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - - # pylint: disable=too-many-arguments - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.human_readable_cert_info') - @mock.patch('certbot.storage.RenewableCert') - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_certname_and_cert_path_mismatch(self, mock_get_utility, - mock_cert_path_to_lineage, mock_renewal_file_for_certname, - mock_RenewableCert, mock_human_readable_cert_info, - mock_delete, mock_archive, mock_overlapping_archive_dirs): - # pylint: disable=unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage = "something else" - mock_RenewableCert.return_value = mock.Mock() - mock_human_readable_cert_info.return_value = "" - mock_overlapping_archive_dirs.return_value = False - from certbot.display import util as display_util - util_mock = mock_get_utility() - util_mock.menu.return_value = (display_util.OK, 0) - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - - # pylint: disable=too-many-arguments - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.human_readable_cert_info') - @mock.patch('certbot.storage.RenewableCert') - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_noninteractive_certname_cert_path_mismatch(self, mock_get_utility, - mock_cert_path_to_lineage, mock_renewal_file_for_certname, - mock_RenewableCert, mock_human_readable_cert_info, - mock_delete, mock_archive, mock_overlapping_archive_dirs): - # pylint: disable=unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage.return_value = "some-reasonable-path.com" - mock_RenewableCert.return_value = mock.Mock() - mock_human_readable_cert_info.return_value = "" - mock_overlapping_archive_dirs.return_value = False - # Test for non-interactive mode - util_mock = mock_get_utility() - util_mock.menu.side_effect = errors.MissingCommandlineFlag("Oh no.") - self.assertRaises(errors.Error, self._call, config) - mock_delete.assert_not_called() - - @mock.patch('certbot.cert_manager.delete') - @test_util.patch_get_utility() - def test_no_certname_or_cert_path(self, mock_get_utility, mock_delete): - # pylint: disable=unused-argument - config = self.config - config.certname = None - config.cert_path = None - self.assertRaises(errors.Error, self._call, config) - mock_delete.assert_not_called() - class DetermineAccountTest(test_util.ConfigTestCase): """Tests for certbot.main._determine_account.""" From eb7a10b5c00c10e6c746342de4534d27cf25ad03 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 20 Sep 2018 20:12:51 -0700 Subject: [PATCH 359/639] use safe args --- certbot/tests/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e1f76223c..8334068c9 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -271,7 +271,7 @@ class RevokeTest(test_util.TempDirTestCase): for patch in self.patches: patch.stop() - def _call(self, args=[]): + def _call(self, args=None): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() From e8eeab3eab33daf5e1f0817bda86b33879ea3285 Mon Sep 17 00:00:00 2001 From: Peter Lebbing Date: Sun, 23 Sep 2018 14:20:59 +0200 Subject: [PATCH 360/639] Respect --quiet when reporting sudo invocation --- letsencrypt-auto-source/letsencrypt-auto | 2 +- letsencrypt-auto-source/letsencrypt-auto.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 09e728aa4..740cb9c73 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 9d0f27009..6d2977832 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi From 19f74c3dc7227f6afcb1616f54b260550c6c01d5 Mon Sep 17 00:00:00 2001 From: fghzxm Date: Sun, 7 Oct 2018 11:14:09 +0800 Subject: [PATCH 361/639] Fix typo in using.rst --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 76db22c84..2f45feca9 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -905,7 +905,7 @@ Lock Files When processing a validation Certbot writes a number of lock files on your system to prevent multiple instances from overwriting each other's changes. This means -that be default two instances of Certbot will not be able to run in parallel. +that by default two instances of Certbot will not be able to run in parallel. Since the directories used by Certbot are configurable, Certbot will write a lock file for all of the directories it uses. This include Certbot's From 139ef206504ca241ae276b383c3cddee73db15cc Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 15 Oct 2018 10:41:04 -0700 Subject: [PATCH 362/639] Add debugging info for Nginx tls-sni and http integration tests purposes (#6414) --- certbot-nginx/certbot_nginx/http_01.py | 1 + certbot-nginx/certbot_nginx/tls_sni_01.py | 2 ++ certbot-nginx/tests/boulder-integration.sh | 1 + 3 files changed, 4 insertions(+) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 677ce0737..9b385bc3b 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -102,6 +102,7 @@ class NginxHttp01(common.ChallengePerformer): config = [self._make_or_mod_server_block(achall) for achall in self.achalls] config = [x for x in config if x is not None] config = nginxparser.UnspacedList(config) + logger.debug("Generated server block:\n%s", str(config)) self.configurator.reverter.register_file_creation( True, self.challenge_conf) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 0fd37e0cb..d49ec8643 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -141,6 +141,8 @@ class NginxTlsSni01(common.TLSSNI01): with open(self.challenge_conf, "w") as new_conf: nginxparser.dump(config, new_conf) + logger.debug("Generated server block:\n%s", str(config)) + def _make_server_block(self, achall, addrs): """Creates a server block for a challenge. diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 980b5d45a..194413f1d 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -35,6 +35,7 @@ test_deployment_and_rollback() { } export default_server="default_server" +nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf From 22da2447d5d89c11b9353ebbd7777690fee999df Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Oct 2018 10:54:43 -0700 Subject: [PATCH 363/639] Stop caching the results of ipv6_info in http01.py (#6411) Stop caching the results of ipv6_info in http01.py. A call to choose_vhosts might change the ipv6 results of later calls. Add tests for this and default_listen_addresses more broadly. --- CHANGELOG.md | 1 + certbot-nginx/certbot_nginx/http_01.py | 6 +--- .../certbot_nginx/tests/http_01_test.py | 36 +++++++++++++++++++ certbot-nginx/certbot_nginx/tls_sni_01.py | 6 ++-- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def17accc..81cc12b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Match Nginx parser update in allowing variable names to start with `${`. * Correct OVH integration tests on machines without internet access. +* Stop caching the results of ipv6_info in http01.py ## 0.27.1 - 2018-09-06 diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 9b385bc3b..e46d7b9b9 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -40,8 +40,6 @@ class NginxHttp01(common.ChallengePerformer): super(NginxHttp01, self).__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") - self._ipv6 = None - self._ipv6only = None def perform(self): """Perform a challenge on Nginx. @@ -121,9 +119,7 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.config.http01_port) port = self.configurator.config.http01_port - if self._ipv6 is None or self._ipv6only is None: - self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) - ipv6, ipv6only = self._ipv6, self._ipv6only + ipv6, ipv6only = self.configurator.ipv6_info(port) if ipv6: # If IPv6 is active in Nginx configuration diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index 0f764e92e..ed3c257ee 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -12,6 +12,7 @@ from certbot import achallenges from certbot.plugins import common_test from certbot.tests import acme_util +from certbot_nginx.obj import Addr from certbot_nginx.tests import util @@ -108,6 +109,41 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_no_memoization(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 1) + ipv6_info.return_value = (False, False) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 2) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_t(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80 ipv6only=on") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_f_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (False, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + self.assertEqual(addrs, [http_addr]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index d49ec8643..60ec1ed1a 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,9 +51,6 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) - ipv6, ipv6only = self.configurator.ipv6_info( - self.configurator.config.tls_sni_01_port) - for achall in self.achalls: vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) @@ -61,6 +58,9 @@ class NginxTlsSni01(common.TLSSNI01): if vhosts and vhosts[0].addrs: addresses.append(list(vhosts[0].addrs)) else: + # choose_vhosts might have modified vhosts, so put this after + ipv6, ipv6only = self.configurator.ipv6_info( + self.configurator.config.tls_sni_01_port) if ipv6: # If IPv6 is active in Nginx configuration ipv6_addr = "[::]:{0} ssl".format( From 819f95c37d6d8471a5413137e033d30cea6822d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 17 Oct 2018 22:48:49 +0200 Subject: [PATCH 364/639] certbot_dns_linode: increase the default propagation interval (#6320) Using the default value of 16 minutes (960 seconds) for --dns-linode-propagation-seconds leads to DNS failures when the randomly selected Linode DNS is not the first one out of six, due to an additional delay before the other five are updated. The problem can be easily solved by increasing the wait interval, so this commit increases the default value to 20 minutes. More details: https://community.letsencrypt.org/t/dns-servers-used-by-letsencrypt-for-challenges/32127/16 --- certbot-dns-linode/certbot_dns_linode/__init__.py | 14 ++++++++++---- .../certbot_dns_linode/dns_linode.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index 0c445f45d..0a6ccec61 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -14,7 +14,11 @@ Named Arguments DNS to propagate before asking the ACME server to verify the DNS record. - (Default: 960) + (Default: 1200 because Linode + updates its first DNS every 15 + minutes and we allow 5 more minutes + for the update to reach the other 5 + servers) ========================================== =================================== @@ -74,13 +78,15 @@ Examples -d www.example.com .. code-block:: bash - :caption: To acquire a certificate for ``example.com``, waiting 60 seconds - for DNS propagation + :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds + for DNS propagation (Linode updates its first DNS every 15 minutes + and we allow some extra time for the update to reach the other 5 + servers) certbot certonly \\ --dns-linode \\ --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ - --dns-linode-propagation-seconds 60 \\ + --dns-linode-propagation-seconds 1000 \\ -d example.com """ diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 323c0810a..cc29ce842 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960) + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-docstring,no-self-use From 92501eaf8fe6afc514999b207037e5003f38c5db Mon Sep 17 00:00:00 2001 From: schoen Date: Wed, 17 Oct 2018 14:08:59 -0700 Subject: [PATCH 365/639] Note about running on web server, not PC (#6422) --- README.rst | 2 +- docs/install.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0dbe1cdef..d75b44a65 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. -If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. +Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME diff --git a/docs/install.rst b/docs/install.rst index f7504baa5..fc6abad7a 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -9,6 +9,8 @@ Get Certbot About Certbot ============= +*Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for From 3de3188dd6fe4c8c9952848c155a56944b55ec2d Mon Sep 17 00:00:00 2001 From: schoen Date: Thu, 18 Oct 2018 04:44:45 -0700 Subject: [PATCH 366/639] Warn manual authenticator users not to remove/undo previous challenges (#6370) * Warn users not to remove/undo previous challenges * Even more specific DNS challenge message * Fix spacing and variable names * Create a second test DNS challenge for UI testing * Changelog for subsequent manual challenge behavior --- CHANGELOG.md | 2 +- certbot/plugins/manual.py | 21 +++++++++++++++++++++ certbot/plugins/manual_test.py | 3 ++- certbot/tests/acme_util.py | 3 +++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81cc12b15..5dd51ef16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed -* +* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. ### Fixed diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 53533d35a..8723a1c62 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -94,6 +94,16 @@ using the secret key {key} when it receives a TLS ClientHello with the SNI extension set to {sni_domain} +""" + _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet.) +""" + _SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet. Note that you might be +asked to create multiple distinct TXT records with the same name. This is +permitted by DNS standards.) """ def __init__(self, *args, **kwargs): @@ -103,6 +113,8 @@ when it receives a TLS ClientHello with the SNI extension set to self.env = dict() \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None + self.subsequent_dns_challenge = False + self.subsequent_any_challenge = False @classmethod def add_parser_arguments(cls, add): @@ -212,8 +224,17 @@ when it receives a TLS ClientHello with the SNI extension set to key=self.tls_sni_01.get_key_path(achall), port=self.config.tls_sni_01_port, sni_domain=self.tls_sni_01.get_z_domain(achall)) + if isinstance(achall.chall, challenges.DNS01): + if self.subsequent_dns_challenge: + # 2nd or later dns-01 challenge + msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + self.subsequent_dns_challenge = True + elif self.subsequent_any_challenge: + # 2nd or later challenge of another type + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS display = zope.component.getUtility(interfaces.IDisplay) display.notification(msg, wrap=False, force_interactive=True) + self.subsequent_any_challenge = True def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index e5c22b377..ba4eb6889 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -20,8 +20,9 @@ class AuthenticatorTest(test_util.TempDirTestCase): super(AuthenticatorTest, self).setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A + self.dns_achall_2 = acme_util.DNS01_A_2 self.tls_sni_achall = acme_util.TLSSNI01_A - self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall] + self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2] for d in ["config_dir", "work_dir", "in_progress"]: os.mkdir(os.path.join(self.tempdir, d)) # "backup_dir" and "temp_checkpoint_dir" get created in diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 53a2f214a..2f9445694 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -21,6 +21,7 @@ HTTP01 = challenges.HTTP01( TLSSNI01 = challenges.TLSSNI01( token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA")) DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a") +DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac") CHALLENGES = [HTTP01, TLSSNI01, DNS01] @@ -49,6 +50,7 @@ def chall_to_challb(chall, status): # pylint: disable=redefined-outer-name TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING) HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING) DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING) +DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING) CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] @@ -57,6 +59,7 @@ CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com") TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net") DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org") +DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org") ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A] From a3a3840e91d4ee086dd183400cbcd39ebd307938 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:19:57 -0700 Subject: [PATCH 367/639] replace status field --- acme/acme/client_test.py | 1 + acme/acme/messages.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 965ece55d..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -697,6 +697,7 @@ class ClientV2Test(ClientTestBase): self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 405fe7d9a..df295bf2b 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -513,6 +513,7 @@ class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -521,6 +522,8 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) From ee02ed65afe907a767e18b1536b5cde806a2cdd6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:26:37 -0700 Subject: [PATCH 368/639] remove default status from Order so that the status field isn't filled in upon boulder deserialization --- acme/acme/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index df295bf2b..7e86b0c3b 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -523,7 +523,7 @@ class Order(ResourceBody): """ identifiers = jose.Field('identifiers', omitempty=True) status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) + omitempty=True) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) From 6500b9095e4e7b4d7b02cb39d7c78d322339dd11 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:37:56 -0700 Subject: [PATCH 369/639] Add test to confirm that status isn't set on neworder object --- acme/acme/messages_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 0e2d8c62d..876fbe825 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -424,6 +424,19 @@ class OrderResourceTest(unittest.TestCase): 'authorizations': None, }) +class NewOrderTest(unittest.TestCase): + """Tests for acme.messages.NewOrder.""" + + def setUp(self): + from acme.messages import NewOrder + self.reg = NewOrder( + identifiers=mock.sentinel.identifiers) + + def test_to_partial_json(self): + self.assertEqual(self.reg.to_json(), { + 'identifiers': mock.sentinel.identifiers, + }) + if __name__ == '__main__': unittest.main() # pragma: no cover From bfaf0296de4065a3b100580ff4f0b6190ebda800 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 18 Oct 2018 11:39:21 -0700 Subject: [PATCH 370/639] Also write README file to /etc/letsencrypt/live (#6377) We want to discourage people from moving things around in `/etc/letsencrypt/live`! So we dropped an extra README in the `/etc/` directory when it's first created. --- CHANGELOG.md | 3 ++- certbot/storage.py | 39 +++++++++++++++++++++-------------- certbot/tests/storage_test.py | 2 ++ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd51ef16..20e82b5d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed -* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Write README to the base of (config-dir)/live directory +* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. ### Fixed diff --git a/certbot/storage.py b/certbot/storage.py index 32d6771c2..c16ea35b8 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -214,6 +214,26 @@ def get_link_target(link): target = os.path.join(os.path.dirname(link), target) return os.path.abspath(target) +def _write_live_readme_to(readme_path, is_base_dir=False): + prefix = "" + if is_base_dir: + prefix = "[cert name]/" + with open(readme_path, "w") as f: + logger.debug("Writing README to %s.", readme_path) + f.write("This directory contains your keys and certificates.\n\n" + "`{prefix}privkey.pem` : the private key for your certificate.\n" + "`{prefix}fullchain.pem`: the certificate file used in most server software.\n" + "`{prefix}chain.pem` : used for OCSP stapling in Nginx >=1.3.7.\n" + "`{prefix}cert.pem` : will break many server configurations, and " + "should not be used\n" + " without reading further documentation (see link below).\n\n" + "WARNING: DO NOT MOVE OR RENAME THESE FILES!\n" + " Certbot expects these files to remain in this location in order\n" + " to function properly!\n\n" + "We recommend not moving these files. For more information, see the Certbot\n" + "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" + "certificates.\n".format(prefix=prefix)) + def _relevant(option): """ @@ -1003,6 +1023,9 @@ class RenewableCert(object): logger.debug("Creating directory %s.", i) config_file, config_filename = util.unique_lineage_name( cli_config.renewal_configs_dir, lineagename) + base_readme_path = os.path.join(cli_config.live_dir, README) + if not os.path.exists(base_readme_path): + _write_live_readme_to(base_readme_path, is_base_dir=True) # Determine where on disk everything will go # lineagename will now potentially be modified based on which @@ -1045,21 +1068,7 @@ class RenewableCert(object): # Write a README file to the live directory readme_path = os.path.join(live_dir, README) - with open(readme_path, "w") as f: - logger.debug("Writing README to %s.", readme_path) - f.write("This directory contains your keys and certificates.\n\n" - "`privkey.pem` : the private key for your certificate.\n" - "`fullchain.pem`: the certificate file used in most server software.\n" - "`chain.pem` : used for OCSP stapling in Nginx >=1.3.7.\n" - "`cert.pem` : will break many server configurations, and " - "should not be used\n" - " without reading further documentation (see link below).\n\n" - "WARNING: DO NOT MOVE THESE FILES!\n" - " Certbot expects these files to remain in this location in order\n" - " to function properly!\n\n" - "We recommend not moving these files. For more information, see the Certbot\n" - "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" - "certificates.\n") + _write_live_readme_to(readme_path) # Document what we've done in a new renewal config file config_file.close() diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 53a976f8d..078a2858f 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -625,6 +625,8 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(result._consistent()) self.assertTrue(os.path.exists(os.path.join( self.config.renewal_configs_dir, "the-lineage.com.conf"))) + self.assertTrue(os.path.exists(os.path.join( + self.config.live_dir, "README"))) self.assertTrue(os.path.exists(os.path.join( self.config.live_dir, "the-lineage.com", "README"))) with open(result.fullchain, "rb") as f: From 0dab41ee139a9186cfb82707981acd56b60bf095 Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Thu, 18 Oct 2018 16:12:47 -0400 Subject: [PATCH 371/639] docs: remove mentions of #letsencrypt on Freenode. (#6419) * docs: remove mentions of #letsencrypt on Freenode. * docs: remove unused Freenode link --- CHANGELOG.md | 1 + README.rst | 4 ---- docs/using.rst | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e82b5d7..5060a7038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed +* Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. diff --git a/README.rst b/README.rst index d75b44a65..62681c7eb 100644 --- a/README.rst +++ b/README.rst @@ -91,8 +91,6 @@ Main Website: https://certbot.eff.org Let's Encrypt Website: https://letsencrypt.org -IRC Channel: #letsencrypt on `Freenode`_ - Community: https://community.letsencrypt.org ACME spec: http://ietf-wg-acme.github.io/acme/ @@ -101,8 +99,6 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme |build-status| |coverage| |docs| |container| -.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt - .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status diff --git a/docs/using.rst b/docs/using.rst index 2f45feca9..1fa13e022 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -988,9 +988,6 @@ Getting help If you're having problems, we recommend posting on the Let's Encrypt `Community Forum `_. -You can also chat with us on IRC: `(#letsencrypt @ -freenode) `_ - If you find a bug in the software, please do report it in our `issue tracker `_. Remember to give us as much information as possible: From 8dd68a655104db2d642e65f48738b92bf133e709 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 19 Oct 2018 12:30:32 -0700 Subject: [PATCH 372/639] Add and test new nginx parsing abstractions (#6383) * feat(nginx): add and test new parsing abstractions * chore(nginx parser): fix mypy and address small comments * chore(nginx parser): clean up by removing context object * fix integration test and lint --- certbot-nginx/certbot_nginx/parser_obj.py | 392 ++++++++++++++++++ .../certbot_nginx/tests/parser_obj_test.py | 253 +++++++++++ tests/integration/_common.sh | 1 + 3 files changed, 646 insertions(+) create mode 100644 certbot-nginx/certbot_nginx/parser_obj.py create mode 100644 certbot-nginx/certbot_nginx/tests/parser_obj_test.py diff --git a/certbot-nginx/certbot_nginx/parser_obj.py b/certbot-nginx/certbot_nginx/parser_obj.py new file mode 100644 index 000000000..f01cb2fd3 --- /dev/null +++ b/certbot-nginx/certbot_nginx/parser_obj.py @@ -0,0 +1,392 @@ +""" This file contains parsing routines and object classes to help derive meaning from +raw lists of tokens from pyparsing. """ + +import abc +import logging +import six + +from certbot import errors + +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + +logger = logging.getLogger(__name__) +COMMENT = " managed by Certbot" +COMMENT_BLOCK = ["#", COMMENT] + +class Parsable(object): + """ Abstract base class for "Parsable" objects whose underlying representation + is a tree of lists. + + :param .Parsable parent: This object's parsed parent in the tree + """ + + __metaclass__ = abc.ABCMeta + + def __init__(self, parent=None): + self._data = [] # type: List[object] + self._tabs = None + self.parent = parent + + @classmethod + def parsing_hooks(cls): + """Returns object types that this class should be able to `parse` recusrively. + The order of the objects indicates the order in which the parser should + try to parse each subitem. + :returns: A list of Parsable classes. + :rtype list: + """ + return (Block, Sentence, Statements) + + @staticmethod + @abc.abstractmethod + def should_parse(lists): + """ Returns whether the contents of `lists` can be parsed into this object. + + :returns: Whether `lists` can be parsed as this object. + :rtype bool: + """ + raise NotImplementedError() + + @abc.abstractmethod + def parse(self, raw_list, add_spaces=False): + """ Loads information into this object from underlying raw_list structure. + Each Parsable object might make different assumptions about the structure of + raw_list. + + :param list raw_list: A list or sublist of tokens from pyparsing, containing whitespace + as separate tokens. + :param bool add_spaces: If set, the method can and should manipulate and insert spacing + between non-whitespace tokens and lists to delimit them. + :raises .errors.MisconfigurationError: when the assumptions about the structure of + raw_list are not met. + """ + raise NotImplementedError() + + @abc.abstractmethod + def iterate(self, expanded=False, match=None): + """ Iterates across this object. If this object is a leaf object, only yields + itself. If it contains references other parsing objects, and `expanded` is set, + this function should first yield itself, then recursively iterate across all of them. + :param bool expanded: Whether to recursively iterate on possible children. + :param callable match: If provided, an object is only iterated if this callable + returns True when called on that object. + + :returns: Iterator over desired objects. + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_tabs(self): + """ Guess at the tabbing style of this parsed object, based on whitespace. + + If this object is a leaf, it deducts the tabbing based on its own contents. + Other objects may guess by calling `get_tabs` recursively on child objects. + + :returns: Guess at tabbing for this object. Should only return whitespace strings + that does not contain newlines. + :rtype str: + """ + raise NotImplementedError() + + @abc.abstractmethod + def set_tabs(self, tabs=" "): + """This tries to set and alter the tabbing of the current object to a desired + whitespace string. Primarily meant for objects that were constructed, so they + can conform to surrounding whitespace. + + :param str tabs: A whitespace string (not containing newlines). + """ + raise NotImplementedError() + + def dump(self, include_spaces=False): + """ Dumps back to pyparsing-like list tree. The opposite of `parse`. + + Note: if this object has not been modified, `dump` with `include_spaces=True` + should always return the original input of `parse`. + + :param bool include_spaces: If set to False, magically hides whitespace tokens from + dumped output. + + :returns: Pyparsing-like list tree. + :rtype list: + """ + return [elem.dump(include_spaces) for elem in self._data] + +class Statements(Parsable): + """ A group or list of "Statements". A Statement is either a Block or a Sentence. + + The underlying representation is simply a list of these Statement objects, with + an extra `_trailing_whitespace` string to keep track of the whitespace that does not + precede any more statements. + """ + def __init__(self, parent=None): + super(Statements, self).__init__(parent) + self._trailing_whitespace = None + + # ======== Begin overridden functions + + @staticmethod + def should_parse(lists): + return isinstance(lists, list) + + def set_tabs(self, tabs=" "): + """ Sets the tabbing for this set of statements. Does this by calling `set_tabs` + on each of the child statements. + + Then, if a parent is present, sets trailing whitespace to parent tabbing. This + is so that the trailing } of any Block that contains Statements lines up + with parent tabbing. + """ + for statement in self._data: + statement.set_tabs(tabs) + if self.parent is not None: + self._trailing_whitespace = "\n" + self.parent.get_tabs() + + def parse(self, parse_this, add_spaces=False): + """ Parses a list of statements. + Expects all elements in `parse_this` to be parseable by `type(self).parsing_hooks`, + with an optional whitespace string at the last index of `parse_this`. + """ + if not isinstance(parse_this, list): + raise errors.MisconfigurationError("Statements parsing expects a list!") + # If there's a trailing whitespace in the list of statements, keep track of it. + if len(parse_this) > 0 and isinstance(parse_this[-1], six.string_types) \ + and parse_this[-1].isspace(): + self._trailing_whitespace = parse_this[-1] + parse_this = parse_this[:-1] + self._data = [parse_raw(elem, self, add_spaces) for elem in parse_this] + + def get_tabs(self): + """ Takes a guess at the tabbing of all contained Statements by retrieving the + tabbing of the first Statement.""" + if len(self._data) > 0: + return self._data[0].get_tabs() + return "" + + def dump(self, include_spaces=False): + """ Dumps this object by first dumping each statement, then appending its + trailing whitespace (if `include_spaces` is set) """ + data = super(Statements, self).dump(include_spaces) + if include_spaces and self._trailing_whitespace is not None: + return data + [self._trailing_whitespace] + return data + + def iterate(self, expanded=False, match=None): + """ Combines each statement's iterator. """ + for elem in self._data: + for sub_elem in elem.iterate(expanded, match): + yield sub_elem + + # ======== End overridden functions + +def _space_list(list_): + """ Inserts whitespace between adjacent non-whitespace tokens. """ + spaced_statement = [] # type: List[str] + for i in reversed(six.moves.xrange(len(list_))): + spaced_statement.insert(0, list_[i]) + if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): + spaced_statement.insert(0, " ") + return spaced_statement + +class Sentence(Parsable): + """ A list of words. Non-whitespace words are typically separated with whitespace tokens. """ + + # ======== Begin overridden functions + + @staticmethod + def should_parse(lists): + """ Returns True if `lists` can be parseable as a `Sentence`-- that is, + every element is a string type. + + :param list lists: The raw unparsed list to check. + + :returns: whether this lists is parseable by `Sentence`. + """ + return isinstance(lists, list) and len(lists) > 0 and \ + all([isinstance(elem, six.string_types) for elem in lists]) + + def parse(self, parse_this, add_spaces=False): + """ Parses a list of string types into this object. + If add_spaces is set, adds whitespace tokens between adjacent non-whitespace tokens.""" + if add_spaces: + parse_this = _space_list(parse_this) + if not isinstance(parse_this, list) or \ + any([not isinstance(elem, six.string_types) for elem in parse_this]): + raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") + self._data = parse_this + + def iterate(self, expanded=False, match=None): + """ Simply yields itself. """ + if match is None or match(self): + yield self + + def set_tabs(self, tabs=" "): + """ Sets the tabbing on this sentence. Inserts a newline and `tabs` at the + beginning of `self._data`. """ + if self._data[0].isspace(): + return + self._data.insert(0, "\n" + tabs) + + def dump(self, include_spaces=False): + """ Dumps this sentence. If include_spaces is set, includes whitespace tokens.""" + if not include_spaces: + return self.words + return self._data + + def get_tabs(self): + """ Guesses at the tabbing of this sentence. If the first element is whitespace, + returns the whitespace after the rightmost newline in the string. """ + first = self._data[0] + if not first.isspace(): + return "" + rindex = first.rfind("\n") + return first[rindex+1:] + + # ======== End overridden functions + + @property + def words(self): + """ Iterates over words, but without spaces. Like Unspaced List. """ + return [word.strip("\"\'") for word in self._data if not word.isspace()] + + def __getitem__(self, index): + return self.words[index] + + def __contains__(self, word): + return word in self.words + +class Block(Parsable): + """ Any sort of bloc, denoted by a block name and curly braces, like so: + The parsed block: + block name { + content 1; + content 2; + } + might be represented with the list [names, contents], where + names = ["block", " ", "name", " "] + contents = [["\n ", "content", " ", "1"], ["\n ", "content", " ", "2"], "\n"] + """ + def __init__(self, parent=None): + super(Block, self).__init__(parent) + self.names = None # type: Sentence + self.contents = None # type: Block + + @staticmethod + def should_parse(lists): + """ Returns True if `lists` can be parseable as a `Block`-- that is, + it's got a length of 2, the first element is a `Sentence` and the second can be + a `Statements`. + + :param list lists: The raw unparsed list to check. + + :returns: whether this lists is parseable by `Block`. """ + return isinstance(lists, list) and len(lists) == 2 and \ + Sentence.should_parse(lists[0]) and isinstance(lists[1], list) + + def set_tabs(self, tabs=" "): + """ Sets tabs by setting equivalent tabbing on names, then adding tabbing + to contents.""" + self.names.set_tabs(tabs) + self.contents.set_tabs(tabs + " ") + + def iterate(self, expanded=False, match=None): + """ Iterator over self, and if expanded is set, over its contents. """ + if match is None or match(self): + yield self + if expanded: + for elem in self.contents.iterate(expanded, match): + yield elem + + def parse(self, parse_this, add_spaces=False): + """ Parses a list that resembles a block. + + The assumptions that this routine makes are: + 1. the first element of `parse_this` is a valid Sentence. + 2. the second element of `parse_this` is a valid Statement. + If add_spaces is set, we call it recursively on `names` and `contents`, and + add an extra trailing space to `names` (to separate the block's opening bracket + and the block name). + """ + if not Block.should_parse(parse_this): + raise errors.MisconfigurationError("Block parsing expects a list of length 2. " + "First element should be a list of string types (the bloc names), " + "and second should be another list of statements (the bloc content).") + self.names = Sentence(self) + if add_spaces: + parse_this[0].append(" ") + self.names.parse(parse_this[0], add_spaces) + self.contents = Statements(self) + self.contents.parse(parse_this[1], add_spaces) + self._data = [self.names, self.contents] + + def get_tabs(self): + """ Guesses tabbing by retrieving tabbing guess of self.names. """ + return self.names.get_tabs() + +def _is_comment(parsed_obj): + """ Checks whether parsed_obj is a comment. + + :param .Parsable parsed_obj: + + :returns: whether parsed_obj represents a comment sentence. + :rtype bool: + """ + if not isinstance(parsed_obj, Sentence): + return False + return parsed_obj.words[0] == "#" + +def _is_certbot_comment(parsed_obj): + """ Checks whether parsed_obj is a "managed by Certbot" comment. + + :param .Parsable parsed_obj: + + :returns: whether parsed_obj is a "managed by Certbot" comment. + :rtype bool: + """ + if not _is_comment(parsed_obj): + return False + if len(parsed_obj.words) != len(COMMENT_BLOCK): + return False + for i, word in enumerate(parsed_obj.words): + if word != COMMENT_BLOCK[i]: + return False + return True + +def _certbot_comment(parent, preceding_spaces=4): + """ A "Managed by Certbot" comment. + :param int preceding_spaces: Number of spaces between the end of the previous + statement and the comment. + :returns: Sentence containing the comment. + :rtype: .Sentence + """ + result = Sentence(parent) + result.parse([" " * preceding_spaces] + COMMENT_BLOCK) + return result + +def _choose_parser(parent, list_): + """ Choose a parser from type(parent).parsing_hooks, depending on whichever hook + returns True first. """ + hooks = Parsable.parsing_hooks() + if parent: + hooks = type(parent).parsing_hooks() + for type_ in hooks: + if type_.should_parse(list_): + return type_(parent) + raise errors.MisconfigurationError( + "None of the parsing hooks succeeded, so we don't know how to parse this set of lists.") + +def parse_raw(lists_, parent=None, add_spaces=False): + """ Primary parsing factory function. + + :param list lists_: raw lists from pyparsing to parse. + :param .Parent parent: The parent containing this object. + :param bool add_spaces: Whether to pass add_spaces to the parser. + + :returns .Parsable: The parsed object. + + :raises errors.MisconfigurationError: If no parsing hook passes, and we can't + determine which type to parse the raw lists into. + """ + parser = _choose_parser(parent, lists_) + parser.parse(lists_, add_spaces) + return parser diff --git a/certbot-nginx/certbot_nginx/tests/parser_obj_test.py b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py new file mode 100644 index 000000000..c9c9dd440 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py @@ -0,0 +1,253 @@ +""" Tests for functions and classes in parser_obj.py """ + +import unittest +import mock + +from certbot_nginx.parser_obj import parse_raw +from certbot_nginx.parser_obj import COMMENT_BLOCK + +class CommentHelpersTest(unittest.TestCase): + def test_is_comment(self): + from certbot_nginx.parser_obj import _is_comment + self.assertTrue(_is_comment(parse_raw(['#']))) + self.assertTrue(_is_comment(parse_raw(['#', ' literally anything else']))) + self.assertFalse(_is_comment(parse_raw(['not', 'even', 'a', 'comment']))) + + def test_is_certbot_comment(self): + from certbot_nginx.parser_obj import _is_certbot_comment + self.assertTrue(_is_certbot_comment( + parse_raw(COMMENT_BLOCK))) + self.assertFalse(_is_certbot_comment( + parse_raw(['#', ' not a certbot comment']))) + self.assertFalse(_is_certbot_comment( + parse_raw(['#', ' managed by Certbot', ' also not a certbot comment']))) + self.assertFalse(_is_certbot_comment( + parse_raw(['not', 'even', 'a', 'comment']))) + + def test_certbot_comment(self): + from certbot_nginx.parser_obj import _certbot_comment, _is_certbot_comment + comment = _certbot_comment(None) + self.assertTrue(_is_certbot_comment(comment)) + self.assertEqual(comment.dump(), COMMENT_BLOCK) + self.assertEqual(comment.dump(True), [' '] + COMMENT_BLOCK) + self.assertEqual(_certbot_comment(None, 2).dump(True), + [' '] + COMMENT_BLOCK) + +class ParsingHooksTest(unittest.TestCase): + def test_is_sentence(self): + from certbot_nginx.parser_obj import Sentence + self.assertFalse(Sentence.should_parse([])) + self.assertTrue(Sentence.should_parse([''])) + self.assertTrue(Sentence.should_parse(['word'])) + self.assertTrue(Sentence.should_parse(['two', 'words'])) + self.assertFalse(Sentence.should_parse([[]])) + self.assertFalse(Sentence.should_parse(['word', []])) + + def test_is_block(self): + from certbot_nginx.parser_obj import Block + self.assertFalse(Block.should_parse([])) + self.assertFalse(Block.should_parse([''])) + self.assertFalse(Block.should_parse(['two', 'words'])) + self.assertFalse(Block.should_parse([[[]], []])) + self.assertFalse(Block.should_parse([['block_name'], ['hi', []], []])) + self.assertFalse(Block.should_parse([['block_name'], 'lol'])) + self.assertTrue(Block.should_parse([['block_name'], ['hi', []]])) + self.assertTrue(Block.should_parse([['hello'], []])) + self.assertTrue(Block.should_parse([['block_name'], [['many'], ['statements'], 'here']])) + self.assertTrue(Block.should_parse([['if', ' ', '(whatever)'], ['hi']])) + + def test_parse_raw(self): + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: True + fake_parser2 = mock.Mock() + fake_parser2.should_parse = lambda x: False + # First encountered "match" should parse. + parse_raw([]) + fake_parser1.called_once() + fake_parser2.not_called() + fake_parser1.reset_mock() + # "match" that returns False shouldn't parse. + parse_raw([]) + fake_parser1.not_called() + fake_parser2.called_once() + + @mock.patch("certbot_nginx.parser_obj.Parsable.parsing_hooks") + def test_parse_raw_no_match(self, parsing_hooks): + from certbot import errors + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: False + parsing_hooks.return_value = (fake_parser1,) + self.assertRaises(errors.MisconfigurationError, parse_raw, []) + parsing_hooks.return_value = tuple() + self.assertRaises(errors.MisconfigurationError, parse_raw, []) + + def test_parse_raw_passes_add_spaces(self): + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: True + parse_raw([]) + fake_parser1.parse.called_with([None]) + parse_raw([], add_spaces=True) + fake_parser1.parse.called_with([None, True]) + +class SentenceTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Sentence + self.sentence = Sentence(None) + + def test_parse_bad_sentence_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, 'lol') + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [[]]) + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [5]) + + def test_parse_sentence_words_hides_spaces(self): + og_sentence = ['\r\n', 'hello', ' ', ' ', '\t\n ', 'lol', ' ', 'spaces'] + self.sentence.parse(og_sentence) + self.assertEquals(self.sentence.words, ['hello', 'lol', 'spaces']) + self.assertEquals(self.sentence.dump(), ['hello', 'lol', 'spaces']) + self.assertEquals(self.sentence.dump(True), og_sentence) + + def test_parse_sentence_with_add_spaces(self): + self.sentence.parse(['hi', 'there'], add_spaces=True) + self.assertEquals(self.sentence.dump(True), ['hi', ' ', 'there']) + self.sentence.parse(['one', ' ', 'space', 'none'], add_spaces=True) + self.assertEquals(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none']) + + def test_iterate(self): + expected = [['1', '2', '3']] + self.sentence.parse(['1', ' ', '2', ' ', '3']) + for i, sentence in enumerate(self.sentence.iterate()): + self.assertEquals(sentence.dump(), expected[i]) + + def test_set_tabs(self): + self.sentence.parse(['tabs', 'pls'], add_spaces=True) + self.sentence.set_tabs() + self.assertEquals(self.sentence.dump(True)[0], '\n ') + self.sentence.parse(['tabs', 'pls'], add_spaces=True) + + def test_get_tabs(self): + self.sentence.parse(['no', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '') + self.sentence.parse(['\n \n ', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), ' ') + self.sentence.parse(['\n\t ', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '\t ') + self.sentence.parse(['\n\t \n', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '') + +class BlockTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Block + self.bloc = Block(None) + self.name = ['server', 'name'] + self.contents = [['thing', '1'], ['thing', '2'], ['another', 'one']] + self.bloc.parse([self.name, self.contents]) + + def test_iterate(self): + # Iterates itself normally + self.assertEquals(self.bloc, next(self.bloc.iterate())) + # Iterates contents while expanded + expected = [self.bloc.dump()] + self.contents + for i, elem in enumerate(self.bloc.iterate(expanded=True)): + self.assertEquals(expected[i], elem.dump()) + + def test_iterate_match(self): + # can match on contents while expanded + from certbot_nginx.parser_obj import Block, Sentence + expected = [['thing', '1'], ['thing', '2']] + for i, elem in enumerate(self.bloc.iterate(expanded=True, + match=lambda x: isinstance(x, Sentence) and 'thing' in x.words)): + self.assertEquals(expected[i], elem.dump()) + # can match on self + self.assertEquals(self.bloc, next(self.bloc.iterate( + expanded=True, + match=lambda x: isinstance(x, Block) and 'server' in x.names))) + + def test_parse_with_added_spaces(self): + import copy + self.bloc.parse([copy.copy(self.name), self.contents], add_spaces=True) + self.assertEquals(self.bloc.dump(), [self.name, self.contents]) + self.assertEquals(self.bloc.dump(True), [ + ['server', ' ', 'name', ' '], + [['thing', ' ', '1'], + ['thing', ' ', '2'], + ['another', ' ', 'one']]]) + + def test_bad_parse_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, [[[]], [[]]]) + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['lol']) + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['fake', 'news']) + + def test_set_tabs(self): + self.bloc.set_tabs() + self.assertEquals(self.bloc.names.dump(True)[0], '\n ') + for elem in self.bloc.contents.dump(True)[:-1]: + self.assertEquals(elem[0], '\n ') + self.assertEquals(self.bloc.contents.dump(True)[-1][0], '\n') + + def test_get_tabs(self): + self.bloc.parse([[' \n \t', 'lol'], []]) + self.assertEquals(self.bloc.get_tabs(), ' \t') + +class StatementsTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Statements + self.statements = Statements(None) + self.raw = [ + ['sentence', 'one'], + ['sentence', 'two'], + ['and', 'another'] + ] + self.raw_spaced = [ + ['\n ', 'sentence', ' ', 'one'], + ['\n ', 'sentence', ' ', 'two'], + ['\n ', 'and', ' ', 'another'], + '\n\n' + ] + + def test_set_tabs(self): + self.statements.parse(self.raw) + self.statements.set_tabs() + for statement in self.statements.iterate(): + self.assertEquals(statement.dump(True)[0], '\n ') + + def test_set_tabs_with_parent(self): + # Trailing whitespace should inherit from parent tabbing. + self.statements.parse(self.raw) + self.statements.parent = mock.Mock() + self.statements.parent.get_tabs.return_value = '\t\t' + self.statements.set_tabs() + for statement in self.statements.iterate(): + self.assertEquals(statement.dump(True)[0], '\n ') + self.assertEquals(self.statements.dump(True)[-1], '\n\t\t') + + def test_get_tabs(self): + self.raw[0].insert(0, '\n \n \t') + self.statements.parse(self.raw) + self.assertEquals(self.statements.get_tabs(), ' \t') + self.statements.parse([]) + self.assertEquals(self.statements.get_tabs(), '') + + def test_parse_with_added_spaces(self): + self.statements.parse(self.raw, add_spaces=True) + self.assertEquals(self.statements.dump(True)[0], ['sentence', ' ', 'one']) + + def test_parse_bad_list_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.statements.parse, 'lol not a list') + + def test_parse_hides_trailing_whitespace(self): + self.statements.parse(self.raw + ['\n\n ']) + self.assertTrue(isinstance(self.statements.dump()[-1], list)) + self.assertTrue(self.statements.dump(True)[-1].isspace()) + self.assertEquals(self.statements.dump(True)[-1], '\n\n ') + + def test_iterate(self): + self.statements.parse(self.raw) + expected = [['sentence', 'one'], ['sentence', 'two']] + for i, elem in enumerate(self.statements.iterate(match=lambda x: 'sentence' in x)): + self.assertEquals(expected[i], elem.dump()) + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index a8d35ed89..1e444fa26 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -25,6 +25,7 @@ certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" + omit_patterns="$omit_patterns,certbot-nginx/certbot_nginx/parser_obj.py" coverage run \ --append \ --source $sources \ From 1e8c13ebf905d1972e3c3c2474650520df1be921 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 19 Oct 2018 23:53:15 +0200 Subject: [PATCH 373/639] [Windows] Create the CI logic (#6374) So here we are: after #6361 has been merged, time is to provide an environment to execute the automated testing on Windows. Here are the assertions used to build the CI on Windows: every test running on Linux should ultimately be runnable on Windows, in a cross-platform compatible manner (there is one or two exception, when a test does not have any meaning for Windows), currently some tests are not runnable on Windows: theses tests are ignored by default when the environment is Windows using a custom decorator: @broken_on_windows, test environment should have functionalities similar to Travis, in particular an execution test matrix against various versions of Python and Windows, so test execution is done through AppVeyor, as it supports the requirements: it add a CI step along Travis and Codecov for each PR, all of this ensuring that Certbot is entirely functional on both Linux and Windows, code in tests can be changed, but code in Certbot should be changed as little as possible, to avoid regression risks. So far in this PR, I focused on the tests on Certbot core and ACME library. Concerning the plugins, it will be done later, for plugins which have an interest on Windows. Test are executed against Python 3.4, 3.5, 3.6 and 3.7, for Windows Server 2012 R2 and Windows Server 2016. I succeeded at making 258/259 of acme tests to work, and 828/868 of certbot core tests to work. Most of the errors where not because of Certbot itself, but because of how the tests are written. After redesigning some test utilitaries, and things like file path handling, or CRLF/LF, a lot of the errors vanished. I needed also to ignore a lot of IO errors typically occurring when a tearDown test process tries to delete a file before it has been closed: this kind of behavior is acceptable for Linux, but not for Windows. As a consequence, and until the tearDown process is improved, a lot of temporary files are not cleared on Windows after a test campaign. Remaining broken tests requires a more subtile approach to solve the errors, I will correct them progressively in future PR. Last words about tox. I did not used the existing tox.ini for now. It is just to far from what is supported on Windows: lot of bash scripts that should be rewritten completely, and that contain test logic not ready/relevant for Windows (plugin tests, Docker compilation/test, GNU distribution versatility handling and so on). So I use an independent file tox-win.ini for now, with the goal to merge it ultimately with the existing logic. * Define a tox configuration for windows, to execute tests against Python 3.4, 3.5, 3.6 and 3.7 + code coverage on Codecov.io * Correct windows compatibility on certbot codebase * Correct windows compatibility on certbot display functionalities * Correct windows compatibility on certbot plugins * Correct test utils to run tests on windows. Add decorator to skip (permanently) or mark broken (temporarily) tests on windows * Correct tests on certbot core to run them both on windows and linux. Mark some of them as broken on windows for now. * Lock tests are completely skipped on windows. Planned to be replace in next PR. * Correct tests on certbot display to run them both on windows and linux. Mark some of them as broken on windows for now. * Correct test utils for acme on windows. Add decorator to skip (permanently) or mark broken (temporarily) tests on windows. * Correct acme tests to run them both on windows and linux. Allow a reduction of code coverage of 1% on acme code base. * Create AppVeyor CI for Certbot on Windows, to run the test matrix (py34,35,36,37+coverage) on Windows Server 2012 R2 and Windows Server 2016. * Update changelog with Windows compatibility of Certbot. * Corrections about tox, pyreadline and CI logic * Correct english * Some corrections for acme * Newlines corrections * Remove changelog * Use os.devnull instead of /dev/null to be used on Windows * Uid is a always a number now. * Correct linting * PR https://github.com/python/typeshed/pull/2136 has been merge to third-party upstream 6 months ago, so code patch can be removed. * And so acme coverage should be 100% again. * More compatible tests Windows+Linux * Use stable line separator * Remove unused import * Do not rely on pytest in certbot tests * Use json.dumps to another json embedding weird characters * Change comment * Add import * Test rolling builds #1 * Test rolling builds #2 * Correction on json serialization * It seems that rolling builds are not canceling jobs on PR. Revert back to fail fast code in the pipeline. --- acme/acme/client_test.py | 4 +- acme/acme/crypto_util.py | 10 +-- acme/acme/standalone_test.py | 18 +++--- acme/acme/test_util.py | 9 +++ appveyor.yml | 30 +++++++-- certbot/compat.py | 12 +++- certbot/crypto_util.py | 7 ++- certbot/display/completer.py | 4 +- certbot/display/util.py | 2 +- certbot/main.py | 3 +- certbot/plugins/manual_test.py | 16 +++-- certbot/plugins/util_test.py | 7 ++- certbot/plugins/webroot_test.py | 12 ++-- certbot/renewal.py | 2 +- certbot/tests/account_test.py | 10 +++ certbot/tests/cert_manager_test.py | 4 +- certbot/tests/cli_test.py | 1 + certbot/tests/crypto_util_test.py | 3 +- certbot/tests/display/completer_test.py | 13 ++-- certbot/tests/display/util_test.py | 6 +- certbot/tests/error_handler_test.py | 10 +++ certbot/tests/hook_test.py | 1 + certbot/tests/lock_test.py | 2 + certbot/tests/log_test.py | 3 +- certbot/tests/main_test.py | 81 ++++++++++++++++--------- certbot/tests/reverter_test.py | 10 +++ certbot/tests/storage_test.py | 1 + certbot/tests/util.py | 46 +++++++++++++- certbot/tests/util_test.py | 28 ++++----- certbot/util.py | 4 +- tox-win.ini | 13 ++++ 31 files changed, 269 insertions(+), 103 deletions(-) create mode 100644 tox-win.ini diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..acfd7eaff 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1038,8 +1038,8 @@ class ClientNetworkTest(unittest.TestCase): # Requests Library Exceptions except requests.exceptions.ConnectionError as z: #pragma: no cover - self.assertEqual("('Connection aborted.', " - "error(111, 'Connection refused'))", str(z)) + self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))" + == str(z) or "[WinError 10061]" in str(z)) class ClientNetworkWithMockedResponseTest(unittest.TestCase): """Tests for acme.client.ClientNetwork which mock out response.""" diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d0e203417..c88cab943 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -136,22 +136,16 @@ def probe_sni(name, host, port=443, timeout=300, socket_kwargs = {'source_address': source_address} - host_protocol_agnostic = host - if host == '::' or host == '0': - # https://github.com/python/typeshed/pull/2136 - # while PR is not merged, we need to ignore - host_protocol_agnostic = None - try: # pylint: disable=star-args logger.debug( - "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, + "Attempting to connect to %s:%d%s.", host, port, " from {0}:{1}".format( source_address[0], source_address[1] ) if socket_kwargs else "" ) - socket_tuple = (host_protocol_agnostic, port) # type: Tuple[Optional[str], int] + socket_tuple = (host, port) # type: Tuple[str, int] sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 6beea038e..ee527782a 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -48,7 +48,7 @@ class TLSSNI01ServerTest(unittest.TestCase): test_util.load_cert('rsa2048_cert.pem'), )} from acme.standalone import TLSSNI01Server - self.server = TLSSNI01Server(("", 0), certs=self.certs) + self.server = TLSSNI01Server(('localhost', 0), certs=self.certs) # pylint: disable=no-member self.thread = threading.Thread(target=self.server.serve_forever) self.thread.start() @@ -133,8 +133,11 @@ class BaseDualNetworkedServersTest(unittest.TestCase): self.address_family = socket.AF_INET socketserver.TCPServer.__init__(self, *args, **kwargs) if ipv6: + # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing. + # We use the corresponding value (41) instead. + level = getattr(socket, "IPPROTO_IPV6", 41) # pylint: disable=no-member - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) + self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1) try: self.server_bind() self.server_activate() @@ -147,15 +150,15 @@ class BaseDualNetworkedServersTest(unittest.TestCase): mock_bind.side_effect = socket.error from acme.standalone import BaseDualNetworkedServers self.assertRaises(socket.error, BaseDualNetworkedServers, - BaseDualNetworkedServersTest.SingleProtocolServer, - ("", 0), - socketserver.BaseRequestHandler) + BaseDualNetworkedServersTest.SingleProtocolServer, + ('', 0), + socketserver.BaseRequestHandler) def test_ports_equal(self): from acme.standalone import BaseDualNetworkedServers servers = BaseDualNetworkedServers( BaseDualNetworkedServersTest.SingleProtocolServer, - ("", 0), + ('', 0), socketserver.BaseRequestHandler) socknames = servers.getsocknames() prev_port = None @@ -177,7 +180,7 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase): test_util.load_cert('rsa2048_cert.pem'), )} from acme.standalone import TLSSNI01DualNetworkedServers - self.servers = TLSSNI01DualNetworkedServers(("", 0), certs=self.certs) + self.servers = TLSSNI01DualNetworkedServers(('localhost', 0), certs=self.certs) self.servers.serve_forever() def tearDown(self): @@ -245,6 +248,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@test_util.broken_on_windows class TestSimpleTLSSNI01Server(unittest.TestCase): """Tests for acme.standalone.simple_tls_sni_01_server.""" diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index 1a0b67056..f97614700 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -4,6 +4,7 @@ """ import os +import sys import pkg_resources import unittest @@ -94,3 +95,11 @@ def skip_unless(condition, reason): # pragma: no cover return lambda cls: cls else: return lambda cls: None + +def broken_on_windows(function): + """Decorator to skip temporarily a broken test on Windows.""" + reason = 'Test is broken and ignored on windows but should be fixed.' + return unittest.skipIf( + sys.platform == 'win32' + and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', + reason)(function) diff --git a/appveyor.yml b/appveyor.yml index 67ad67c16..2d1c463ac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,12 +1,34 @@ -# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2 +image: + # => Windows Server 2012 R2 + - Visual Studio 2015 + # => Windows Server 2016 + - Visual Studio 2017 + branches: only: - master - - /^\d+\.\d+\.x$/ # version branches like X.X.X + - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +install: + # Fail fast if a newer build is queued for the same PR + - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + throw "There are newer queued builds for this pull request, failing early." } + # Use Python 3.7 by default + - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" + # Check env + - "python --version" + # Upgrade pip to avoid warnings + - "python -m pip install --upgrade pip" + # Ready to install tox and coverage + - "pip install tox codecov" + build: off test_script: - - ps: Write-Host "Hello, world!" - \ No newline at end of file + - tox -c tox-win.ini -e py34,py35,py36,py37-cover + +on_success: + - codecov diff --git a/certbot/compat.py b/certbot/compat.py index 1dc89dfd8..e3e1bc4e1 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -2,7 +2,7 @@ Compatibility layer to run certbot both on Linux and Windows. The approach used here is similar to Modernizr for Web browsers. -We do not check the plateform type to determine if a particular logic is supported. +We do not check the platform type to determine if a particular logic is supported. Instead, we apply a logic, and then fallback to another logic if first logic is not supported at runtime. @@ -13,6 +13,7 @@ import select import sys import errno import ctypes +import stat from certbot import errors @@ -138,3 +139,12 @@ def release_locked_file(fd, path): raise finally: os.close(fd) + +def compare_file_modes(mode1, mode2): + """Return true if the two modes can be considered as equals for this platform""" + if 'fcntl' in sys.modules: + # Linux specific: standard compare + return oct(stat.S_IMODE(mode1)) == oct(stat.S_IMODE(mode2)) + # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights. + return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD + and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 6193a8fbf..942f8502f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -449,14 +449,17 @@ def _notAfterBefore(cert_path, method): def sha256sum(filename): """Compute a sha256sum of a file. + NB: In given file, platform specific newlines characters will be converted + into their equivalent unicode counterparts before calculating the hash. + :param str filename: path to the file whose hash will be computed :returns: sha256 digest of the file in hexadecimal :rtype: str """ sha256 = hashlib.sha256() - with open(filename, 'rb') as f: - sha256.update(f.read()) + with open(filename, 'rU') as file_d: + sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() def cert_and_chain_from_fullchain(fullchain_pem): diff --git a/certbot/display/completer.py b/certbot/display/completer.py index 08b55fdea..509a1051a 100644 --- a/certbot/display/completer.py +++ b/certbot/display/completer.py @@ -49,9 +49,9 @@ class Completer(object): readline.set_completer(self.complete) readline.set_completer_delims(' \t\n;') - # readline can be implemented using GNU readline or libedit + # readline can be implemented using GNU readline, pyreadline or libedit # which have different configuration syntax - if 'libedit' in readline.__doc__: + if readline.__doc__ is not None and 'libedit' in readline.__doc__: readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') diff --git a/certbot/display/util.py b/certbot/display/util.py index 9a813d4b7..772b67d74 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -53,7 +53,7 @@ def _wrap_lines(msg): break_long_words=False, break_on_hyphens=False)) - return os.linesep.join(fixed_l) + return '\n'.join(fixed_l) def input_with_timeout(prompt=None, timeout=36000.0): diff --git a/certbot/main.py b/certbot/main.py index 6cd2bbfac..5d5251dd2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1135,7 +1135,8 @@ def _csr_get_and_save_cert(config, le_client): "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - cert, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, os.path.normpath(config.cert_path), + os.path.normpath(config.chain_path), os.path.normpath(config.fullchain_path)) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ba4eb6889..0938e8a7d 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -4,6 +4,7 @@ import unittest import six import mock +import sys from acme import challenges @@ -75,12 +76,14 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = ( - 'echo ${CERTBOT_DOMAIN}; ' - 'echo ${CERTBOT_TOKEN:-notoken}; ' - 'echo ${CERTBOT_CERT_PATH:-nocert}; ' - 'echo ${CERTBOT_KEY_PATH:-nokey}; ' - 'echo ${CERTBOT_SNI_DOMAIN:-nosnidomain}; ' - 'echo ${CERTBOT_VALIDATION:-novalidation};') + '{0} -c "from __future__ import print_function;' + 'import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' + 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' + 'print(os.environ.get(\'CERTBOT_CERT_PATH\', \'nocert\'));' + 'print(os.environ.get(\'CERTBOT_KEY_PATH\', \'nokey\'));' + 'print(os.environ.get(\'CERTBOT_SNI_DOMAIN\', \'nosnidomain\'));' + 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' + .format(sys.executable)) dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format( self.dns_achall.domain, 'notoken', 'nocert', 'nokey', 'nosnidomain', @@ -128,6 +131,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): achall.validation(achall.account_key) in args[0]) self.assertFalse(kwargs['wrap']) + @test_util.broken_on_windows def test_cleanup(self): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = 'echo foo;' diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index b2f9c79ea..8ecd380b8 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -4,13 +4,14 @@ import unittest import mock - class GetPrefixTest(unittest.TestCase): """Tests for certbot.plugins.get_prefixes.""" def test_get_prefix(self): from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes('/'), ['/']) + self.assertEqual( + get_prefixes('/a/b/c'), + [os.path.normpath(path) for path in ['/a/b/c', '/a/b', '/a', '/']]) + self.assertEqual(get_prefixes('/'), [os.path.normpath('/')]) self.assertEqual(get_prefixes('a'), ['a']) class PathSurgeryTest(unittest.TestCase): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 59133f0aa..5303fe4da 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -4,9 +4,9 @@ from __future__ import print_function import argparse import errno +import json import os import shutil -import stat import tempfile import unittest @@ -17,6 +17,7 @@ import six from acme import challenges from certbot import achallenges +from certbot import compat from certbot import errors from certbot.display import util as display_util @@ -142,6 +143,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.auth.perform, []) os.chmod(self.path, 0o700) + @test_util.skip_on_windows('On Windows, there is no chown.') @mock.patch("certbot.plugins.webroot.os.chown") def test_failed_chown(self, mock_chown): mock_chown.side_effect = OSError(errno.EACCES, "msg") @@ -169,16 +171,14 @@ class AuthenticatorTest(unittest.TestCase): # Remove exec bit from permission check, so that it # matches the file self.auth.perform([self.achall]) - path_permissions = stat.S_IMODE(os.stat(self.validation_path).st_mode) - self.assertEqual(path_permissions, 0o644) + self.assertTrue(compat.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644)) # Check permissions of the directories for dirpath, dirnames, _ in os.walk(self.path): for directory in dirnames: full_path = os.path.join(dirpath, directory) - dir_permissions = stat.S_IMODE(os.stat(full_path).st_mode) - self.assertEqual(dir_permissions, 0o755) + self.assertTrue(compat.compare_file_modes(os.stat(full_path).st_mode, 0o755)) parent_gid = os.stat(self.path).st_gid parent_uid = os.stat(self.path).st_uid @@ -274,7 +274,7 @@ class WebrootActionTest(unittest.TestCase): def test_webroot_map_action(self): args = self.parser.parse_args( - ["--webroot-map", '{{"thing.com":"{0}"}}'.format(self.path)]) + ["--webroot-map", json.dumps({'thing.com': self.path})]) self.assertEqual(args.webroot_map["thing.com"], self.path) def test_domain_before_webroot(self): diff --git a/certbot/renewal.py b/certbot/renewal.py index ecc8b1f2f..a1508fa60 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -301,7 +301,7 @@ def renew_cert(config, domains, le_client, lineage): domains = lineage.names() # The private key is the existing lineage private key if reuse_key is set. # Otherwise, generate a fresh private key by passing None. - new_key = lineage.privkey if config.reuse_key else None + new_key = os.path.normpath(lineage.privkey) if config.reuse_key else None new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index b2be47d0f..278b0c545 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -116,6 +116,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_init_creates_dir(self): self.assertTrue(os.path.isdir(self.config.accounts_dir)) + @test_util.broken_on_windows def test_save_and_restore(self): self.storage.save(self.acc, self.mock_client) account_path = os.path.join(self.config.accounts_dir, self.acc.id) @@ -218,12 +219,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_version_staging(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([self.acc], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_version_production(self): self._set_server('https://acme-v01.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -241,6 +244,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_load(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -249,6 +253,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + @test_util.broken_on_windows def test_upgrade_load_single_account(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -273,6 +278,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): errors.AccountStorageError, self.storage.save, self.acc, self.mock_client) + @test_util.broken_on_windows def test_delete(self): self.storage.save(self.acc, self.mock_client) self.storage.delete(self.acc.id) @@ -307,10 +313,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + @test_util.broken_on_windows def test_delete_folders_up(self): self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + @test_util.broken_on_windows def test_delete_folders_down(self): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() @@ -320,10 +328,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase): with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: f.write('bar') + @test_util.broken_on_windows def test_delete_shared_account_up(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + @test_util.broken_on_windows def test_delete_shared_account_down(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 6ec1d4f5c..1aef33b0c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -204,7 +204,7 @@ class CertificatesTest(BaseCertManagerTest): shutil.rmtree(empty_tempdir) @mock.patch('certbot.cert_manager.ocsp.RevocationChecker.ocsp_revoked') - def test_report_human_readable(self, mock_revoked): + def test_report_human_readable(self, mock_revoked): #pylint: disable=too-many-statements mock_revoked.return_value = None from certbot import cert_manager import datetime, pytz @@ -228,7 +228,7 @@ class CertificatesTest(BaseCertManagerTest): cert.target_expiry += datetime.timedelta(hours=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 hour(s)' in out) + self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out) self.assertTrue('VALID' in out and not 'INVALID' in out) cert.target_expiry += datetime.timedelta(days=1) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 979cd97c1..69ef16597 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -76,6 +76,7 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods return output.getvalue() + @test_util.broken_on_windows @mock.patch("certbot.cli.flag_default") def test_cli_ini_domains(self, mock_flag_default): tmp_config = tempfile.NamedTemporaryFile() diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index baf14b2ef..c092efc51 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -140,7 +140,7 @@ class ImportCSRFileTest(unittest.TestCase): util.CSR(file=csrfile, data=data_pem, form="pem"), - ["Example.com"],), + ["Example.com"]), self._call(csrfile, data)) def test_pem_csr(self): @@ -376,7 +376,6 @@ class NotAfterTest(unittest.TestCase): class Sha256sumTest(unittest.TestCase): """Tests for certbot.crypto_util.notAfter""" - def test_sha256sum(self): from certbot.crypto_util import sha256sum self.assertEqual(sha256sum(CERT_PATH), diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index ac01103b8..455bf5e1e 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,6 +1,9 @@ """Test certbot.display.completer.""" import os -import readline +try: + import readline # pylint: disable=import-error +except ImportError: + import certbot.display.dummy_readline as readline # type: ignore import string import sys import unittest @@ -9,9 +12,9 @@ import mock from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from certbot.tests.util import TempDirTestCase +import certbot.tests.util as test_util -class CompleterTest(TempDirTestCase): +class CompleterTest(test_util.TempDirTestCase): """Test certbot.display.completer.Completer.""" def setUp(self): @@ -47,6 +50,8 @@ class CompleterTest(TempDirTestCase): completion = my_completer.complete(self.tempdir, num_paths) self.assertEqual(completion, None) + @unittest.skipIf('readline' not in sys.modules, + reason='Not relevant if readline is not available.') def test_import_error(self): original_readline = sys.modules['readline'] sys.modules['readline'] = None @@ -91,7 +96,7 @@ class CompleterTest(TempDirTestCase): def enable_tab_completion(unused_command): """Enables readline tab completion using the system specific syntax.""" - libedit = 'libedit' in readline.__doc__ + libedit = readline.__doc__ is not None and 'libedit' in readline.__doc__ command = 'bind ^I rl_complete' if libedit else 'tab: complete' readline.parse_and_bind(command) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index f5cf29047..5672a20bd 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,6 +1,5 @@ """Test :mod:`certbot.display.util`.""" import inspect -import os import socket import tempfile import unittest @@ -10,7 +9,6 @@ import mock from certbot import errors from certbot import interfaces - from certbot.display import util as display_util @@ -281,10 +279,10 @@ class FileOutputDisplayTest(unittest.TestCase): msg = ("This is just a weak test{0}" "This function is only meant to be for easy viewing{0}" "Test a really really really really really really really really " - "really really really really long line...".format(os.linesep)) + "really really really really long line...".format('\n')) text = display_util._wrap_lines(msg) - self.assertEqual(text.count(os.linesep), 3) + self.assertEqual(text.count('\n'), 3) def test_get_valid_int_ans_valid(self): # pylint: disable=protected-access diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index a4a65e2d4..8508a3df5 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -10,6 +10,7 @@ import mock from acme.magic_typing import Callable, Dict, Union # pylint: enable=unused-import, no-name-in-module +import certbot.tests.util as test_util def get_signals(signums): """Get the handlers for an iterable of signums.""" @@ -65,6 +66,8 @@ class ErrorHandlerTest(unittest.TestCase): self.init_func.assert_called_once_with(*self.init_args, **self.init_kwargs) + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows def test_context_manager_with_signal(self): init_signals = get_signals(self.signals) with signal_receiver(self.signals) as signals_received: @@ -95,6 +98,8 @@ class ErrorHandlerTest(unittest.TestCase): **self.init_kwargs) bad_func.assert_called_once_with() + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows def test_bad_recovery_with_signal(self): sig1 = self.signals[0] sig2 = self.signals[-1] @@ -144,5 +149,10 @@ class ExitHandlerTest(ErrorHandlerTest): **self.init_kwargs) func.assert_called_once_with() + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows + def test_bad_recovery_with_signal(self): + super(ExitHandlerTest, self).test_bad_recovery_with_signal() + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index c9cfc69f9..f5bb0c8b5 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -37,6 +37,7 @@ class ValidateHookTest(util.TempDirTestCase): from certbot.hooks import validate_hook return validate_hook(*args, **kwargs) + @util.broken_on_windows def test_not_executable(self): file_path = os.path.join(self.tempdir, "foo") # create a non-executable file diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 51469e8c1..aa82701f3 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -10,6 +10,7 @@ from certbot import errors from certbot.tests import util as test_util +@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -24,6 +25,7 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) +@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index c5991347e..6588bf5ca 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -12,6 +12,7 @@ import six from acme import messages from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import constants from certbot import errors from certbot import util @@ -259,7 +260,7 @@ class TempHandlerTest(unittest.TestCase): def test_permissions(self): self.assertTrue( - util.check_permissions(self.handler.path, 0o600, os.getuid())) + util.check_permissions(self.handler.path, 0o600, compat.os_geteuid())) def test_delete(self): self.handler.close() diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8334068c9..8d6a3e7ae 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -4,6 +4,7 @@ from __future__ import print_function import itertools +import json import mock import os import shutil @@ -11,6 +12,8 @@ import traceback import unittest import datetime import pytz +import tempfile +import sys import josepy as jose import six @@ -588,12 +591,14 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) + @test_util.broken_on_windows def test_noninteractive(self): args = ['-n', 'certonly'] self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") + @test_util.broken_on_windows @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @mock.patch('certbot.main._determine_account') @@ -635,43 +640,46 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_certname(self, _inst, _rec, mock_install): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install', '--cert-name', 'whatever'], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('privkey')) + @test_util.broken_on_windows @mock.patch('certbot.main._install_cert') @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_param_override(self, _inst, _rec, mock_install): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install', '--cert-name', 'whatever', - '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + '--key-path', test_util.temp_join('overriding_privkey')], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.chain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.chain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('overriding_privkey')) mock_install.reset() self._call(['install', '--cert-name', 'whatever', - '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + '--cert-path', test_util.temp_join('overriding_cert')], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('overriding_cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('privkey')) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') @@ -686,15 +694,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.cert_manager.get_certnames') @mock.patch('certbot.main._install_cert') def test_installer_select_cert(self, mock_inst, mock_getcert, _inst, _rec): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install'], mockisfile=True) self.assertTrue(mock_getcert.called) self.assertTrue(mock_inst.called) + @test_util.broken_on_windows @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -710,7 +720,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met # ret, _, _, _ = self._call(args) # self.assertTrue("Too many flags setting" in ret) - args = ["install", "--nginx", "--cert-path", "/tmp/blah", "--key-path", "/tmp/blah", + args = ["install", "--nginx", "--cert-path", + test_util.temp_join('blah'), "--key-path", test_util.temp_join('blah'), "--nginx-server-root", "/nonexistent/thing", "-d", "example.com", "--debug"] if "nginx" in real_plugins: @@ -733,6 +744,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(["auth", "--standalone"]) self.assertEqual(1, mock_certonly.call_count) + @test_util.broken_on_windows def test_rollback(self): _, _, _, client = self._call(['rollback']) self.assertEqual(1, client.rollback.call_count) @@ -760,6 +772,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['delete']) self.assertEqual(1, mock_cert_manager.call_count) + @test_util.broken_on_windows def test_plugins(self): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( @@ -1014,7 +1027,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met # The location of the previous live privkey.pem is passed # to obtain_certificate mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], - os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem")) + os.path.normpath(os.path.join( + self.config.config_dir, "live/sample-renewal/privkey.pem"))) else: mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None) else: @@ -1039,6 +1053,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('fullchain.pem' in cert_msg) self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) + @test_util.broken_on_windows @mock.patch('certbot.crypto_util.notAfter') def test_certonly_renewal_triggers(self, unused_notafter): # --dry-run should force renewal @@ -1087,6 +1102,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('No renewals were attempted.' in stdout.getvalue()) self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + @test_util.broken_on_windows def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] @@ -1204,7 +1220,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met renewalparams = {'authenticator': 'webroot'} self._test_renew_common( renewalparams=renewalparams, assert_oc_called=True, - args=['renew', '--webroot-map', '{"example.com": "/tmp"}']) + args=['renew', '--webroot-map', json.dumps({'example.com': tempfile.gettempdir()})]) def test_renew_reconstitute_error(self): # pylint: disable=protected-access @@ -1234,7 +1250,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_no_renewal_with_hooks(self): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, - args=['renew', '--post-hook', 'echo hello world']) + args=['renew', '--post-hook', + '{0} -c "from __future__ import print_function; print(\'hello world\');"' + .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) @test_util.patch_get_utility() @@ -1254,13 +1272,19 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met chain = 'chain' mock_client = mock.MagicMock() mock_client.obtain_certificate_from_csr.return_value = (certr, chain) - cert_path = '/etc/letsencrypt/live/example.com/cert_512.pem' - full_path = '/etc/letsencrypt/live/example.com/fullchain.pem' + cert_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/cert_512.pem')) + full_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/fullchain.pem')) mock_client.save_certificate.return_value = cert_path, None, full_path with mock.patch('certbot.main._init_le_client') as mock_init: mock_init.return_value = mock_client with test_util.patch_get_utility() as mock_get_utility: - chain_path = '/etc/letsencrypt/live/example.com/chain.pem' + chain_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/chain.pem')) args = ('-a standalone certonly --csr {0} --cert-path {1} ' '--chain-path {2} --fullchain-path {3}').format( CSR, cert_path, chain_path, full_path).split() @@ -1334,6 +1358,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(['-c', test_util.vector_path('cli.ini')]) self.assertTrue(mocked_run.called) + @test_util.broken_on_windows def test_register(self): with mock.patch('certbot.main.client') as mocked_client: acc = mock.MagicMock() diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index b048737c2..999c6225c 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -50,6 +50,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): x = f.read() self.assertTrue("No changes" in x) + @test_util.broken_on_windows def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -91,6 +92,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, set([config3]), "invalid save") + @test_util.broken_on_windows def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") update_file(self.config1, "updated-directive") @@ -120,6 +122,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertFalse(os.path.isfile(config3)) self.assertFalse(os.path.isfile(config4)) + @test_util.broken_on_windows def test_multiple_registration_same_file(self): self.reverter.register_file_creation(True, self.config1) self.reverter.register_file_creation(True, self.config1) @@ -144,6 +147,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_file_creation, "filepath") + @test_util.broken_on_windows def test_register_undo_command(self): coms = [ ["a2dismod", "ssl"], @@ -166,6 +170,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_undo_command, True, ["command"]) + @test_util.broken_on_windows @mock.patch("certbot.util.run_script") def test_run_undo_commands(self, mock_run): mock_run.side_effect = ["", errors.SubprocessError] @@ -229,6 +234,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) + @test_util.broken_on_windows @mock.patch("certbot.reverter.logger.warning") def test_recover_checkpoint_missing_new_files(self, mock_warn): self.reverter.register_file_creation( @@ -243,6 +249,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) + @test_util.broken_on_windows def test_recovery_routine_temp_and_perm(self): # Register a new perm checkpoint file config3 = os.path.join(self.dir1, "config3.txt") @@ -306,6 +313,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.rollback_checkpoints, "one") + @test_util.broken_on_windows def test_rollback_finalize_checkpoint_valid_inputs(self): config3 = self._setup_three_checkpoints() @@ -357,6 +365,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") + @test_util.broken_on_windows @mock.patch("certbot.reverter.logger") def test_rollback_too_many(self, mock_logger): # Test no exist warning... @@ -369,6 +378,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.reverter.rollback_checkpoints(4) self.assertEqual(mock_logger.warning.call_count, 1) + @test_util.broken_on_windows def test_multi_rollback(self): config3 = self._setup_three_checkpoints() self.reverter.rollback_checkpoints(3) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 078a2858f..03d595652 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -480,6 +480,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(self.test_rc.should_autorenew()) mock_ocsp.return_value = False + @test_util.broken_on_windows @mock.patch("certbot.storage.relevant_values") def test_save_successor(self, mock_rv): # Mock relevant_values() to claim that all values are relevant here diff --git a/certbot/tests/util.py b/certbot/tests/util.py index d505ea76c..822597dd4 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -9,6 +9,8 @@ import pkg_resources import shutil import tempfile import unittest +import sys +import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -36,8 +38,15 @@ def vector_path(*names): def load_vector(*names): """Load contents of a test vector.""" # luckily, resource_string opens file in binary mode - return pkg_resources.resource_string( + data = pkg_resources.resource_string( __name__, os.path.join('testdata', *names)) + # Try at most to convert CRLF to LF when data is text + try: + return data.decode().replace('\r\n', '\n').encode() + except ValueError: + # Failed to process the file with standard encoding. + # Most likely not a text file, return its bytes untouched. + return data def _guess_loader(filename, loader_pem, loader_der): @@ -314,10 +323,21 @@ class TempDirTestCase(unittest.TestCase): """Base test class which sets up and tears down a temporary directory""" def setUp(self): + """Execute before test""" self.tempdir = tempfile.mkdtemp() def tearDown(self): - shutil.rmtree(self.tempdir) + """Execute after test""" + # Then we have various files which are not correctly closed at the time of tearDown. + # On Windows, it is visible for the same reasons as above. + # For know, we log them until a proper file close handling is written. + def onerror_handler(_, path, excinfo): + """On error handler""" + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) + shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object. @@ -378,3 +398,25 @@ def hold_lock(cv, lock_path): # pragma: no cover cv.notify() cv.wait() my_lock.release() + +def skip_on_windows(reason): + """Decorator to skip permanently a test on Windows. A reason is required.""" + def wrapper(function): + """Wrapped version""" + return unittest.skipIf(sys.platform == 'win32', reason)(function) + return wrapper + +def broken_on_windows(function): + """Decorator to skip temporarily a broken test on Windows.""" + reason = 'Test is broken and ignored on windows but should be fixed.' + return unittest.skipIf( + sys.platform == 'win32' + and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', + reason)(function) + +def temp_join(path): + """ + Return the given path joined to the tempdir path for the current platform + Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows) + """ + return os.path.join(tempfile.gettempdir(), path) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 689b4108d..45cc55249 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -3,7 +3,6 @@ import argparse import errno import os import shutil -import stat import unittest import mock @@ -89,6 +88,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): import certbot.util reload_module(certbot.util) + @test_util.broken_on_windows @mock.patch('certbot.util.logger') @mock.patch('certbot.util.atexit_register') def test_it(self, mock_register, mock_logger): @@ -140,9 +140,9 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): super(MakeOrVerifyDirTest, self).setUp() self.path = os.path.join(self.tempdir, "foo") - os.mkdir(self.path, 0o400) + os.mkdir(self.path, 0o600) - self.uid = os.getuid() + self.uid = compat.os_geteuid() def _call(self, directory, mode): from certbot.util import make_or_verify_dir @@ -152,14 +152,15 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): path = os.path.join(self.tempdir, "bar") self._call(path, 0o650) self.assertTrue(os.path.isdir(path)) - self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o650) + self.assertTrue(compat.compare_file_modes(os.stat(path).st_mode, 0o650)) def test_existing_correct_mode_does_not_fail(self): - self._call(self.path, 0o400) - self.assertEqual(stat.S_IMODE(os.stat(self.path).st_mode), 0o400) + self._call(self.path, 0o600) + self.assertTrue(compat.compare_file_modes(os.stat(self.path).st_mode, 0o600)) + @test_util.skip_on_windows('Umask modes are mostly ignored on Windows.') def test_existing_wrong_mode_fails(self): - self.assertRaises(errors.Error, self._call, self.path, 0o600) + self.assertRaises(errors.Error, self._call, self.path, 0o400) def test_reraises_os_error(self): with mock.patch.object(os, "makedirs") as makedirs: @@ -178,7 +179,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def setUp(self): super(CheckPermissionsTest, self).setUp() - self.uid = os.getuid() + self.uid = compat.os_geteuid() def _call(self, mode): from certbot.util import check_permissions @@ -212,8 +213,8 @@ class UniqueFileTest(test_util.TempDirTestCase): self.assertEqual(open(name).read(), "bar") def test_right_mode(self): - self.assertEqual(0o700, os.stat(self._call(0o700)[1]).st_mode & 0o777) - self.assertEqual(0o100, os.stat(self._call(0o100)[1]).st_mode & 0o777) + self.assertTrue(compat.compare_file_modes(0o700, os.stat(self._call(0o700)[1]).st_mode)) + self.assertTrue(compat.compare_file_modes(0o600, os.stat(self._call(0o600)[1]).st_mode)) def test_default_exists(self): name1 = self._call()[1] # create 0000_foo.txt @@ -513,17 +514,16 @@ class OsInfoTest(unittest.TestCase): def test_systemd_os_release(self): from certbot.util import (get_os_info, get_systemd_os_info, - get_os_info_ua) + get_os_info_ua) with mock.patch('os.path.isfile', return_value=True): self.assertEqual(get_os_info( test_util.vector_path("os-release"))[0], 'systemdos') self.assertEqual(get_os_info( test_util.vector_path("os-release"))[1], '42') - self.assertEqual(get_systemd_os_info("/dev/null"), ("", "")) + self.assertEqual(get_systemd_os_info(os.devnull), ("", "")) self.assertEqual(get_os_info_ua( - test_util.vector_path("os-release")), - "SystemdOS") + test_util.vector_path("os-release")), "SystemdOS") with mock.patch('os.path.isfile', return_value=False): self.assertEqual(get_systemd_os_info(), ("", "")) diff --git a/certbot/util.py b/certbot/util.py index 8e84c29ba..d7c542465 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -12,7 +12,6 @@ import platform import re import six import socket -import stat import subprocess import sys @@ -21,6 +20,7 @@ from collections import OrderedDict import configargparse from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import constants from certbot import errors from certbot import lock @@ -204,7 +204,7 @@ def check_permissions(filepath, mode, uid=0): """ file_stat = os.stat(filepath) - return stat.S_IMODE(file_stat.st_mode) == mode and file_stat.st_uid == uid + return compat.compare_file_modes(file_stat.st_mode, mode) and file_stat.st_uid == uid def safe_open(path, mode="w", chmod=None, buffering=None): diff --git a/tox-win.ini b/tox-win.ini new file mode 100644 index 000000000..fe063c264 --- /dev/null +++ b/tox-win.ini @@ -0,0 +1,13 @@ +[tox] +skipsdist = True +envlist = py{34,35,36,37}-cover + +[testenv] +deps = -e acme[dev] + -e .[dev] +commands = pytest -n auto --pyargs acme + pytest -n auto --pyargs certbot + +[testenv:cover] +commands = pytest -n auto --cov acme --pyargs acme + pytest -n auto --cov certbot --cov-append --pyargs certbot From 7b17c84dd97eeab474ceb96c1093fc735381ee33 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 20 Oct 2018 02:37:22 +0200 Subject: [PATCH 374/639] Remove custom code for fail fast because rolling builds in AppVeyor are enabled. (#6431) --- appveyor.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2d1c463ac..796070081 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,11 +11,6 @@ branches: - /^test-.*$/ install: - # Fail fast if a newer build is queued for the same PR - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env From 36ebce4a5fa30a0e73bd8656aea54ded2e9d982e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 19 Oct 2018 19:16:54 -0700 Subject: [PATCH 375/639] Fix ranking of vhosts in Nginx so that all port-matching vhosts come first (#6412) To more closely match how Nginx ranks things. --- CHANGELOG.md | 1 + certbot-nginx/certbot_nginx/configurator.py | 44 ++++++++--- .../certbot_nginx/tests/configurator_test.py | 73 ++++++++++++------- 3 files changed, 78 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5060a7038..0f44258af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed * Match Nginx parser update in allowing variable names to start with `${`. +* Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index d526381a2..0f9c43e2d 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -8,7 +8,6 @@ import tempfile import time import OpenSSL -import six import zope.interface from acme import challenges @@ -32,6 +31,12 @@ from certbot_nginx import obj # pylint: disable=unused-import from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module +NAME_RANK = 0 +START_WILDCARD_RANK = 1 +END_WILDCARD_RANK = 2 +REGEX_RANK = 3 +NO_SSL_MODIFIER = 4 + logger = logging.getLogger(__name__) @@ -405,7 +410,8 @@ class NginxConfigurator(common.Installer): """ if not matches: return None - elif matches[0]['rank'] in six.moves.range(2, 6): + elif matches[0]['rank'] in [START_WILDCARD_RANK, END_WILDCARD_RANK, + START_WILDCARD_RANK + NO_SSL_MODIFIER, END_WILDCARD_RANK + NO_SSL_MODIFIER]: # Wildcard match - need to find the longest one rank = matches[0]['rank'] wildcards = [x for x in matches if x['rank'] == rank] @@ -414,10 +420,9 @@ class NginxConfigurator(common.Installer): # Exact or regex match return matches[0]['vhost'] - - def _rank_matches_by_name_and_ssl(self, vhost_list, target_name): + def _rank_matches_by_name(self, vhost_list, target_name): """Returns a ranked list of vhosts from vhost_list that match target_name. - The ranking gives preference to SSL vhosts. + This method should always be followed by a call to _select_best_name_match. :param list vhost_list: list of vhosts to filter and rank :param str target_name: The name to match @@ -437,21 +442,37 @@ class NginxConfigurator(common.Installer): if name_type == 'exact': matches.append({'vhost': vhost, 'name': name, - 'rank': 0 if vhost.ssl else 1}) + 'rank': NAME_RANK}) elif name_type == 'wildcard_start': matches.append({'vhost': vhost, 'name': name, - 'rank': 2 if vhost.ssl else 3}) + 'rank': START_WILDCARD_RANK}) elif name_type == 'wildcard_end': matches.append({'vhost': vhost, 'name': name, - 'rank': 4 if vhost.ssl else 5}) + 'rank': END_WILDCARD_RANK}) elif name_type == 'regex': matches.append({'vhost': vhost, 'name': name, - 'rank': 6 if vhost.ssl else 7}) + 'rank': REGEX_RANK}) return sorted(matches, key=lambda x: x['rank']) + def _rank_matches_by_name_and_ssl(self, vhost_list, target_name): + """Returns a ranked list of vhosts from vhost_list that match target_name. + The ranking gives preference to SSLishness before name match level. + + :param list vhost_list: list of vhosts to filter and rank + :param str target_name: The name to match + :returns: list of dicts containing the vhost, the matching name, and + the numerical rank + :rtype: list + + """ + matches = self._rank_matches_by_name(vhost_list, target_name) + for match in matches: + if not match['vhost'].ssl: + match['rank'] += NO_SSL_MODIFIER + return sorted(matches, key=lambda x: x['rank']) def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. @@ -531,9 +552,7 @@ class NginxConfigurator(common.Installer): matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] - # We can use this ranking function because sslishness doesn't matter to us, and - # there shouldn't be conflicting plaintextish servers listening on 80. - return self._rank_matches_by_name_and_ssl(matching_vhosts, target_name) + return self._rank_matches_by_name(matching_vhosts, target_name) def get_all_names(self): """Returns all names found in the Nginx Configuration. @@ -568,6 +587,7 @@ class NginxConfigurator(common.Installer): return util.get_filtered_names(all_names) def _get_snakeoil_paths(self): + """Generate invalid certs that let us create ssl directives for Nginx""" # TODO: generate only once tmp_dir = os.path.join(self.config.work_dir, "snakeoil") le_key = crypto_util.init_save_key( diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 4d23f3518..b180bb930 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -128,22 +128,39 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhosts(self): - localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) - server_conf = set(['somename', 'another.alias', 'alias']) - example_conf = set(['.example.com', 'example.*']) - foo_conf = set(['*.www.foo.com', '*.www.example.com']) - ipv6_conf = set(['ipv6.com']) + def test_choose_vhosts_alias(self): + self._test_choose_vhosts_common('alias', 'server_conf') - results = {'localhost': localhost_conf, - 'alias': server_conf, - 'example.com': example_conf, - 'example.com.uk.test': example_conf, - 'www.example.com': example_conf, - 'test.www.example.com': foo_conf, - 'abc.www.foo.com': foo_conf, - 'www.bar.co.uk': localhost_conf, - 'ipv6.com': ipv6_conf} + def test_choose_vhosts_example_com(self): + self._test_choose_vhosts_common('example.com', 'example_conf') + + def test_choose_vhosts_localhost(self): + self._test_choose_vhosts_common('localhost', 'localhost_conf') + + def test_choose_vhosts_example_com_uk_test(self): + self._test_choose_vhosts_common('example.com.uk.test', 'example_conf') + + def test_choose_vhosts_www_example_com(self): + self._test_choose_vhosts_common('www.example.com', 'example_conf') + + def test_choose_vhosts_test_www_example_com(self): + self._test_choose_vhosts_common('test.www.example.com', 'foo_conf') + + def test_choose_vhosts_abc_www_foo_com(self): + self._test_choose_vhosts_common('abc.www.foo.com', 'foo_conf') + + def test_choose_vhosts_www_bar_co_uk(self): + self._test_choose_vhosts_common('www.bar.co.uk', 'localhost_conf') + + def test_choose_vhosts_ipv6_com(self): + self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf') + + def _test_choose_vhosts_common(self, name, conf): + conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']), + 'server_conf': set(['somename', 'another.alias', 'alias']), + 'example_conf': set(['.example.com', 'example.*']), + 'foo_conf': set(['*.www.foo.com', '*.www.example.com']), + 'ipv6_conf': set(['ipv6.com'])} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -155,22 +172,22 @@ class NginxConfiguratorTest(util.NginxTest): 'www.bar.co.uk': "etc_nginx/nginx.conf", 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + vhost = self.config.choose_vhosts(name)[0] + path = os.path.relpath(vhost.filep, self.temp_dir) + + self.assertEqual(conf_names[conf], vhost.names) + self.assertEqual(conf_path[name], path) + # IPv6 specific checks + if name == "ipv6.com": + self.assertTrue(vhost.ipv6_enabled()) + # Make sure that we have SSL enabled also for IPv6 addr + self.assertTrue( + any([True for x in vhost.addrs if x.ssl and x.ipv6])) + + def test_choose_vhosts_bad(self): bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] - for name in results: - vhost = self.config.choose_vhosts(name)[0] - path = os.path.relpath(vhost.filep, self.temp_dir) - - self.assertEqual(results[name], vhost.names) - self.assertEqual(conf_path[name], path) - # IPv6 specific checks - if name == "ipv6.com": - self.assertTrue(vhost.ipv6_enabled()) - # Make sure that we have SSL enabled also for IPv6 addr - self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) - for name in bad_results: self.assertRaises(errors.MisconfigurationError, self.config.choose_vhosts, name) From 9264561944fc847e4a5d2de9a2be1c7da811349d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 29 Oct 2018 23:56:30 +0100 Subject: [PATCH 376/639] Check pattern for both old and new openssl (#6450) --- tests/certbot-boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index e250e591b..73e668e28 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -394,7 +394,7 @@ openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' # OCSP Must Staple common auth --must-staple --domains "must-staple.le.wtf" -openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep -E 'status_request|1\.3\.6\.1\.5\.5\.7\.1\.24' # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke From aad266369597281c79022cc4d807bc2be2b08b43 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 30 Oct 2018 17:13:40 -0700 Subject: [PATCH 377/639] Only error on DeprecationWarnings --- pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 085c7258e..2669c1105 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,4 @@ [pytest] -addopts = -Werror --numprocesses auto --pyargs +addopts = --numprocesses auto --pyargs +filterwarnings = + error::DeprecationWarning From b0b871083559e45b8f935748bb101783348613b4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 30 Oct 2018 17:24:29 -0700 Subject: [PATCH 378/639] Move back to using a warning whitelist --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 2669c1105..4a1dcca16 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] addopts = --numprocesses auto --pyargs filterwarnings = - error::DeprecationWarning + error From 1d783fd4b9c83c669b33adb2afaff8ec21e00cb6 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 31 Oct 2018 18:34:14 +0200 Subject: [PATCH 379/639] Update Augeas lens to fix some Apache configuration parsing issues (#6438) * Update Augeas lens to fix some Apache configuration parsing issues * Added CHANGELOG entry --- CHANGELOG.md | 1 + .../certbot_apache/augeas_lens/httpd.aug | 112 ++++++++++++++---- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f44258af..1e0ba82ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py +* The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. ## 0.27.1 - 2018-09-06 diff --git a/certbot-apache/certbot_apache/augeas_lens/httpd.aug b/certbot-apache/certbot_apache/augeas_lens/httpd.aug index 2729f4b60..5600088cf 100644 --- a/certbot-apache/certbot_apache/augeas_lens/httpd.aug +++ b/certbot-apache/certbot_apache/augeas_lens/httpd.aug @@ -44,67 +44,134 @@ autoload xfm *****************************************************************) let dels (s:string) = del s s +(* The continuation sequence that indicates that we should consider the + * next line part of the current line *) +let cont = /\\\\\r?\n/ + +(* Whitespace within a line: space, tab, and the continuation sequence *) +let ws = /[ \t]/ | cont + +(* Any possible character - '.' does not match \n *) +let any = /(.|\n)/ + +(* Any character preceded by a backslash *) +let esc_any = /\\\\(.|\n)/ + +(* Newline sequence - both for Unix and DOS newlines *) +let nl = /\r?\n/ + +(* Whitespace at the end of a line *) +let eol = del (ws* . nl) "\n" + (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " " -let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ "" -let sep_eq = del /[ \t]*=[ \t]*/ "=" +let sep_spc = del ws+ " " +let sep_osp = del ws* "" +let sep_eq = del (ws* . "=" . ws*) "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ let word = /[a-z][a-z0-9._-]*/i -let eol = Util.doseol -let empty = Util.empty_dos +(* A complete line that is either just whitespace or a comment that only + * contains whitespace *) +let empty = [ del (ws* . /#?/ . ws* . nl) "\n" ] + let indent = Util.indent -let comment_val_re = /([^ \t\r\n](.|\\\\\r?\n)*[^ \\\t\r\n]|[^ \t\r\n])/ -let comment = [ label "#comment" . del /[ \t]*#[ \t]*/ "# " - . store comment_val_re . eol ] +(* A comment that is not just whitespace. We define it in terms of the + * things that are not allowed as part of such a comment: + * 1) Starts with whitespace + * 2) Ends with whitespace, a backslash or \r + * 3) Unescaped newlines + *) +let comment = + let comment_start = del (ws* . "#" . ws* ) "# " in + let unesc_eol = /[^\]?/ . nl in + let w = /[^\t\n\r \\]/ in + let r = /[\r\\]/ in + let s = /[\t\r ]/ in + (* + * we'd like to write + * let b = /\\\\/ in + * let t = /[\t\n\r ]/ in + * let x = b . (t? . (s|w)* ) in + * but the definition of b depends on commit 244c0edd in 1.9.0 and + * would make the lens unusable with versions before 1.9.0. So we write + * x out which works in older versions, too + *) + let x = /\\\\[\t\n\r ]?[^\n\\]*/ in + let line = ((r . s* . w|w|r) . (s|w)* . x*|(r.s* )?).w.(s*.w)* in + [ label "#comment" . comment_start . store line . eol ] (* borrowed from shellvars.aug *) -let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ / let char_arg_sec = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ / let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ -let cdot = /\\\\./ -let cl = /\\\\\n/ let dquot = let no_dquot = /[^"\\\r\n]/ - in /"/ . (no_dquot|cdot|cl)* . /"/ + in /"/ . (no_dquot|esc_any)* . /"/ let dquot_msg = let no_dquot = /([^ \t"\\\r\n]|[^"\\\r\n]+[^ \t"\\\r\n])/ - in /"/ . (no_dquot|cdot|cl)* + in /"/ . (no_dquot|esc_any)* . no_dquot + let squot = let no_squot = /[^'\\\r\n]/ - in /'/ . (no_squot|cdot|cl)* . /'/ + in /'/ . (no_squot|esc_any)* . /'/ let comp = /[<>=]?=/ (****************************************************************** * Attributes *****************************************************************) -let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ] +(* The arguments for a directive come in two flavors: quoted with single or + * double quotes, or bare. Bare arguments may not start with a single or + * double quote; since we also treat "word lists" special, i.e. lists + * enclosed in curly braces, bare arguments may not start with those, + * either. + * + * Bare arguments may not contain unescaped spaces, but we allow escaping + * with '\\'. Quoted arguments can contain anything, though the quote must + * be escaped with '\\'. + *) +let bare = /([^{"' \t\n\r]|\\\\.)([^ \t\n\r]|\\\\.)*[^ \t\n\r\\]|[^{"' \t\n\r\\]/ + +let arg_quoted = [ label "arg" . store (dquot|squot) ] +let arg_bare = [ label "arg" . store bare ] + (* message argument starts with " but ends at EOL *) let arg_dir_msg = [ label "arg" . store dquot_msg ] -let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] let arg_wl = [ label "arg" . store (char_arg_wl+|dquot|squot) ] (* comma-separated wordlist as permitted in the SSLRequire directive *) let arg_wordlist = - let wl_start = Util.del_str "{" in - let wl_end = Util.del_str "}" in + let wl_start = dels "{" in + let wl_end = dels "}" in let wl_sep = del /[ \t]*,[ \t]*/ ", " in [ label "wordlist" . wl_start . arg_wl . (wl_sep . arg_wl)* . wl_end ] let argv (l:lens) = l . (sep_spc . l)* +(* the arguments of a directive. We use this once we have parsed the name + * of the directive, and the space right after it. When dir_args is used, + * we also know that we have at least one argument. We need to be careful + * with the spacing between arguments: quoted arguments and word lists do + * not need to have space between them, but bare arguments do. + * + * Apache apparently is also happy if the last argument starts with a double + * quote, but has no corresponding closing duoble quote, which is what + * arg_dir_msg handles + *) +let dir_args = + let arg_nospc = arg_quoted|arg_wordlist in + (arg_bare . sep_spc | arg_nospc . sep_osp)* . (arg_bare|arg_nospc|arg_dir_msg) + let directive = - (* arg_dir_msg may be the last or only argument *) - let dir_args = (argv (arg_dir|arg_wordlist) . (sep_spc . arg_dir_msg)?) | arg_dir_msg - in [ indent . label "directive" . store word . (sep_spc . dir_args)? . eol ] + [ indent . label "directive" . store word . (sep_spc . dir_args)? . eol ] + +let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] let section (body:lens) = (* opt_eol includes empty lines *) - let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in + let opt_eol = del /([ \t]*#?[ \t]*\r?\n)*/ "\n" in let inner = (sep_spc . argv arg_sec)? . sep_osp . dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . indent . dels " Date: Wed, 31 Oct 2018 17:20:31 -0700 Subject: [PATCH 380/639] getargspec is deprecated in python 3 --- certbot/tests/display/util_test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 5672a20bd..f7d1a8a37 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -16,6 +16,11 @@ CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] +if six.PY2: + getargspec = inspect.getargspec +else: + getargspec = inspect.getfullargspec + class InputWithTimeoutTest(unittest.TestCase): """Tests for certbot.display.util.input_with_timeout.""" @@ -314,7 +319,7 @@ class FileOutputDisplayTest(unittest.TestCase): # Every IDisplay method implemented by FileDisplay must take # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): # pylint: disable=no-member - arg_spec = inspect.getargspec(getattr(self.displayer, name)) + arg_spec = getargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -371,7 +376,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): for name in interfaces.IDisplay.names(): # pylint: disable=no-member method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments - self.assertFalse(inspect.getargspec(method).keywords is None) + self.assertFalse(getargspec(method).keywords is None) class SeparateListInputTest(unittest.TestCase): From 2ddf47e3cce6433d361717267b6c6ab48db20163 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 17:28:47 -0700 Subject: [PATCH 381/639] assertEquals --> assertEqual --- .../certbot_apache/tests/autohsts_test.py | 2 +- .../certbot_nginx/tests/parser_obj_test.py | 58 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 8a7c7ee76..fd9927b60 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -120,7 +120,7 @@ class AutoHSTSTest(util.ApacheTest): self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), cur_val) # Ensure that the value is raised to max - self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path), maxage.format(constants.AUTOHSTS_STEPS[-1])) # Make permanent self.config.deploy_autohsts(mock_lineage) diff --git a/certbot-nginx/certbot_nginx/tests/parser_obj_test.py b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py index c9c9dd440..2217be54f 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py @@ -103,37 +103,37 @@ class SentenceTest(unittest.TestCase): def test_parse_sentence_words_hides_spaces(self): og_sentence = ['\r\n', 'hello', ' ', ' ', '\t\n ', 'lol', ' ', 'spaces'] self.sentence.parse(og_sentence) - self.assertEquals(self.sentence.words, ['hello', 'lol', 'spaces']) - self.assertEquals(self.sentence.dump(), ['hello', 'lol', 'spaces']) - self.assertEquals(self.sentence.dump(True), og_sentence) + self.assertEqual(self.sentence.words, ['hello', 'lol', 'spaces']) + self.assertEqual(self.sentence.dump(), ['hello', 'lol', 'spaces']) + self.assertEqual(self.sentence.dump(True), og_sentence) def test_parse_sentence_with_add_spaces(self): self.sentence.parse(['hi', 'there'], add_spaces=True) - self.assertEquals(self.sentence.dump(True), ['hi', ' ', 'there']) + self.assertEqual(self.sentence.dump(True), ['hi', ' ', 'there']) self.sentence.parse(['one', ' ', 'space', 'none'], add_spaces=True) - self.assertEquals(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none']) + self.assertEqual(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none']) def test_iterate(self): expected = [['1', '2', '3']] self.sentence.parse(['1', ' ', '2', ' ', '3']) for i, sentence in enumerate(self.sentence.iterate()): - self.assertEquals(sentence.dump(), expected[i]) + self.assertEqual(sentence.dump(), expected[i]) def test_set_tabs(self): self.sentence.parse(['tabs', 'pls'], add_spaces=True) self.sentence.set_tabs() - self.assertEquals(self.sentence.dump(True)[0], '\n ') + self.assertEqual(self.sentence.dump(True)[0], '\n ') self.sentence.parse(['tabs', 'pls'], add_spaces=True) def test_get_tabs(self): self.sentence.parse(['no', 'tabs']) - self.assertEquals(self.sentence.get_tabs(), '') + self.assertEqual(self.sentence.get_tabs(), '') self.sentence.parse(['\n \n ', 'tabs']) - self.assertEquals(self.sentence.get_tabs(), ' ') + self.assertEqual(self.sentence.get_tabs(), ' ') self.sentence.parse(['\n\t ', 'tabs']) - self.assertEquals(self.sentence.get_tabs(), '\t ') + self.assertEqual(self.sentence.get_tabs(), '\t ') self.sentence.parse(['\n\t \n', 'tabs']) - self.assertEquals(self.sentence.get_tabs(), '') + self.assertEqual(self.sentence.get_tabs(), '') class BlockTest(unittest.TestCase): def setUp(self): @@ -145,11 +145,11 @@ class BlockTest(unittest.TestCase): def test_iterate(self): # Iterates itself normally - self.assertEquals(self.bloc, next(self.bloc.iterate())) + self.assertEqual(self.bloc, next(self.bloc.iterate())) # Iterates contents while expanded expected = [self.bloc.dump()] + self.contents for i, elem in enumerate(self.bloc.iterate(expanded=True)): - self.assertEquals(expected[i], elem.dump()) + self.assertEqual(expected[i], elem.dump()) def test_iterate_match(self): # can match on contents while expanded @@ -157,17 +157,17 @@ class BlockTest(unittest.TestCase): expected = [['thing', '1'], ['thing', '2']] for i, elem in enumerate(self.bloc.iterate(expanded=True, match=lambda x: isinstance(x, Sentence) and 'thing' in x.words)): - self.assertEquals(expected[i], elem.dump()) + self.assertEqual(expected[i], elem.dump()) # can match on self - self.assertEquals(self.bloc, next(self.bloc.iterate( + self.assertEqual(self.bloc, next(self.bloc.iterate( expanded=True, match=lambda x: isinstance(x, Block) and 'server' in x.names))) def test_parse_with_added_spaces(self): import copy self.bloc.parse([copy.copy(self.name), self.contents], add_spaces=True) - self.assertEquals(self.bloc.dump(), [self.name, self.contents]) - self.assertEquals(self.bloc.dump(True), [ + self.assertEqual(self.bloc.dump(), [self.name, self.contents]) + self.assertEqual(self.bloc.dump(True), [ ['server', ' ', 'name', ' '], [['thing', ' ', '1'], ['thing', ' ', '2'], @@ -181,14 +181,14 @@ class BlockTest(unittest.TestCase): def test_set_tabs(self): self.bloc.set_tabs() - self.assertEquals(self.bloc.names.dump(True)[0], '\n ') + self.assertEqual(self.bloc.names.dump(True)[0], '\n ') for elem in self.bloc.contents.dump(True)[:-1]: - self.assertEquals(elem[0], '\n ') - self.assertEquals(self.bloc.contents.dump(True)[-1][0], '\n') + self.assertEqual(elem[0], '\n ') + self.assertEqual(self.bloc.contents.dump(True)[-1][0], '\n') def test_get_tabs(self): self.bloc.parse([[' \n \t', 'lol'], []]) - self.assertEquals(self.bloc.get_tabs(), ' \t') + self.assertEqual(self.bloc.get_tabs(), ' \t') class StatementsTest(unittest.TestCase): def setUp(self): @@ -210,7 +210,7 @@ class StatementsTest(unittest.TestCase): self.statements.parse(self.raw) self.statements.set_tabs() for statement in self.statements.iterate(): - self.assertEquals(statement.dump(True)[0], '\n ') + self.assertEqual(statement.dump(True)[0], '\n ') def test_set_tabs_with_parent(self): # Trailing whitespace should inherit from parent tabbing. @@ -219,19 +219,19 @@ class StatementsTest(unittest.TestCase): self.statements.parent.get_tabs.return_value = '\t\t' self.statements.set_tabs() for statement in self.statements.iterate(): - self.assertEquals(statement.dump(True)[0], '\n ') - self.assertEquals(self.statements.dump(True)[-1], '\n\t\t') + self.assertEqual(statement.dump(True)[0], '\n ') + self.assertEqual(self.statements.dump(True)[-1], '\n\t\t') def test_get_tabs(self): self.raw[0].insert(0, '\n \n \t') self.statements.parse(self.raw) - self.assertEquals(self.statements.get_tabs(), ' \t') + self.assertEqual(self.statements.get_tabs(), ' \t') self.statements.parse([]) - self.assertEquals(self.statements.get_tabs(), '') + self.assertEqual(self.statements.get_tabs(), '') def test_parse_with_added_spaces(self): self.statements.parse(self.raw, add_spaces=True) - self.assertEquals(self.statements.dump(True)[0], ['sentence', ' ', 'one']) + self.assertEqual(self.statements.dump(True)[0], ['sentence', ' ', 'one']) def test_parse_bad_list_raises_error(self): from certbot import errors @@ -241,13 +241,13 @@ class StatementsTest(unittest.TestCase): self.statements.parse(self.raw + ['\n\n ']) self.assertTrue(isinstance(self.statements.dump()[-1], list)) self.assertTrue(self.statements.dump(True)[-1].isspace()) - self.assertEquals(self.statements.dump(True)[-1], '\n\n ') + self.assertEqual(self.statements.dump(True)[-1], '\n\n ') def test_iterate(self): self.statements.parse(self.raw) expected = [['sentence', 'one'], ['sentence', 'two']] for i, elem in enumerate(self.statements.iterate(match=lambda x: 'sentence' in x)): - self.assertEquals(expected[i], elem.dump()) + self.assertEqual(expected[i], elem.dump()) if __name__ == "__main__": unittest.main() # pragma: no cover From 594fbe3ae8cb4a81032cb85884dabe36dcccffeb Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 17:33:39 -0700 Subject: [PATCH 382/639] disable pylint no-member for x-version --- certbot/tests/display/util_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index f7d1a8a37..36ea91027 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -17,9 +17,9 @@ TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] if six.PY2: - getargspec = inspect.getargspec + getargspec = inspect.getargspec # pylint:disable=no-member else: - getargspec = inspect.getfullargspec + getargspec = inspect.getfullargspec # pylint:disable=no-member class InputWithTimeoutTest(unittest.TestCase): From a5e0e801be7194db5894d0d3cc4313b85b7eb3aa Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 17:49:50 -0700 Subject: [PATCH 383/639] correctly use getfullargspec --- certbot/tests/display/util_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 36ea91027..eccffa47a 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -16,11 +16,6 @@ CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] -if six.PY2: - getargspec = inspect.getargspec # pylint:disable=no-member -else: - getargspec = inspect.getfullargspec # pylint:disable=no-member - class InputWithTimeoutTest(unittest.TestCase): """Tests for certbot.display.util.input_with_timeout.""" @@ -319,6 +314,10 @@ class FileOutputDisplayTest(unittest.TestCase): # Every IDisplay method implemented by FileDisplay must take # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): # pylint: disable=no-member + if six.PY2: + getargspec = inspect.getargspec + else: + getargspec = inspect.getfullargspec arg_spec = getargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -376,7 +375,10 @@ class NoninteractiveDisplayTest(unittest.TestCase): for name in interfaces.IDisplay.names(): # pylint: disable=no-member method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments - self.assertFalse(getargspec(method).keywords is None) + if six.PY2: + self.assertFalse(inspect.getargspec(method).keywords is None) + else: + self.assertFalse(inspect.getfullargspec(method).varkw is None) class SeparateListInputTest(unittest.TestCase): From b837f218c9e8755fbc5e4a6d47a68c5fe8f7f81c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 17:59:24 -0700 Subject: [PATCH 384/639] U mode on open --> newline=None --- certbot/crypto_util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 942f8502f..7b558b060 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -458,8 +458,12 @@ def sha256sum(filename): :rtype: str """ sha256 = hashlib.sha256() - with open(filename, 'rU') as file_d: - sha256.update(file_d.read().encode('UTF-8')) + if six.PY2: + with open(filename, 'rU') as file_d: + sha256.update(file_d.read().encode('UTF-8')) + else: + with open(filename, 'r', newline=None) as file_d: + sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() def cert_and_chain_from_fullchain(fullchain_pem): From 35510bfc6cafa5c5d02446bed8d783b2959b579b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 17:59:44 -0700 Subject: [PATCH 385/639] assertNotEquals --> assertNotEqual --- certbot-apache/certbot_apache/tests/autohsts_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index fd9927b60..bf92a13ff 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -112,7 +112,7 @@ class AutoHSTSTest(util.ApacheTest): for i in range(len(constants.AUTOHSTS_STEPS)-1): # Ensure that value is not made permanent prematurely self.config.deploy_autohsts(mock_lineage) - self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path), + self.assertNotEqual(self.get_autohsts_value(self.vh_truth[7].path), max_val) self.config.update_autohsts(mock.MagicMock()) # Value should match pre-permanent increment step From 1eabb4bae39b9567f987f6740cd554029c157492 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 31 Oct 2018 18:05:00 -0700 Subject: [PATCH 386/639] warn-->warning --- .../certbot_dns_cloudflare/dns_cloudflare.py | 2 +- .../certbot_dns_digitalocean/dns_digitalocean.py | 2 +- certbot-dns-google/certbot_dns_google/dns_google.py | 4 ++-- certbot-nginx/certbot_nginx/parser.py | 2 +- certbot/ocsp.py | 2 +- certbot/storage.py | 2 +- certbot/tests/ocsp_test.py | 6 +++--- certbot/tests/storage_test.py | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py index f1156642f..7604a8baf 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py @@ -122,7 +122,7 @@ class _CloudflareClient(object): self.cf.zones.dns_records.delete(zone_id, record_id) logger.debug('Successfully deleted TXT record.') except CloudFlare.exceptions.CloudFlareAPIError as e: - logger.warn('Encountered CloudFlareAPIError deleting TXT record: %s', e) + logger.warning('Encountered CloudFlareAPIError deleting TXT record: %s', e) else: logger.debug('TXT record not found; no cleanup needed.') else: diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py index 4bf279279..5a4f22327 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py @@ -134,7 +134,7 @@ class _DigitalOceanClient(object): logger.debug('Removing TXT record with id: %s', record.id) record.destroy() except digitalocean.Error as e: - logger.warn('Error deleting TXT record %s using the DigitalOcean API: %s', + logger.warning('Error deleting TXT record %s using the DigitalOcean API: %s', record.id, e) def _find_domain(self, domain_name): diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index c204cb0ca..0b84dddb0 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -179,7 +179,7 @@ class _GoogleClient(object): try: zone_id = self._find_managed_zone_id(domain) except errors.PluginError as e: - logger.warn('Error finding zone. Skipping cleanup.') + logger.warning('Error finding zone. Skipping cleanup.') return record_contents = self.get_existing_txt_rrset(zone_id, record_name) @@ -219,7 +219,7 @@ class _GoogleClient(object): request = changes.create(project=self.project_id, managedZone=zone_id, body=data) request.execute() except googleapiclient_errors.Error as e: - logger.warn('Encountered error deleting TXT record: %s', e) + logger.warning('Encountered error deleting TXT record: %s', e) def get_existing_txt_rrset(self, zone_id, record_name): """ diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index a5cf2892e..622eb8d55 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -415,7 +415,7 @@ def _parse_ssl_options(ssl_options): with open(ssl_options) as _file: return nginxparser.load(_file) except IOError: - logger.warn("Missing NGINX TLS options file: %s", ssl_options) + logger.warning("Missing NGINX TLS options file: %s", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] diff --git a/certbot/ocsp.py b/certbot/ocsp.py index d34110f88..049e14827 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -114,7 +114,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): logger.info("OCSP revocation warning: %s", warning) return True else: - logger.warn("Unable to properly parse OCSP output: %s\nstderr:%s", + logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", ocsp_output, ocsp_errors) return False diff --git a/certbot/storage.py b/certbot/storage.py index c16ea35b8..2aadaa67a 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -791,7 +791,7 @@ class RenewableCert(object): May need to recover from rare interrupted / crashed states.""" if self.has_pending_deployment(): - logger.warn("Found a new cert /archive/ that was not linked to in /live/; " + logger.warning("Found a new cert /archive/ that was not linked to in /live/; " "fixing...") self.update_all_links_to(self.latest_common_version()) return False diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 2d54274f0..55cd24adb 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -96,15 +96,15 @@ class OCSPTest(unittest.TestCase): self.assertEqual(ocsp._translate_ocsp_query(*openssl_happy), False) self.assertEqual(ocsp._translate_ocsp_query(*openssl_confused), False) self.assertEqual(mock_log.debug.call_count, 1) - self.assertEqual(mock_log.warn.call_count, 0) + self.assertEqual(mock_log.warning.call_count, 0) mock_log.debug.call_count = 0 self.assertEqual(ocsp._translate_ocsp_query(*openssl_unknown), False) self.assertEqual(mock_log.debug.call_count, 1) - self.assertEqual(mock_log.warn.call_count, 0) + self.assertEqual(mock_log.warning.call_count, 0) self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) self.assertEqual(mock_log.debug.call_count, 2) self.assertEqual(ocsp._translate_ocsp_query(*openssl_broken), False) - self.assertEqual(mock_log.warn.call_count, 1) + self.assertEqual(mock_log.warning.call_count, 1) mock_log.info.call_count = 0 self.assertEqual(ocsp._translate_ocsp_query(*openssl_revoked), True) self.assertEqual(mock_log.info.call_count, 0) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 03d595652..912cd7984 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -264,12 +264,12 @@ class RenewableCertTests(BaseRenewableCertTest): mock_has_pending.return_value = False self.assertEqual(self.test_rc.ensure_deployed(), True) self.assertEqual(mock_update.call_count, 0) - self.assertEqual(mock_logger.warn.call_count, 0) + self.assertEqual(mock_logger.warning.call_count, 0) mock_has_pending.return_value = True self.assertEqual(self.test_rc.ensure_deployed(), False) self.assertEqual(mock_update.call_count, 1) - self.assertEqual(mock_logger.warn.call_count, 1) + self.assertEqual(mock_logger.warning.call_count, 1) def test_update_link_to(self): From 6f662fa489dd48548cde5ec739bddf19720dd8d2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 16:39:01 -0700 Subject: [PATCH 387/639] ignore deprecation and resource warnings in certbot-dns-rfc2136, which are inherited from dnspython --- certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 89ce3d93e..9d3a0fbfd 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -7,6 +7,7 @@ import dns.flags import dns.rcode import dns.tsig import mock +import pytest from certbot import errors from certbot.plugins import dns_test_common @@ -70,6 +71,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic self.auth.perform([self.achall]) +@pytest.mark.filterwarnings("ignore::ResourceWarning", "ignore:decodestring:DeprecationWarning") class RFC2136ClientTest(unittest.TestCase): def setUp(self): From 49619fc0ab467f5ce7a6eed1404855e1201cd1ed Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 16:39:29 -0700 Subject: [PATCH 388/639] Error on warning, unless it's a ResourceWarning --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 4a1dcca16..7d0b19223 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,6 @@ [pytest] addopts = --numprocesses auto --pyargs +# ResourceWarnings are ignored as errors, since they're raised at close filterwarnings = error + always::ResourceWarning From 36e8d748a15044047054577f341f37bff621f63d Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 16:39:54 -0700 Subject: [PATCH 389/639] Clean up many warnings --- .../certbot_apache/tests/configurator_test.py | 6 ++-- certbot/cli.py | 4 ++- certbot/plugins/standalone_test.py | 2 ++ certbot/renewal.py | 4 ++- certbot/storage.py | 2 ++ certbot/tests/cert_manager_test.py | 5 ++-- certbot/tests/display/util_test.py | 1 + certbot/tests/log_test.py | 1 + certbot/tests/storage_test.py | 5 ++-- certbot/tests/util_test.py | 28 ++++++++++++++----- 10 files changed, 41 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index f18d3aac8..a77dbf637 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1651,7 +1651,8 @@ class MultiVhostsTest(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "RewriteEngine", "on", ssl_vhost.path, False)) - conf_text = open(ssl_vhost.filep).read() + with open(ssl_vhost.filep) as the_file: + conf_text = the_file.read() commented_rewrite_rule = ("# RewriteRule \"^/secrets/(.+)\" " "\"https://new.example.com/docs/$1\" [R,L]") uncommented_rewrite_rule = ("RewriteRule \"^/docs/(.+)\" " @@ -1667,7 +1668,8 @@ class MultiVhostsTest(util.ApacheTest): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3]) - conf_lines = open(ssl_vhost.filep).readlines() + with open(ssl_vhost.filep) as the_file: + conf_lines = the_file.readlines() conf_line_set = [l.strip() for l in conf_lines] not_commented_cond1 = ("RewriteCond " "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f") diff --git a/certbot/cli.py b/certbot/cli.py index 99bf33180..6502d2987 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -286,7 +286,9 @@ def read_file(filename, mode="rb"): """ try: filename = os.path.abspath(filename) - return filename, open(filename, mode).read() + with open(filename, mode) as the_file: + contents = the_file.read() + return filename, contents except IOError as exc: raise argparse.ArgumentTypeError(exc.strerror) diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 47f44ff77..9b741fc6f 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -72,6 +72,8 @@ class ServerManagerTest(unittest.TestCase): errors.StandaloneBindError, self.mgr.run, port, challenge_type=challenges.HTTP01) self.assertEqual(self.mgr.running(), {}) + some_server.close() + maybe_another_server.close() class SupportedChallengesActionTest(unittest.TestCase): diff --git a/certbot/renewal.py b/certbot/renewal.py index a1508fa60..5b2e00740 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -276,8 +276,10 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" # Some lineages may have begun with --staging, but then had production certs # added to them + with open(lineage.cert) as the_file: + contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, open(lineage.cert).read()) + OpenSSL.crypto.FILETYPE_PEM, contents) # all our test certs are from happy hacker fake CA, though maybe one day # we should test more methodically now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() diff --git a/certbot/storage.py b/certbot/storage.py index 2aadaa67a..2a7ccee02 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -1034,9 +1034,11 @@ class RenewableCert(object): archive = full_archive_path(None, cli_config, lineagename) live_dir = _full_live_path(cli_config, lineagename) if os.path.exists(archive): + config_file.close() raise errors.CertStorageError( "archive directory exists for " + lineagename) if os.path.exists(live_dir): + config_file.close() raise errors.CertStorageError( "live directory exists for " + lineagename) os.mkdir(archive) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 7c3f4c85a..84774ca77 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -39,9 +39,8 @@ class BaseCertManagerTest(test_util.ConfigTestCase): # We also create a file that isn't a renewal config in the same # location to test that logic that reads in all-and-only renewal # configs will ignore it and NOT attempt to parse it. - junk = open(os.path.join(self.config.renewal_configs_dir, "IGNORE.THIS"), "w") - junk.write("This file should be ignored!") - junk.close() + with open(os.path.join(self.config.renewal_configs_dir, "IGNORE.THIS"), "w") as junk: + junk.write("This file should be ignored!") def _set_up_config(self, domain, custom_archive): # TODO: maybe provide NamespaceConfig.make_dirs? diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index eccffa47a..60131d853 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -49,6 +49,7 @@ class InputWithTimeoutTest(unittest.TestCase): stdin.listen(1) with mock.patch("certbot.display.util.sys.stdin", stdin): self.assertRaises(errors.Error, self._call, timeout=0.001) + stdin.close() class FileOutputDisplayTest(unittest.TestCase): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 6588bf5ca..b82cc6ca1 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -86,6 +86,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.memory_handler.close() self.stream_handler.close() self.temp_handler.close() + self.devnull.close() super(PostArgParseSetupTest, self).tearDown() def test_common(self): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 912cd7984..484ad7e16 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -73,9 +73,8 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): # We also create a file that isn't a renewal config in the same # location to test that logic that reads in all-and-only renewal # configs will ignore it and NOT attempt to parse it. - junk = open(os.path.join(self.config.config_dir, "renewal", "IGNORE.THIS"), "w") - junk.write("This file should be ignored!") - junk.close() + with open(os.path.join(self.config.config_dir, "renewal", "IGNORE.THIS"), "w") as junk: + junk.write("This file should be ignored!") self.defaults = configobj.ConfigObj() diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 45cc55249..6685b88c6 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -210,16 +210,21 @@ class UniqueFileTest(test_util.TempDirTestCase): fd, name = self._call() fd.write("bar") fd.close() - self.assertEqual(open(name).read(), "bar") + with open(name) as f: + self.assertEqual(f.read(), "bar") def test_right_mode(self): - self.assertTrue(compat.compare_file_modes(0o700, os.stat(self._call(0o700)[1]).st_mode)) - self.assertTrue(compat.compare_file_modes(0o600, os.stat(self._call(0o600)[1]).st_mode)) + fd1, name1 = self._call(0o700) + fd2, name2 = self._call(0o600) + self.assertTrue(compat.compare_file_modes(0o700, os.stat(name1).st_mode)) + self.assertTrue(compat.compare_file_modes(0o600, os.stat(name2).st_mode)) + fd1.close() + fd2.close() def test_default_exists(self): - name1 = self._call()[1] # create 0000_foo.txt - name2 = self._call()[1] - name3 = self._call()[1] + fd1, name1 = self._call() # create 0000_foo.txt + fd2, name2 = self._call() + fd3, name3 = self._call() self.assertNotEqual(name1, name2) self.assertNotEqual(name1, name3) @@ -236,6 +241,10 @@ class UniqueFileTest(test_util.TempDirTestCase): basename3 = os.path.basename(name3) self.assertTrue(basename3.endswith("foo.txt")) + fd1.close() + fd2.close() + fd3.close() + try: file_type = file @@ -255,13 +264,18 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): f, path = self._call("wow") self.assertTrue(isinstance(f, file_type)) self.assertEqual(os.path.join(self.tempdir, "wow.conf"), path) + f.close() def test_multiple(self): + items = [] for _ in six.moves.range(10): - f, name = self._call("wow") + items.append(self._call("wow")) + f, name = items[-1] self.assertTrue(isinstance(f, file_type)) self.assertTrue(isinstance(name, six.string_types)) self.assertTrue("wow-0009.conf" in name) + for f, _ in items: + f.close() @mock.patch("certbot.util.os.fdopen") def test_failure(self, mock_fdopen): From 6e7d42f2980586028bec94373c2b4614027c4cc7 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 17:17:19 -0700 Subject: [PATCH 390/639] ignore ResourceWarnings in acme tests --- acme/acme/crypto_util_test.py | 4 ++++ acme/acme/messages_test.py | 2 ++ acme/acme/standalone_test.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 44b245bbe..ea39fd0fa 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -10,11 +10,15 @@ from six.moves import socketserver #type: ignore # pylint: disable=import-erro import josepy as jose import OpenSSL +import pytest from acme import errors from acme import test_util from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +# turns all warnings into errors for this module +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 876fbe825..4518da9b9 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -3,6 +3,7 @@ import unittest import josepy as jose import mock +import pytest from acme import challenges from acme import test_util @@ -166,6 +167,7 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_iter_meta(self): result = False for k in self.dir.meta: diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index ee527782a..fa0426563 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -12,6 +12,7 @@ from six.moves import socketserver # type: ignore # pylint: disable=import-err import josepy as jose import mock +import pytest import requests from acme import challenges @@ -19,6 +20,9 @@ from acme import crypto_util from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +# turns all warnings into errors for this module +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class TLSServerTest(unittest.TestCase): """Tests for acme.standalone.TLSServer.""" From de1554f78517589357de35ae936ac2b24e11591f Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 17:28:40 -0700 Subject: [PATCH 391/639] Ignore more warnings. These are the ones I'm less sure about --- certbot/tests/log_test.py | 3 +++ certbot/tests/main_test.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index b82cc6ca1..db453f88f 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -7,6 +7,7 @@ import time import unittest import mock +import pytest import six from acme import messages @@ -89,6 +90,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.devnull.close() super(PostArgParseSetupTest, self).tearDown() + @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_common(self): with mock.patch('certbot.log.logging.getLogger') as mock_get_logger: mock_get_logger.return_value = self.root_logger @@ -267,6 +269,7 @@ class TempHandlerTest(unittest.TestCase): self.handler.close() self.assertFalse(os.path.exists(self.handler.path)) + @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_no_delete(self): self.handler.emit(mock.MagicMock()) self.handler.close() diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index f42319c4f..25613b3b9 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -16,6 +16,7 @@ import tempfile import sys import josepy as jose +import pytest import six from six.moves import reload_module # pylint: disable=import-error @@ -1359,6 +1360,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(mocked_run.called) @test_util.broken_on_windows + @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_register(self): with mock.patch('certbot.main.client') as mocked_client: acc = mock.MagicMock() From a5e26ca89008afebdc255873b81212c38f819499 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 17:32:59 -0700 Subject: [PATCH 392/639] Ignore all ResourceWarnings in main_test.py, since they're flaky --- certbot/tests/main_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 25613b3b9..3aabb8d32 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -48,6 +48,9 @@ JWK = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) RSA2048_KEY_PATH = test_util.vector_path('rsa2048_key.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') +# turns all warnings into errors for this module +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class TestHandleIdenticalCerts(unittest.TestCase): """Test for certbot.main._handle_identical_cert_request""" @@ -1360,7 +1363,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(mocked_run.called) @test_util.broken_on_windows - @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_register(self): with mock.patch('certbot.main.client') as mocked_client: acc = mock.MagicMock() From c699a50983ef9422a4723709d9aa1b428e162656 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 17:35:12 -0700 Subject: [PATCH 393/639] Ignore all ResourceWarnings in log_test.py --- certbot/tests/log_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index db453f88f..87b12246e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -19,6 +19,9 @@ from certbot import errors from certbot import util from certbot.tests import util as test_util +# turns all warnings into errors for this module +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot.log.pre_arg_parse_setup.""" @@ -90,7 +93,6 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.devnull.close() super(PostArgParseSetupTest, self).tearDown() - @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_common(self): with mock.patch('certbot.log.logging.getLogger') as mock_get_logger: mock_get_logger.return_value = self.root_logger @@ -269,7 +271,6 @@ class TempHandlerTest(unittest.TestCase): self.handler.close() self.assertFalse(os.path.exists(self.handler.path)) - @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_no_delete(self): self.handler.emit(mock.MagicMock()) self.handler.close() From 36b6328acded5b1e5340524f652d7c8c90428024 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 1 Nov 2018 17:56:01 -0700 Subject: [PATCH 394/639] Ignore ResourceWarnings in various modules in a 2-compatible way. Sometimes tests are flaky about giving this warning, particularly in ACME. I recommend just ignoring them. --- acme/acme/crypto_util_test.py | 5 +++-- acme/acme/messages_test.py | 6 +++++- acme/acme/standalone_test.py | 6 ++++-- acme/acme/util_test.py | 7 +++++++ .../certbot_dns_rfc2136/dns_rfc2136_test.py | 7 ++++++- certbot/tests/log_test.py | 5 +++-- certbot/tests/main_test.py | 5 +++-- 7 files changed, 31 insertions(+), 10 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index ea39fd0fa..58cb1cd69 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -16,8 +16,9 @@ from acme import errors from acme import test_util from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -# turns all warnings into errors for this module -pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") class SSLSocketAndProbeSNITest(unittest.TestCase): diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 4518da9b9..7124f8bd1 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -4,6 +4,7 @@ import unittest import josepy as jose import mock import pytest +import six from acme import challenges from acme import test_util @@ -14,6 +15,10 @@ CERT = test_util.load_comparable_cert('cert.der') CSR = test_util.load_comparable_csr('csr.der') KEY = test_util.load_rsa_private_key('rsa512_key.pem') +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class ErrorTest(unittest.TestCase): """Tests for acme.messages.Error.""" @@ -167,7 +172,6 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) - @pytest.mark.filterwarnings("ignore::ResourceWarning") def test_iter_meta(self): result = False for k in self.dir.meta: diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index fa0426563..4e93157d0 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -6,6 +6,7 @@ import threading import tempfile import unittest +import six from six.moves import http_client # pylint: disable=import-error from six.moves import queue # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error @@ -20,8 +21,9 @@ from acme import crypto_util from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -# turns all warnings into errors for this module -pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") class TLSServerTest(unittest.TestCase): diff --git a/acme/acme/util_test.py b/acme/acme/util_test.py index 00aa8b02d..7bb308635 100644 --- a/acme/acme/util_test.py +++ b/acme/acme/util_test.py @@ -1,6 +1,13 @@ """Tests for acme.util.""" import unittest +import pytest +import six + +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class MapKeysTest(unittest.TestCase): """Tests for acme.util.map_keys.""" diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 9d3a0fbfd..4af8ab9ef 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -8,6 +8,7 @@ import dns.rcode import dns.tsig import mock import pytest +import six from certbot import errors from certbot.plugins import dns_test_common @@ -20,6 +21,10 @@ NAME = 'a-tsig-key.' SECRET = 'SSB3b25kZXIgd2hvIHdpbGwgYm90aGVyIHRvIGRlY29kZSB0aGlzIHRleHQK' VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret": SECRET} +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): @@ -71,7 +76,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic self.auth.perform([self.achall]) -@pytest.mark.filterwarnings("ignore::ResourceWarning", "ignore:decodestring:DeprecationWarning") +@pytest.mark.filterwarnings("ignore:decodestring:DeprecationWarning") class RFC2136ClientTest(unittest.TestCase): def setUp(self): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 87b12246e..f722f2c97 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -19,8 +19,9 @@ from certbot import errors from certbot import util from certbot.tests import util as test_util -# turns all warnings into errors for this module -pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") class PreArgParseSetupTest(unittest.TestCase): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 3aabb8d32..bf0e6b1ca 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -48,8 +48,9 @@ JWK = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) RSA2048_KEY_PATH = test_util.vector_path('rsa2048_key.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') -# turns all warnings into errors for this module -pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") +# turns all ResourceWarnings into errors for this module +if six.PY3: + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") class TestHandleIdenticalCerts(unittest.TestCase): From a1af42bc5f79a5d1a3938bb9aedd6c3a69830c51 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 2 Nov 2018 18:59:27 +0200 Subject: [PATCH 395/639] Dummy AWS credentials for Route53 tests to prevent outbound connections (#6456) Boto3 / botocore library has a feature that tries to fetch AWS credentials from IAM if a set of credentials isn't available otherwise. This happens when boto loops through different credential providers in order to find the keys. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=912103 This PR simply adds dummy environmental variables for the tests that will be picked up by the credential provider iterator in order to prevent making outbound connections. * Hardcode dummy AWS credentials to prevent boto3 making outgoing connections * Remove the dummy credentials when tearing down test case --- CHANGELOG.md | 1 + .../certbot_dns_route53/dns_route53_test.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0ba82ad..f033b3691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py +* Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. ## 0.27.1 - 2018-09-06 diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index 7534e132c..71326c2af 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -1,5 +1,6 @@ """Tests for certbot_dns_route53.dns_route53.Authenticator""" +import os import unittest import mock @@ -20,8 +21,18 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest self.config = mock.MagicMock() + # Set up dummy credentials for testing + os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key" + self.auth = Authenticator(self.config, "route53") + def tearDown(self): + # Remove the dummy credentials from env vars + del os.environ["AWS_ACCESS_KEY_ID"] + del os.environ["AWS_SECRET_ACCESS_KEY"] + super(AuthenticatorTest, self).tearDown() + def test_perform(self): self.auth._change_txt_record = mock.MagicMock() self.auth._wait_for_change = mock.MagicMock() @@ -117,8 +128,18 @@ class ClientTest(unittest.TestCase): self.config = mock.MagicMock() + # Set up dummy credentials for testing + os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key" + self.client = Authenticator(self.config, "route53") + def tearDown(self): + # Remove the dummy credentials from env vars + del os.environ["AWS_ACCESS_KEY_ID"] + del os.environ["AWS_SECRET_ACCESS_KEY"] + super(ClientTest, self).tearDown() + def test_find_zone_id_for_domain(self): self.client.r53.get_paginator = mock.MagicMock() self.client.r53.get_paginator().paginate.return_value = [ From 28c117abe0b332cec0eb4d67c14a0091745e9d1f Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 12:31:55 -0700 Subject: [PATCH 396/639] If ResourceWarning is specified in pytest.ini, tests fail on Python3. We should be catching all of them, and they usually fail to successfully error anyway, so this just means we get sligthly worse error messages when they do occur. --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 7d0b19223..c35e0de7c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,3 @@ addopts = --numprocesses auto --pyargs # ResourceWarnings are ignored as errors, since they're raised at close filterwarnings = error - always::ResourceWarning From d02ea812a596cb96c7c44b42102f32c5d9beb7b1 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 12:36:42 -0700 Subject: [PATCH 397/639] Cover is run on 2.7, so mark 3-only lines as no cover --- acme/acme/crypto_util_test.py | 2 +- acme/acme/messages_test.py | 2 +- acme/acme/standalone_test.py | 2 +- acme/acme/util_test.py | 2 +- certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py | 2 +- certbot/tests/log_test.py | 2 +- certbot/tests/main_test.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 58cb1cd69..05d9f419e 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -18,7 +18,7 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class SSLSocketAndProbeSNITest(unittest.TestCase): diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 7124f8bd1..5b257836a 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -17,7 +17,7 @@ KEY = test_util.load_rsa_private_key('rsa512_key.pem') # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class ErrorTest(unittest.TestCase): diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 4e93157d0..7224515d4 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -23,7 +23,7 @@ from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-mo # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class TLSServerTest(unittest.TestCase): diff --git a/acme/acme/util_test.py b/acme/acme/util_test.py index 7bb308635..58e8aeb83 100644 --- a/acme/acme/util_test.py +++ b/acme/acme/util_test.py @@ -6,7 +6,7 @@ import six # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class MapKeysTest(unittest.TestCase): diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 4af8ab9ef..3e8c1e6ee 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -23,7 +23,7 @@ VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret" # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index f722f2c97..35cd6b388 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -21,7 +21,7 @@ from certbot.tests import util as test_util # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class PreArgParseSetupTest(unittest.TestCase): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index bf0e6b1ca..90de986bb 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -50,7 +50,7 @@ SS_CERT_PATH = test_util.vector_path('cert_2048.pem') # turns all ResourceWarnings into errors for this module if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover class TestHandleIdenticalCerts(unittest.TestCase): From e37a8fbded9e21512dd8fda4200db4faee50f1b5 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 13:01:14 -0700 Subject: [PATCH 398/639] Use a newer version of requests because of the upcoming Callable import Deprecation in Python 3.8 that warns in Python 3.7 --- acme/setup.py | 2 +- letsencrypt-auto-source/pieces/dependency-requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 85492d9a3..75f889648 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -17,7 +17,7 @@ install_requires = [ 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests[security]>=2.20.0', # DeprecationWarning on Callable in Python3.7 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index ae6079d96..f4a6f30d1 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -150,9 +150,9 @@ pytz==2015.7 \ --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.12.1 \ - --hash=sha256:3f3f27a9d0f9092935efc78054ef324eb9f8166718270aefe036dfa1e4f68e1e \ - --hash=sha256:2109ecea94df90980be040490ff1d879971b024861539abb00054062388b612e +requests==2.20.0 \ + --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ + --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 six==1.10.0 \ --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a From a83afb61350fd5109e4b501bdde5ea71ec29b084 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 13:31:04 -0700 Subject: [PATCH 399/639] properly disable no-member --- certbot/tests/display/util_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 60131d853..726eb0b0f 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -316,9 +316,9 @@ class FileOutputDisplayTest(unittest.TestCase): # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): # pylint: disable=no-member if six.PY2: - getargspec = inspect.getargspec + getargspec = inspect.getargspec # pylint: disable=no-member else: - getargspec = inspect.getfullargspec + getargspec = inspect.getfullargspec # pylint: disable=no-member arg_spec = getargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -377,9 +377,11 @@ class NoninteractiveDisplayTest(unittest.TestCase): method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments if six.PY2: - self.assertFalse(inspect.getargspec(method).keywords is None) + result = inspect.getargspec(method).keywords # pylint: disable=no-member + self.assertFalse(result is None) else: - self.assertFalse(inspect.getfullargspec(method).varkw is None) + result = inspect.getfullargspec(method).varkw # pylint: disable=no-member + self.assertFalse(result is None) class SeparateListInputTest(unittest.TestCase): From 1b595f26d867e6ee292f319ff93a2d8bf8ce3f0f Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 13:31:48 -0700 Subject: [PATCH 400/639] Requests no longer vendorizes urllib3 --- acme/acme/client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index bd86657b9..7a8a15219 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -32,11 +32,8 @@ logger = logging.getLogger(__name__) # for SSL, which does allow these options to be configured. # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning if sys.version_info < (2, 7, 9): # pragma: no cover - try: - requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore - except AttributeError: - import urllib3.contrib.pyopenssl # pylint: disable=import-error - urllib3.contrib.pyopenssl.inject_into_urllib3() + import urllib3.contrib.pyopenssl # pylint: disable=import-error + urllib3.contrib.pyopenssl.inject_into_urllib3() DEFAULT_NETWORK_TIMEOUT = 45 From 64a61fa6d4b7ea0bb5bc7b6e59fb02d748a2614a Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 14:53:55 -0700 Subject: [PATCH 401/639] Use a newer version of botocore that doesn't vendor requests, which we need because newer versions of requests don't have the DeprecationWarning --- tools/dev_constraints.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 00ecee03e..6fb8ed0d9 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -7,8 +7,8 @@ astroid==1.3.5 attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 -boto3==1.4.7 -botocore==1.7.41 +boto3==1.9.36 +botocore==1.12.36 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 From 1228f5da9995ef47f58ddd1c4d09f5dfad4a411e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 14:59:29 -0700 Subject: [PATCH 402/639] update letsencrypt-auto --- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 740cb9c73..2e45a32f7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1146,9 +1146,9 @@ pytz==2015.7 \ --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.12.1 \ - --hash=sha256:3f3f27a9d0f9092935efc78054ef324eb9f8166718270aefe036dfa1e4f68e1e \ - --hash=sha256:2109ecea94df90980be040490ff1d879971b024861539abb00054062388b612e +requests==2.20.0 \ + --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ + --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 six==1.10.0 \ --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a From ac8d6f58bb655d78fa819be98c2c3d3dea06459b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 15:30:42 -0700 Subject: [PATCH 403/639] Update requests version in oldest-constraints.txt --- tools/oldest_constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index de2b83ad8..6722d7f15 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -43,7 +43,7 @@ oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.4.1 +requests[security]==2.20.0 # Ubuntu Xenial constraints ConfigArgParse==0.10.0 From 0caaf872fbc72f099e254f0da2cace129b7288af Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 15:53:44 -0700 Subject: [PATCH 404/639] bring requests back down to 2.4.1 in setup and oldest constraints --- acme/setup.py | 2 +- tools/oldest_constraints.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 75f889648..85492d9a3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -17,7 +17,7 @@ install_requires = [ 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', - 'requests[security]>=2.20.0', # DeprecationWarning on Callable in Python3.7 + 'requests[security]>=2.4.1', # security extras added in 2.4.1 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 6722d7f15..de2b83ad8 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -43,7 +43,7 @@ oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.20.0 +requests[security]==2.4.1 # Ubuntu Xenial constraints ConfigArgParse==0.10.0 From c98bf18a770c12e7f393847425eaad4f2158bdb0 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 16:00:32 -0700 Subject: [PATCH 405/639] Use older version of boto in oldest tests, because new versions can't handle old versions of requests --- tools/oldest_constraints.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index de2b83ad8..8dd96dfea 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -49,3 +49,7 @@ requests[security]==2.4.1 ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 + +# Plugin constraints +boto3==1.4.7 +botocore==1.7.41 From eab7aa7bf158db1d33029b90176141add5f9df75 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 2 Nov 2018 16:14:10 -0700 Subject: [PATCH 406/639] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f033b3691..fa0fcd334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Stop caching the results of ipv6_info in http01.py * Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. +* Update code and dependencies to clean up Resource and Deprecation Warnings. ## 0.27.1 - 2018-09-06 From 2c1964c639916bcedc4d10d073cbcca212e7b8f1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 2 Nov 2018 17:32:33 -0700 Subject: [PATCH 407/639] Use the ACMEv2 newNonce endpoint when a new nonce is needed (#6442) Also, add checking to the newNonce HEAD request, and check responses in general before attempting to save a nonce, for a better error message. * check response before adding nonce to the pool * fix tests so that they test what they're supposed to test, and also allow the order of _add_nonce and _check_response to be switched * make _get_nonce take acme_version * Send HEAD to newNonce endpoint when using ACMEv2 * check the HEAD newNonce response * remove unnecessary try; get returns None if the item doesn't exist * instead of setting new_nonce_url on ClientNetwork, use the saved directory in ClientBase and pass that into ClientNetwork.post * no need to test acme_version in _get_nonce * pop new_nonce_url out of kwargs before passing to _send_request --- CHANGELOG.md | 1 + acme/acme/client.py | 20 ++++++++--- acme/acme/client_test.py | 74 +++++++++++++++++++++++++++++++--------- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f033b3691..b33c4ad97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added * `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`. +* Use the ACMEv2 newNonce endpoint when a new nonce is needed, and newNonce is available in the directory. ### Changed diff --git a/acme/acme/client.py b/acme/acme/client.py index bd86657b9..adc8ad9e3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -89,6 +89,8 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes """ kwargs.setdefault('acme_version', self.acme_version) + if hasattr(self.directory, 'newNonce'): + kwargs.setdefault('new_nonce_url', getattr(self.directory, 'newNonce')) return self.net.post(*args, **kwargs) def update_registration(self, regr, update=None): @@ -1106,10 +1108,15 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise errors.MissingNonce(response) - def _get_nonce(self, url): + def _get_nonce(self, url, new_nonce_url): if not self._nonces: logger.debug('Requesting fresh nonce') - self._add_nonce(self.head(url)) + if new_nonce_url is None: + response = self.head(url) + else: + # request a new nonce from the acme newNonce endpoint + response = self._check_response(self.head(new_nonce_url), content_type=None) + self._add_nonce(response) return self._nonces.pop() def post(self, *args, **kwargs): @@ -1130,8 +1137,13 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, acme_version=1, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) + try: + new_nonce_url = kwargs.pop('new_nonce_url') + except KeyError: + new_nonce_url = None + data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) + response = self._check_response(response, content_type=content_type) self._add_nonce(response) - return self._check_response(response, content_type=content_type) + return response diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acfd7eaff..2046d2377 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -805,7 +805,8 @@ class ClientV2Test(ClientTestBase): def test_revoke(self): self.client.revoke(messages_test.CERT, self.rsn) self.net.post.assert_called_once_with( - self.directory["revokeCert"], mock.ANY, acme_version=2) + self.directory["revokeCert"], mock.ANY, acme_version=2, + new_nonce_url=DIRECTORY_V2['newNonce']) def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: @@ -1052,7 +1053,10 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.response = mock.MagicMock(ok=True, status_code=http_client.OK) self.response.headers = {} self.response.links = {} - self.checked_response = mock.MagicMock() + self.response.checked = False + self.acmev1_nonce_response = mock.MagicMock(ok=False, + status_code=http_client.METHOD_NOT_ALLOWED) + self.acmev1_nonce_response.headers = {} self.obj = mock.MagicMock() self.wrapped_obj = mock.MagicMock() self.content_type = mock.sentinel.content_type @@ -1064,13 +1068,21 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def send_request(*args, **kwargs): # pylint: disable=unused-argument,missing-docstring + self.assertFalse("new_nonce_url" in kwargs) + method = args[0] + uri = args[1] + if method == 'HEAD' and uri != "new_nonce_uri": + response = self.acmev1_nonce_response + else: + response = self.response + if self.available_nonces: - self.response.headers = { + response.headers = { self.net.REPLAY_NONCE_HEADER: self.available_nonces.pop().decode()} else: - self.response.headers = {} - return self.response + response.headers = {} + return response # pylint: disable=protected-access self.net._send_request = self.send_request = mock.MagicMock( @@ -1082,28 +1094,39 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): # pylint: disable=missing-docstring self.assertEqual(self.response, response) self.assertEqual(self.content_type, content_type) - return self.checked_response + self.assertTrue(self.response.ok) + self.response.checked = True + return self.response def test_head(self): - self.assertEqual(self.response, self.net.head( + self.assertEqual(self.acmev1_nonce_response, self.net.head( 'http://example.com/', 'foo', bar='baz')) self.send_request.assert_called_once_with( 'HEAD', 'http://example.com/', 'foo', bar='baz') + def test_head_v2(self): + self.assertEqual(self.response, self.net.head( + 'new_nonce_uri', 'foo', bar='baz')) + self.send_request.assert_called_once_with( + 'HEAD', 'new_nonce_uri', 'foo', bar='baz') + def test_get(self): - self.assertEqual(self.checked_response, self.net.get( + self.assertEqual(self.response, self.net.get( 'http://example.com/', content_type=self.content_type, bar='baz')) + self.assertTrue(self.response.checked) self.send_request.assert_called_once_with( 'GET', 'http://example.com/', bar='baz') def test_post_no_content_type(self): self.content_type = self.net.JOSE_CONTENT_TYPE - self.assertEqual(self.checked_response, self.net.post('uri', self.obj)) + self.assertEqual(self.response, self.net.post('uri', self.obj)) + self.assertTrue(self.response.checked) def test_post(self): # pylint: disable=protected-access - self.assertEqual(self.checked_response, self.net.post( + self.assertEqual(self.response, self.net.post( 'uri', self.obj, content_type=self.content_type)) + self.assertTrue(self.response.checked) self.net._wrap_in_jws.assert_called_once_with( self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) @@ -1135,7 +1158,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def test_post_not_retried(self): check_response = mock.MagicMock() check_response.side_effect = [messages.Error.with_code('malformed'), - self.checked_response] + self.response] # pylint: disable=protected-access self.net._check_response = check_response @@ -1143,13 +1166,12 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.obj, content_type=self.content_type) def test_post_successful_retry(self): - check_response = mock.MagicMock() - check_response.side_effect = [messages.Error.with_code('badNonce'), - self.checked_response] + post_once = mock.MagicMock() + post_once.side_effect = [messages.Error.with_code('badNonce'), + self.response] # pylint: disable=protected-access - self.net._check_response = check_response - self.assertEqual(self.checked_response, self.net.post( + self.assertEqual(self.response, self.net.post( 'uri', self.obj, content_type=self.content_type)) def test_head_get_post_error_passthrough(self): @@ -1160,6 +1182,26 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertRaises(requests.exceptions.RequestException, self.net.post, 'uri', obj=self.obj) + def test_post_bad_nonce_head(self): + # pylint: disable=protected-access + # regression test for https://github.com/certbot/certbot/issues/6092 + bad_response = mock.MagicMock(ok=False, status_code=http_client.SERVICE_UNAVAILABLE) + self.net._send_request = mock.MagicMock() + self.net._send_request.return_value = bad_response + self.content_type = None + check_response = mock.MagicMock() + self.net._check_response = check_response + self.assertRaises(errors.ClientError, self.net.post, 'uri', + self.obj, content_type=self.content_type, acme_version=2, + new_nonce_url='new_nonce_uri') + self.assertEqual(check_response.call_count, 1) + + def test_new_nonce_uri_removed(self): + self.content_type = None + self.net.post('uri', self.obj, content_type=None, + acme_version=2, new_nonce_url='new_nonce_uri') + + class ClientNetworkSourceAddressBindingTest(unittest.TestCase): """Tests that if ClientNetwork has a source IP set manually, the underlying library has used the provided source address.""" From 94f0a915c0f0034cfc93965ada76617ae183bd01 Mon Sep 17 00:00:00 2001 From: Georgio Nicolas Date: Sun, 4 Nov 2018 01:23:47 -0400 Subject: [PATCH 408/639] Remove mentioning of TLS-SNI-01 --- docs/challenges.rst | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/docs/challenges.rst b/docs/challenges.rst index 25d190147..034f79956 100644 --- a/docs/challenges.rst +++ b/docs/challenges.rst @@ -5,8 +5,7 @@ To receive a certificate from Let's Encrypt certificate authority (CA), you must prove you control each of the domain names that will be listed in the certificate. A challenge is one of three tasks that only someone who controls the domain should be able to accomplish: -* Posting a specified file in a specified location on a web site (the HTTP-01 challenge) -* Offering a specified temporary certificate on a web site (the TLS-SNI-01 challenge) +* Posting a specified file in a specified location on a web site (the HTTP-01 challenge) * Posting a specified DNS record in the domain name system (the DNS-01 challenge) It’s possible to complete each type of challenge *automatically* (Certbot directly makes the necessary @@ -16,18 +15,12 @@ design favors performing challenges automatically, and this is the normal case f Some plugins offer an *authenticator*, meaning that they can satisfy challenges: -* Apache plugin: (TLS-SNI-01) Tries to edit your Apache configuration files to temporarily serve - a Certbot-generated certificate for a specified name. Use the Apache plugin when you're running - Certbot on a web server with Apache listening on port 443. -* NGINX plugin: (TLS-SNI-01) Tries to edit your NGINX configuration files to temporarily serve a - Certbot-generated certificate for a specified name. Use the NGINX plugin when you're running - Certbot on a web server with NGINX listening on port 443. * Webroot plugin: (HTTP-01) Tries to place a file where it can be served over HTTP on port 80 by a web server running on your system. Use the Webroot plugin when you're running Certbot on a web server with any server application listening on port 80 serving files from a folder on disk in response. -* Standalone plugin: (TLS-SNI-01 or HTTP-01) Tries to run a temporary web server listening on either HTTP on - port 80 (for HTTP-01) or HTTPS on port 443 (for TLS-SNI-01). Use the Standalone plugin if no existing program - is listening to these ports. Choose TLS-SNI-01 or HTTP-01 using the `--preferred-challenges` option. +* Standalone plugin: (HTTP-01) Tries to run a temporary web server listening on HTTP on + port 80 (for HTTP-01). Use the Standalone plugin if no existing program + is listening to these ports. Choose HTTP-01 using the `--preferred-challenges` option. * Manual plugin: (DNS-01 or HTTP-01) Either tells you what changes to make to your configuration or updates your DNS records using an external script (for DNS-01) or your webroot (for HTTP-01). Use the Manual plugin if you have the technical knowledge to make configuration changes yourself when asked to do so. @@ -63,20 +56,6 @@ HTTP-01 Challenge * When using the Standalone plugin, make sure another program is not already listening to port 80 on the server. * When using the Webroot plugin, make sure there is a web server listening on port 80. -TLS-SNI-01 Challenge -~~~~~~~~~~~~~~~~~~~~ - -* The TLS-SNI-01 challenge doesn’t work with content delivery networks (CDNs) - like CloudFlare and Akamai because the domain name is pointed at the CDN, not directly at your server. -* Make sure port 443 is open, publicly reachable from the Internet, and not blocked by a router or firewall. -* When using the Apache plugin, make sure you are running Apache and no other web server on port 443. -* When using the NGINX plugin, make sure you are running NGINX and no other web server on port 443. -* With either the Apache or NGINX plugin, certbot modifies your web server configuration. If you get - an error after successfully completing the challenge, then you have received a certificate but the - plugin was unable to modify your web server configuration, meaning that you'll have to install the certificate manually. - In that case, please file a bug to help us improve certbot! -* When using the Standalone plugin, make sure another program is not already listening to port 443 on the server. - DNS-01 Challenge ~~~~~~~~~~~~~~~~ From 9fa0a58545caad4b23e77e3e969bf66b398a92cc Mon Sep 17 00:00:00 2001 From: Georgio Nicolas Date: Sun, 4 Nov 2018 01:29:38 -0400 Subject: [PATCH 409/639] Remove mention of TLS-SNI-01 --- docs/using.rst | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 1fa13e022..64def5c06 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -44,14 +44,10 @@ a combination_ of distinct authenticator and installer plugins. =========== ==== ==== =============================================================== ============================= Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= -apache_ Y Y | Automates obtaining and installing a certificate with Apache tls-sni-01_ (443) - | 2.4 on OSes with ``libaugeas0`` 1.0+. webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. -nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. tls-sni-01_ (443) - | Shipped with Certbot 0.9.0. -standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. http-01_ (80) or - | Requires port 80 or 443 to be available. This is useful on tls-sni-01_ (443) +standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. http-01_ (80) + | Requires port 80 to be available. This is useful on | systems with no webserver, or when direct integration with | the local webserver is not supported or not desired. |dns_plugs| Y N | This category of plugins automates obtaining a certificate by dns-01_ (53) @@ -59,9 +55,9 @@ standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. | domain. Doing domain validation in this way is | the only way to obtain wildcard certificates from Let's | Encrypt. -manual_ Y N | Helps you obtain a certificate by giving you instructions to http-01_ (80), - | perform domain validation yourself. Additionally allows you dns-01_ (53) or - | to specify scripts to automate the validation task in a tls-sni-01_ (443) +manual_ Y N | Helps you obtain a certificate by giving you instructions to http-01_ (80) or + | perform domain validation yourself. Additionally allows you dns-01_ (53) + | to specify scripts to automate the validation task in a | customized way. =========== ==== ==== =============================================================== ============================= @@ -69,7 +65,7 @@ manual_ Y N | Helps you obtain a certificate by giving you instruction Under the hood, plugins use one of several ACME protocol challenges_ to prove you control a domain. The options are http-01_ (which uses port 80), -tls-sni-01_ (port 443) and dns-01_ (requiring configuration of a DNS server on +and dns-01_ (requiring configuration of a DNS server on port 53, though that's often not the same machine as your webserver). A few plugins support more than one challenge type, in which case you can choose one with ``--preferred-challenges``. @@ -78,7 +74,6 @@ There are also many third-party-plugins_ available. Below we describe in more de the circumstances in which each plugin can be used, and how to use it. .. _challenges: https://tools.ietf.org/html/draft-ietf-acme-acme-03#section-7 -.. _tls-sni-01: https://tools.ietf.org/html/draft-ietf-acme-acme-03#section-7.3 .. _http-01: https://tools.ietf.org/html/draft-ietf-acme-acme-03#section-7.2 .. _dns-01: https://tools.ietf.org/html/draft-ietf-acme-acme-03#section-7.4 @@ -159,13 +154,12 @@ software running on the machine where you obtain the certificate. To obtain a certificate using a "standalone" webserver, you can use the standalone plugin by including ``certonly`` and ``--standalone`` -on the command line. This plugin needs to bind to port 80 or 443 in +on the command line. This plugin needs to bind to port 80 in order to perform domain validation, so you may need to stop your existing webserver. To control which port the plugin uses, include one of the options shown below on the command line. * ``--preferred-challenges http`` to use port 80 - * ``--preferred-challenges tls-sni`` to use port 443 It must still be possible for your machine to accept inbound connections from the Internet on the specified port using each requested domain name. @@ -222,8 +216,7 @@ the UI, you can use the plugin to obtain a certificate by specifying to copy and paste commands into another terminal session, which may be on a different computer. -The manual plugin can use either the ``http``, ``dns`` or the -``tls-sni`` challenge. You can use the ``--preferred-challenges`` option +The manual plugin can use either the ``http`` or the ``dns`` challenge. You can use the ``--preferred-challenges`` option to choose the challenge of your preference. The ``http`` challenge will ask you to place a file with a specific name and @@ -241,11 +234,6 @@ For example, for the domain ``example.com``, a zone file entry would look like: _acme-challenge.example.com. 300 IN TXT "gfj9Xq...Rg85nM" -When using the ``tls-sni`` challenge, ``certbot`` will prepare a self-signed -SSL certificate for you with the challenge validation appropriately -encoded into a subjectAlternatNames entry. You will need to configure -your SSL server to present this challenge SSL certificate to the ACME -server using SNI. Additionally you can specify scripts to prepare for validation and perform the authentication procedure and/or clean up after it by using @@ -265,8 +253,7 @@ installer plugins. To do so, specify the authenticator plugin with For instance, you may want to create a certificate using the webroot_ plugin for authentication and the apache_ plugin for installation, perhaps because you use a proxy or CDN for SSL and only want to secure the connection between them -and your origin server, which cannot use the tls-sni-01_ challenge due to the -intermediate proxy. +and your origin server. :: @@ -775,9 +762,6 @@ variables to these scripts: - ``CERTBOT_DOMAIN``: The domain being authenticated - ``CERTBOT_VALIDATION``: The validation string (HTTP-01 and DNS-01 only) - ``CERTBOT_TOKEN``: Resource name part of the HTTP-01 challenge (HTTP-01 only) -- ``CERTBOT_CERT_PATH``: The challenge SSL certificate (TLS-SNI-01 only) -- ``CERTBOT_KEY_PATH``: The private key associated with the aforementioned SSL certificate (TLS-SNI-01 only) -- ``CERTBOT_SNI_DOMAIN``: The SNI name for which the ACME server expects to be presented the self-signed certificate located at ``$CERTBOT_CERT_PATH`` (TLS-SNI-01 only) Additionally for cleanup: From 9403c1641dbe8a0755e0e18d890f50cbff99db37 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 13:58:56 -0800 Subject: [PATCH 410/639] Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins (#6461) * flip challenge preference in Nginx * Fix Nginx tests * Flip challenge preference in Apache * Flip challenge preference in standalone * update changelog * continue to run with tls-sni in integration tests for coverage --- CHANGELOG.md | 1 + certbot-apache/certbot_apache/configurator.py | 2 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot-nginx/certbot_nginx/tests/configurator_test.py | 2 +- certbot-nginx/tests/boulder-integration.sh | 2 ++ certbot/plugins/standalone.py | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b33c4ad97..53855cac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index da632dc81..f431b9dab 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -2253,7 +2253,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01, challenges.HTTP01] + return [challenges.HTTP01, challenges.TLSSNI01] def perform(self, achalls): """Perform the configuration related challenge. diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 0f9c43e2d..dd0bf9e8b 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -1039,7 +1039,7 @@ class NginxConfigurator(common.Installer): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01, challenges.HTTP01] + return [challenges.HTTP01, challenges.TLSSNI01] # Entry point in main.py for performing challenges def perform(self, achalls): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index b180bb930..2814cbb8c 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -103,7 +103,7 @@ class NginxConfiguratorTest(util.NginxTest): errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement') def test_get_chall_pref(self): - self.assertEqual([challenges.TLSSNI01, challenges.HTTP01], + self.assertEqual([challenges.HTTP01, challenges.TLSSNI01], self.config.get_chall_pref('myhost')) def test_save(self): diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 194413f1d..2a24e645f 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,6 +39,8 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf +certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni +test_deployment_and_rollback nginx.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index cb2e69511..16f872a3f 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -114,7 +114,7 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \ +SUPPORTED_CHALLENGES = [challenges.HTTP01, challenges.TLSSNI01] \ # type: List[Type[challenges.KeyAuthorizationChallenge]] From bc7763dd0fd86bfac0774a80f6357035200025b7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 5 Nov 2018 23:07:09 +0100 Subject: [PATCH 411/639] Lexicon v3 compatibility (#6474) * Propagate correctly domain to lexicon providers * Pass required parameter to ovh provider * Fix all other lexicon-based dns plugins --- certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py | 1 + certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py | 1 + .../certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py | 1 + certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py | 1 + certbot-dns-linode/certbot_dns_linode/dns_linode.py | 1 + certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py | 1 + certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py | 1 + certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py | 1 + .../certbot_dns_sakuracloud/dns_sakuracloud.py | 1 + certbot/plugins/dns_common_lexicon.py | 7 ++++++- 10 files changed, 15 insertions(+), 1 deletion(-) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py index 674194fee..658db6072 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -70,6 +70,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): super(_CloudXNSLexiconClient, self).__init__() self.provider = cloudxns.Provider({ + 'provider_name': 'cloudxns', 'auth_username': api_key, 'auth_token': secret_key, 'ttl': ttl, diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py index f3a98567e..3eb56e37c 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -66,6 +66,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): super(_DNSimpleLexiconClient, self).__init__() self.provider = dnsimple.Provider({ + 'provider_name': 'dnssimple', 'auth_token': token, 'ttl': ttl, }) diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py index 982edfdd3..4236ce37a 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py @@ -72,6 +72,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): super(_DNSMadeEasyLexiconClient, self).__init__() self.provider = dnsmadeeasy.Provider({ + 'provider_name': 'dnsmadeeasy', 'auth_username': api_key, 'auth_token': secret_key, 'ttl': ttl, diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py index 50bfce1ae..9c35e72ab 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -73,6 +73,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): super(_GehirnLexiconClient, self).__init__() self.provider = gehirn.Provider({ + 'provider_name': 'gehirn', 'auth_token': api_token, 'auth_secret': api_secret, 'ttl': ttl, diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index cc29ce842..01da2cf60 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -62,6 +62,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key): super(_LinodeLexiconClient, self).__init__() self.provider = linode.Provider({ + 'provider_name': 'linode', 'auth_token': api_key }) diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py index 00b62e6e1..bd6a16f69 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py @@ -69,6 +69,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): super(_LuaDNSLexiconClient, self).__init__() self.provider = luadns.Provider({ + 'provider_name': 'luadns', 'auth_username': email, 'auth_token': token, 'ttl': ttl, diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py index 28db126c1..5f33efbba 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py @@ -66,6 +66,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): super(_NS1LexiconClient, self).__init__() self.provider = nsone.Provider({ + 'provider_name': 'nsone', 'auth_token': api_key, 'ttl': ttl, }) diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py index c4ded7748..578ee8e89 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -78,6 +78,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): super(_OVHLexiconClient, self).__init__() self.provider = ovh.Provider({ + 'provider_name': 'ovh', 'auth_entrypoint': endpoint, 'auth_application_key': application_key, 'auth_application_secret': application_secret, diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py index 6f1c74b68..b892330f5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -76,6 +76,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): super(_SakuraCloudLexiconClient, self).__init__() self.provider = sakuracloud.Provider({ + 'provider_name': 'sakuracloud', 'auth_token': api_token, 'auth_secret': api_secret, 'ttl': ttl, diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py index 7a97fc950..f9610b816 100644 --- a/certbot/plugins/dns_common_lexicon.py +++ b/certbot/plugins/dns_common_lexicon.py @@ -68,7 +68,12 @@ class LexiconClient(object): for domain_name in domain_name_guesses: try: - self.provider.options['domain'] = domain_name + if hasattr(self.provider, 'options'): + # For Lexicon 2.x + self.provider.options['domain'] = domain_name + else: + # For Lexicon 3.x + self.provider.domain = domain_name self.provider.authenticate() From cb8dd8a428edb98c6d9750950e5572f5337de7d5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 14:50:20 -0800 Subject: [PATCH 412/639] Warn when using deprecated acme.challenges.TLSSNI01 (#6469) * Warn when using deprecated acme.challenges.TLSSNI01 * Update changelog * remove specific date from warning * add a raw assert for mypy optional type checking --- CHANGELOG.md | 1 + acme/acme/challenges.py | 6 ++++++ acme/acme/challenges_test.py | 22 ++++++++++++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53855cac0..eea0ebb23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Warn when using deprecated acme.challenges.TLSSNI01 * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 2f0bf004d..a65768228 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -4,6 +4,7 @@ import functools import hashlib import logging import socket +import warnings from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose @@ -493,6 +494,11 @@ class TLSSNI01(KeyAuthorizationChallenge): # boulder#962, ietf-wg-acme#22 #n = jose.Field("n", encoder=int, decoder=int) + def __init__(self, *args, **kwargs): + warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.", + DeprecationWarning, stacklevel=2) + super(TLSSNI01, self).__init__(*args, **kwargs) + def validation(self, account_key, **kwargs): """Generate validation. diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 661d25a35..9307eb95b 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,5 +1,6 @@ """Tests for acme.challenges.""" import unittest +import warnings import josepy as jose import mock @@ -360,20 +361,29 @@ class TLSSNI01ResponseTest(unittest.TestCase): class TLSSNI01Test(unittest.TestCase): def setUp(self): - from acme.challenges import TLSSNI01 - self.msg = TLSSNI01( - token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) self.jmsg = { 'type': 'tls-sni-01', 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', } + def _msg(self): + from acme.challenges import TLSSNI01 + with warnings.catch_warnings(record=True) as warn: + warnings.simplefilter("always") + msg = TLSSNI01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) + assert warn is not None # using a raw assert for mypy + self.assertTrue(len(warn) == 1) + self.assertTrue(issubclass(warn[-1].category, DeprecationWarning)) + self.assertTrue('deprecated' in str(warn[-1].message)) + return msg + def test_to_partial_json(self): - self.assertEqual(self.jmsg, self.msg.to_partial_json()) + self.assertEqual(self.jmsg, self._msg().to_partial_json()) def test_from_json(self): from acme.challenges import TLSSNI01 - self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg)) + self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSSNI01 @@ -388,7 +398,7 @@ class TLSSNI01Test(unittest.TestCase): @mock.patch('acme.challenges.TLSSNI01Response.gen_cert') def test_validation(self, mock_gen_cert): mock_gen_cert.return_value = ('cert', 'key') - self.assertEqual(('cert', 'key'), self.msg.validation( + self.assertEqual(('cert', 'key'), self._msg().validation( KEY, cert_key=mock.sentinel.cert_key)) mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) From cbdc2ee23b1020e7450c646688ca6b930af4c795 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 15:01:16 -0800 Subject: [PATCH 413/639] Log warning about TLS-SNI deprecation in Certbot (#6468) For #6319. * print warning in auth_handler * add test --- certbot/auth_handler.py | 6 ++++++ certbot/tests/auth_handler_test.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index e7d658b25..efee49143 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -113,6 +113,12 @@ class AuthHandler(object): aauthzr.authzr, path) aauthzr.achalls.extend(aauthzr_achalls) + for aauthzr in aauthzrs: + for achall in aauthzr.achalls: + if isinstance(achall.chall, challenges.TLSSNI01): + logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") + return + def _has_challenges(self, aauthzrs): """Do we have any challenges to perform?""" return any(aauthzr.achalls for aauthzr in aauthzrs) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 76d1df90f..e1319b614 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -327,6 +327,11 @@ class HandleAuthorizationsTest(unittest.TestCase): azr.body.combinations) aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) + @mock.patch("certbot.auth_handler.logger") + def test_tls_sni_logs(self, logger): + self._test_name1_tls_sni_01_1_common(combos=True) + self.assertTrue("deprecated" in logger.warning.call_args[0][0]) + class PollChallengesTest(unittest.TestCase): # pylint: disable=protected-access From 47062dbfbfa4aaedc5910461e3f1bf14686ce27b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 17:09:03 -0800 Subject: [PATCH 414/639] update changelog (#6476) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eea0ebb23..df95c2d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. * Warn when using deprecated acme.challenges.TLSSNI01 +* Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed From 79b2ea19fbd5b360eabf6abe56530d45e1f2bcd8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:19:57 -0800 Subject: [PATCH 415/639] no need for U flag because we won't support py2 on windows --- certbot/crypto_util.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 7b558b060..820bcd805 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -458,12 +458,8 @@ def sha256sum(filename): :rtype: str """ sha256 = hashlib.sha256() - if six.PY2: - with open(filename, 'rU') as file_d: - sha256.update(file_d.read().encode('UTF-8')) - else: - with open(filename, 'r', newline=None) as file_d: - sha256.update(file_d.read().encode('UTF-8')) + with open(filename, 'r', newline=None) as file_d: + sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() def cert_and_chain_from_fullchain(fullchain_pem): From b7f4b33ffbe8afb5084d95758b122861e3626d50 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:28:26 -0800 Subject: [PATCH 416/639] Remove module-level ignore::ResourceWarnings --- acme/acme/crypto_util_test.py | 5 ----- acme/acme/messages_test.py | 5 ----- acme/acme/standalone_test.py | 5 ----- acme/acme/util_test.py | 5 ----- certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py | 5 ----- certbot/tests/log_test.py | 5 ----- certbot/tests/main_test.py | 5 ----- 7 files changed, 35 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 05d9f419e..44b245bbe 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -10,16 +10,11 @@ from six.moves import socketserver #type: ignore # pylint: disable=import-erro import josepy as jose import OpenSSL -import pytest from acme import errors from acme import test_util from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 5b257836a..1daf0299b 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -3,7 +3,6 @@ import unittest import josepy as jose import mock -import pytest import six from acme import challenges @@ -15,10 +14,6 @@ CERT = test_util.load_comparable_cert('cert.der') CSR = test_util.load_comparable_csr('csr.der') KEY = test_util.load_rsa_private_key('rsa512_key.pem') -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class ErrorTest(unittest.TestCase): """Tests for acme.messages.Error.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 7224515d4..799376cb6 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -13,7 +13,6 @@ from six.moves import socketserver # type: ignore # pylint: disable=import-err import josepy as jose import mock -import pytest import requests from acme import challenges @@ -21,10 +20,6 @@ from acme import crypto_util from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class TLSServerTest(unittest.TestCase): """Tests for acme.standalone.TLSServer.""" diff --git a/acme/acme/util_test.py b/acme/acme/util_test.py index 58e8aeb83..101936091 100644 --- a/acme/acme/util_test.py +++ b/acme/acme/util_test.py @@ -1,13 +1,8 @@ """Tests for acme.util.""" import unittest -import pytest import six -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class MapKeysTest(unittest.TestCase): """Tests for acme.util.map_keys.""" diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 3e8c1e6ee..ce500c0cb 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -7,7 +7,6 @@ import dns.flags import dns.rcode import dns.tsig import mock -import pytest import six from certbot import errors @@ -21,10 +20,6 @@ NAME = 'a-tsig-key.' SECRET = 'SSB3b25kZXIgd2hvIHdpbGwgYm90aGVyIHRvIGRlY29kZSB0aGlzIHRleHQK' VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret": SECRET} -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 35cd6b388..b82cc6ca1 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -7,7 +7,6 @@ import time import unittest import mock -import pytest import six from acme import messages @@ -19,10 +18,6 @@ from certbot import errors from certbot import util from certbot.tests import util as test_util -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot.log.pre_arg_parse_setup.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 90de986bb..f42319c4f 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -16,7 +16,6 @@ import tempfile import sys import josepy as jose -import pytest import six from six.moves import reload_module # pylint: disable=import-error @@ -48,10 +47,6 @@ JWK = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) RSA2048_KEY_PATH = test_util.vector_path('rsa2048_key.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') -# turns all ResourceWarnings into errors for this module -if six.PY3: - pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") # pragma: no cover - class TestHandleIdenticalCerts(unittest.TestCase): """Test for certbot.main._handle_identical_cert_request""" From 6c8652a0a68df61a59cc7edf2f813c69ed68df29 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:30:26 -0800 Subject: [PATCH 417/639] add comment explaining about boto* in oldest_constraints.txt --- tools/oldest_constraints.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 8dd96dfea..80fce8b33 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -51,5 +51,7 @@ funcsigs==0.4 zope.hookable==4.0.4 # Plugin constraints +# These aren't necessarily the oldest versions we need to support +# Tracking at https://github.com/certbot/certbot/issues/6473 boto3==1.4.7 botocore==1.7.41 From 5dc9dd8deaa330c34a48805eacb14d654205c81f Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:34:33 -0800 Subject: [PATCH 418/639] Pin requests' dependencies in certbot-auto --- letsencrypt-auto-source/letsencrypt-auto | 9 +++++++++ .../pieces/dependency-requirements.txt | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 2e45a32f7..25c293898 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1185,6 +1185,15 @@ zope.interface==4.1.3 \ requests-toolbelt==0.8.0 \ --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 +chardet==3.0.2 \ + --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ + --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 +urllib3==1.21.1 \ + --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ + --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +certifi==2017.4.17 \ + --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ + --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index f4a6f30d1..78f430210 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -189,3 +189,12 @@ zope.interface==4.1.3 \ requests-toolbelt==0.8.0 \ --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 +chardet==3.0.2 \ + --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ + --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 +urllib3==1.21.1 \ + --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ + --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +certifi==2017.4.17 \ + --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ + --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a From c3fa05ba7416319fbc8c00b3bb8fb5d08c48322c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:38:29 -0800 Subject: [PATCH 419/639] remove unused six imports --- acme/acme/messages_test.py | 1 - acme/acme/standalone_test.py | 1 - acme/acme/util_test.py | 2 -- certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py | 1 - 4 files changed, 5 deletions(-) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 1daf0299b..876fbe825 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -3,7 +3,6 @@ import unittest import josepy as jose import mock -import six from acme import challenges from acme import test_util diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 799376cb6..ee527782a 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -6,7 +6,6 @@ import threading import tempfile import unittest -import six from six.moves import http_client # pylint: disable=import-error from six.moves import queue # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error diff --git a/acme/acme/util_test.py b/acme/acme/util_test.py index 101936091..00aa8b02d 100644 --- a/acme/acme/util_test.py +++ b/acme/acme/util_test.py @@ -1,8 +1,6 @@ """Tests for acme.util.""" import unittest -import six - class MapKeysTest(unittest.TestCase): """Tests for acme.util.map_keys.""" diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index ce500c0cb..7cb98b25e 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -7,7 +7,6 @@ import dns.flags import dns.rcode import dns.tsig import mock -import six from certbot import errors from certbot.plugins import dns_test_common From 91b3c5d61cbdfd7853bd14d509370e28f0f1ee15 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:41:26 -0800 Subject: [PATCH 420/639] remove pytest.mark, move to specific ignore in pytest.ini --- certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py | 1 - pytest.ini | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 7cb98b25e..89ce3d93e 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -70,7 +70,6 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic self.auth.perform([self.achall]) -@pytest.mark.filterwarnings("ignore:decodestring:DeprecationWarning") class RFC2136ClientTest(unittest.TestCase): def setUp(self): diff --git a/pytest.ini b/pytest.ini index c35e0de7c..a55d72301 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,7 @@ [pytest] addopts = --numprocesses auto --pyargs # ResourceWarnings are ignored as errors, since they're raised at close +# decodestring: https://github.com/rthalley/dnspython/issues/338 filterwarnings = error + ignore:decodestring:DeprecationWarning From 39a008eb834b6eecaf6314ab1a1e2c299f7c9c09 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:42:19 -0800 Subject: [PATCH 421/639] ignore our own TLS-SNI-01 warning --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index a55d72301..de1ecf267 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,8 @@ addopts = --numprocesses auto --pyargs # ResourceWarnings are ignored as errors, since they're raised at close # decodestring: https://github.com/rthalley/dnspython/issues/338 +# ignore our own TLS-SNI-01 warning filterwarnings = error ignore:decodestring:DeprecationWarning + ignore:TLS-SNI-01:DeprecationWarning \ No newline at end of file From 92989956f91878bc58e3bd5eec5a8496400135b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 5 Nov 2018 17:47:38 -0800 Subject: [PATCH 422/639] no newline in py27 --- certbot/crypto_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 820bcd805..c4a389cd5 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -458,7 +458,7 @@ def sha256sum(filename): :rtype: str """ sha256 = hashlib.sha256() - with open(filename, 'r', newline=None) as file_d: + with open(filename, 'r') as file_d: sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() From 4edfb3ef65ee2522f5b09c88793095ebee886bab Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 00:35:09 +0100 Subject: [PATCH 423/639] [Windows] Handle file renaming when the destination path already exists (#6415) On Linux, you can invoke os.rename(src, dst) even if dst already exists. In this case, destination file will be atomically replaced by the source file. On Windows, this will lead to an OSError because changes are not atomic. This cause certbot renew to fail in particular, because the old certificate configuration needs to be replace by the new when a certificate is effectively renewed. One could use the cross-platform function os.replace, but it is available only on Python >= 3.3. This PR add a function in compat to handle correctly this case on Windows, and delegating everything else to os.rename. * Cross platform compatible os.rename (we can use os.replace if its python 3) * Use os.replace instead of custom non-atomic code. * Avoid errors for lint and mypy. Add a test. --- certbot/compat.py | 24 ++++++++++++++++++++++++ certbot/reverter.py | 2 +- certbot/storage.py | 3 ++- certbot/tests/compat_test.py | 21 +++++++++++++++++++++ certbot/tests/reverter_test.py | 2 +- 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 certbot/tests/compat_test.py diff --git a/certbot/compat.py b/certbot/compat.py index e3e1bc4e1..d42febe81 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -65,6 +65,30 @@ def os_geteuid(): # Windows specific return 0 +def os_rename(src, dst): + """ + Rename a file to a destination path and handles situations where the destination exists. + + :param str src: The current file path. + :param str dst: The new file path. + """ + try: + os.rename(src, dst) + except OSError as err: + # Windows specific, renaming a file on an existing path is not possible. + # On Python 3, the best fallback with atomic capabilities we have is os.replace. + if err.errno != errno.EEXIST: + # Every other error is a legitimate exception. + raise + if not hasattr(os, 'replace'): # pragma: no cover + # We should never go on this line. Either we are on Linux and os.rename has succeeded, + # either we are on Windows, and only Python >= 3.4 is supported where os.replace is + # available. + raise RuntimeError('Error: tried to run os_rename on Python < 3.3. ' + 'Certbot supports only Python 3.4 >= on Windows.') + getattr(os, 'replace')(src, dst) + + def readline_with_timeout(timeout, prompt): """ Read user input to return the first line entered, or raise after specified timeout. diff --git a/certbot/reverter.py b/certbot/reverter.py index 5d56615fd..919037358 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -576,7 +576,7 @@ class Reverter(object): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: - os.rename(self.config.in_progress_dir, final_dir) + compat.os_rename(self.config.in_progress_dir, final_dir) return except OSError: logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp) diff --git a/certbot/storage.py b/certbot/storage.py index c16ea35b8..4b8110072 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -14,6 +14,7 @@ import six import certbot from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import errors @@ -188,7 +189,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): # Save only the config items that are relevant to renewal values = relevant_values(vars(cli_config.namespace)) write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - os.rename(temp_filename, config_filename) + compat.os_rename(temp_filename, config_filename) return configobj.ConfigObj(config_filename) diff --git a/certbot/tests/compat_test.py b/certbot/tests/compat_test.py new file mode 100644 index 000000000..552aa5645 --- /dev/null +++ b/certbot/tests/compat_test.py @@ -0,0 +1,21 @@ +"""Tests for certbot.compat.""" +import os + +from certbot import compat +import certbot.tests.util as test_util + +class OsReplaceTest(test_util.TempDirTestCase): + """Test to ensure consistent behavior of os_rename method""" + + def test_os_rename_to_existing_file(self): + """Ensure that os_rename will effectively rename src into dst for all platforms.""" + src = os.path.join(self.tempdir, 'src') + dst = os.path.join(self.tempdir, 'dst') + open(src, 'w').close() + open(dst, 'w').close() + + # On Windows, a direct call to os.rename will fail because dst already exists. + compat.os_rename(src, dst) + + self.assertFalse(os.path.exists(src)) + self.assertTrue(os.path.exists(dst)) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 999c6225c..d04e3c641 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -356,7 +356,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @mock.patch("certbot.reverter.os.rename") + @mock.patch("certbot.reverter.compat.os_rename") def test_finalize_checkpoint_no_rename_directory(self, mock_rename): self.reverter.add_to_checkpoint(self.sets[0], "perm save") From e6e323e3ffd8d74251fa9ad46a6eed1bcffbfe38 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 16:49:13 +0100 Subject: [PATCH 424/639] Update Lexicon to correct use of HTTP proxy on OVH provider (#6479) This PR update requirement of Lexicon to 2.7.14 on OVH plugin, to allow HTTP proxy to be used correctly when underlying OVH provider is invoked. * Update Lexicon to correct use of HTTP proxy on OVH provider * Update dev_constraints.txt * Update CHANGELOG.md --- CHANGELOG.md | 1 + certbot-dns-ovh/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df95c2d36..728c938fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Warn when using deprecated acme.challenges.TLSSNI01 * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins +* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies ### Fixed diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 1f3acbf62..89ff317d9 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -11,7 +11,7 @@ version = '0.28.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.7.3', # Correct OVH integration tests + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 00ecee03e..380d49cb3 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.3 +dns-lexicon==2.7.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 From f3ff548a413a699760ab2239b6e11d65e083d540 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:02:25 -0800 Subject: [PATCH 425/639] Update changelog for 0.28.0 release. --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728c938fb..2f0fd40dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.28.0 - master +## 0.28.0 - 2018-11-7 ### Added @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins * OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies +* Default time the Linode plugin waits for DNS changes to propogate is now 1200 seconds. ### Fixed @@ -27,6 +28,30 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Stop caching the results of ipv6_info in http01.py * Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. +* The CloudXNS, DNSimple, DNS Made Easy, Gehirn, Linode, LuaDNS, NS1, OVH, and + Sakura Cloud DNS plugins are now compatible with Lexicon 3.0+. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-route53 +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/59?closed=1 ## 0.27.1 - 2018-09-06 From c1300a8e1b5b3d7a485895f9a9abe46d582b2975 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:57 -0800 Subject: [PATCH 426/639] Release 0.28.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 28 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 22 +++++++------- letsencrypt-auto | 28 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++++-------- 26 files changed, 91 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 85492d9a3..89337ad71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aa5908f9e..805e1ff5e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 076c45e39..fe87317a7 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8dac3e047..bdda1bcde 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index b823cf98f..4fa1cd5ec 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3daf933cb..a1083f884 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index feb2a0d58..7d355f5e4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index b6efcf360..6f6e9181f 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c268eaa8f..1a099d201 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index fc147f85c..1cec98e24 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 86d36bcb3..696de488d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1d196403f..67592de76 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index a5c06d90e..38d2f144b 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 474093a5b..fa4bde85d 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 89ff317d9..ea2a9aa9f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c009ef032..b1c1cd0ec 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2bae0c3d0..c42f976ea 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 9f8bfbbdb..3f54bb96f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3c8a66ee5..d292fba31 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index ab23926c9..eecca88e0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0.dev0' +__version__ = '0.28.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4ed9f0731..d26da361b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -24,7 +24,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.1 + "". (default: CertbotACMEClient/0.28.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -261,7 +261,8 @@ manage: delete Clean up all files related to a certificate renew Renew all certificates (or one specified with --cert- name) - revoke Revoke a certificate specified with --cert-path + revoke Revoke a certificate specified with --cert-path or + --cert-name update_symlinks Recreate symlinks in your /etc/letsencrypt/live/ directory @@ -475,10 +476,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -492,16 +492,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 @@ -602,7 +602,7 @@ dns-linode: --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate before asking the ACME server to verify the DNS - record. (default: 960) + record. (default: 1200) --dns-linode-credentials DNS_LINODE_CREDENTIALS Linode credentials INI file. (default: None) diff --git a/letsencrypt-auto b/letsencrypt-auto index 076c45e39..fe87317a7 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 747d98e2d..57745758b 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X -dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ -nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn -ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa -I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk -AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm -8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== -=QRnC +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X +dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj +zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i +7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq +rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO +zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg +PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q== +=1hD2 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 740cb9c73..fe87317a7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0.dev0" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index b717e359b2a2c65f6cb588a007f5024572c1e4ff..33f5b2c00007bde13d9f58cf7e48ad17d6998502 100644 GIT binary patch literal 256 zcmV+b0ssC{t4ydxm0o9{N03G=j7lT5c_4fvp=Ak0G{=zFl<_N_YV;M+5gCs_o{c#_ zk}R?1daPA6KJFO+@o7i}i@E(nx&;8|8w zwwanT<%NEK7%&NBc6_9#QU}r> literal 256 zcmV+b0ssEcvxB0LY3lA!OPssV3H<~y?AaAqjuWWrmMIj>00(?0Ek&mFUj?a=WHkII z-RNTHeD@0(kwW#&HKEPltP&F}Il@J^#^yCaE=%~OTz&S0VZ;xAiFO*AjYgLt@?f^0 z5%fBEr@1)1d9JccQ6BE>@C=b&m*{iB+J6X1CVdnJgQ4FUmIMh?O(XLrLz zjb^9RY>s^Yo_H0;@vfC^I1)~)4x~+{{pTYk(%Y}AY2=R!zDwo_S`Vp=%04-BF^8$F zx6n%y(2yohnk6h!P4Oc-;Nb&}2~R5^lWYM-Sp*8Vav}^+f?3H>xwNIK(r-rHpKXnf GZUdUtC3&a- diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index b9cd42694..401a8e25c 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb From 22858c6025c2697ea7842f97a91e366cbac8293a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:59 -0800 Subject: [PATCH 427/639] Bump version to 0.29.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 89337ad71..ad70c2947 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 805e1ff5e..e6f6f1e23 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bdda1bcde..bfbbe0625 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4fa1cd5ec..d615fa999 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a1083f884..f9880270a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 7d355f5e4..a9d46d128 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 6f6e9181f..ac7bb1090 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 1a099d201..ab944d40d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 1cec98e24..94ca74761 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 696de488d..aa1afdc93 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 67592de76..4c2571f96 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 38d2f144b..d0735d1ec 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index fa4bde85d..aebff5304 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index ea2a9aa9f..68ede8006 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index b1c1cd0ec..914bfb6e6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index c42f976ea..6318cbb81 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 3f54bb96f..5af4c8a00 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index d292fba31..0908c4c52 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index eecca88e0..f6b7defbd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0' +__version__ = '0.29.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index fe87317a7..b1a05f6e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0" +LE_AUTO_VERSION="0.29.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 63e0f5678457b78cfee7fa7f8337a4b8f1809fd8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 15:56:29 -0800 Subject: [PATCH 428/639] update changelog for 0.29.0 --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0fd40dc..a25890929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.29.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/62?closed=1 + ## 0.28.0 - 2018-11-7 ### Added From 3d0e16ece3762d5bfe345fde0af286e26c54bacb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 02:16:16 +0100 Subject: [PATCH 429/639] [Windows|Unix] Rewrite bash scripts for tests into python (#6435) Certbot relies heavily on bash scripts to deploy a development environment and to execute tests. This is fine for Linux systems, including Travis, but problematic for Windows machines. This PR converts all theses scripts into Python, to make them platform independant. As a consequence, tox-win.ini is not needed anymore, and tox can be run indifferently on Windows or on Linux using a common tox.ini. AppVeyor is updated accordingly to execute tests for acme, certbot and all dns plugins. Other tests are not executed as they are for Docker, unsupported Apache/Nginx/Postfix plugins (for now) or not relevant for Windows (explicit Linux distribution tests or pylint). Another PR will be done on certbot website to update how a dev environment can be set up. * Replace several shell scripts by python equivalent. * Correction on tox coverage * Extend usage of new python scripts * Various corrections * Replace venv construction bash scripts by python equivalents * Update tox.ini * Unicode lines to compare files * Put modifications on letsencrypt-auto-source instead of generated scripts * Add executable permissions for Linux. * Merge tox win tests into main tox * Skip lock_test on Windows * Correct appveyor config * Update appveyor.yml * Explicit coverage py27 or py37 * Avoid to cover non supported certbot plugins on Windows * Update tox.ini * Remove specific warnings during CI * No cover on a debug code for tests only. * Update documentation and help script on venv/venv3.py * Customize help message for Windows * Quote correctly executable path with potential spaces in it. * Copy pipstrap from upstream --- .travis.yml | 4 +- Dockerfile-dev | 2 +- appveyor.yml | 25 +++- certbot-compatibility-test/Dockerfile | 5 +- certbot-postfix/README.rst | 2 +- certbot/tests/util.py | 13 +- docs/contributing.rst | 8 +- letsencrypt-auto-source/letsencrypt-auto | 11 +- .../pieces/bootstrappers/arch_common.sh | 2 +- letsencrypt-auto-source/pieces/pipstrap.py | 10 +- tests/letstest/scripts/test_apache2.sh | 2 +- tests/letstest/scripts/test_tox.sh | 2 +- tests/lock_test.py | 9 +- tests/modification-check.py | 124 ++++++++++++++++++ tests/modification-check.sh | 59 --------- tools/_venv_common.py | 67 ++++++++++ tools/_venv_common.sh | 26 ---- tools/install_and_test.py | 58 ++++++++ tools/install_and_test.sh | 29 ---- tools/merge_requirements.py | 16 +-- tools/pip_install.py | 90 +++++++++++++ tools/pip_install.sh | 44 ------- tools/pip_install_editable.py | 19 +++ tools/pip_install_editable.sh | 10 -- tools/readlink.py | 7 +- tools/venv.py | 58 ++++++++ tools/venv.sh | 34 ----- tools/venv3.py | 53 ++++++++ tools/venv3.sh | 33 ----- tox-win.ini | 13 -- tox.cover.py | 85 ++++++++++++ tox.cover.sh | 72 ---------- tox.ini | 28 ++-- 33 files changed, 643 insertions(+), 377 deletions(-) create mode 100755 tests/modification-check.py delete mode 100755 tests/modification-check.sh create mode 100755 tools/_venv_common.py delete mode 100755 tools/_venv_common.sh create mode 100755 tools/install_and_test.py delete mode 100755 tools/install_and_test.sh create mode 100755 tools/pip_install.py delete mode 100755 tools/pip_install.sh create mode 100755 tools/pip_install_editable.py delete mode 100755 tools/pip_install_editable.sh create mode 100755 tools/venv.py delete mode 100755 tools/venv.sh create mode 100755 tools/venv3.py delete mode 100755 tools/venv3.sh delete mode 100644 tox-win.ini create mode 100755 tox.cover.py delete mode 100755 tox.cover.sh diff --git a/.travis.yml b/.travis.yml index acdf365bd..2b8eafc13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=py27-cover FYI="py27 tests + code coverage" - sudo: required env: TOXENV=nginx_compat services: docker @@ -95,7 +95,7 @@ script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && codecov' +after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' notifications: email: false diff --git a/Dockerfile-dev b/Dockerfile-dev index 9e35ebec8..1ab56e081 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -16,6 +16,6 @@ RUN apt-get update && \ /tmp/* \ /var/tmp/* -RUN VENV_NAME="../venv" tools/venv.sh +RUN VENV_NAME="../venv" python tools/venv.py ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/appveyor.yml b/appveyor.yml index 796070081..725ecfbff 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,17 @@ -image: - # => Windows Server 2012 R2 - - Visual Studio 2015 - # => Windows Server 2016 - - Visual Studio 2017 +environment: + matrix: + - FYI: Python 3.4 on Windows Server 2012 R2 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - FYI: Python 3.4 on Windows Server 2016 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.5 on Windows Server 2016 + TOXENV: py35 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.7 on Windows Server 2016 + code coverage + TOXENV: py37-cover + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 branches: only: @@ -14,6 +23,7 @@ install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env + - "echo %APPVEYOR_BUILD_WORKER_IMAGE%" - "python --version" # Upgrade pip to avoid warnings - "python -m pip install --upgrade pip" @@ -23,7 +33,8 @@ install: build: off test_script: - - tox -c tox-win.ini -e py34,py35,py36,py37-cover + # Test env is set by TOXENV env variable + - tox on_success: - - codecov + - if exist .coverage codecov diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index 803b4a1b9..cbbdf7193 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... @@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/src/tools/pip_install_editable.sh \ +RUN /opt/certbot/venv/bin/python \ + /opt/certbot/src/tools/pip_install_editable.py \ /opt/certbot/src/acme \ /opt/certbot/src \ /opt/certbot/src/certbot-apache \ diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index e6367e365..1ae9cb980 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -7,7 +7,7 @@ feature requests for this plugin. To install this plugin, in the root of this repo, run:: - ./tools/venv.sh + python tools/venv.py source venv/bin/activate You can use this installer with any `authenticator plugin diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 822597dd4..8c5db2c2f 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -328,15 +328,16 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # Then we have various files which are not correctly closed at the time of tearDown. - # On Windows, it is visible for the same reasons as above. + # On Windows we have various files which are not correctly closed at the time of tearDown. # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. def onerror_handler(_, path, excinfo): """On error handler""" - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): diff --git a/docs/contributing.rst b/docs/contributing.rst index 58db251d4..ead4d7e2b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -38,13 +38,13 @@ Certbot. cd certbot ./certbot-auto --debug --os-packages-only - tools/venv.sh + python tools/venv.py -If you have Python3 available and want to use it, run the ``venv3.sh`` script. +If you have Python3 available and want to use it, run the ``venv3.py`` script. .. code-block:: shell - tools/venv3.sh + python tools/venv3.py .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -353,7 +353,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./tools/venv.sh``. + virtualenv. You can do this by running ``pip tools/venv.py``. (this is a **very important** step) 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors. 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b1a05f6e6..12be26e19 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -594,7 +594,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 @@ -1260,7 +1260,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1272,7 +1272,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1365,7 +1365,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1378,7 +1378,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1397,7 +1397,6 @@ def main(): if __name__ == '__main__': exit(main()) - UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 5759336c5..c55527590 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -8,7 +8,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index d55d5bceb..f21d36657 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -45,7 +45,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -150,7 +150,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +163,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +181,4 @@ def main(): if __name__ == '__main__': - exit(main()) + exit(main()) \ No newline at end of file diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 6b5d63c80..4036e6efa 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then exit 1 fi -tools/_venv_common.sh -e acme[dev] -e .[dev,docs] -e certbot-apache +python tools/_venv_common.py -e acme[dev] -e .[dev,docs] -e certbot-apache sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 84e4bcd22..bb9126673 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -14,5 +14,5 @@ VENV_BIN=${VENV_PATH}/bin "$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./tools/venv.sh +python tools/venv.py venv/bin/tox -e py27 diff --git a/tests/lock_test.py b/tests/lock_test.py index b01cc5d58..0266cf029 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,4 +1,6 @@ """Tests to ensure the lock order is preserved.""" +from __future__ import print_function + import atexit import functools import logging @@ -235,4 +237,9 @@ def log_output(level, out, err): if __name__ == "__main__": - main() + if os.name != 'nt': + main() + else: + print( + 'Warning: lock_test cannot be executed on Windows, ' + 'as it relies on a Nginx distribution for Linux.') diff --git a/tests/modification-check.py b/tests/modification-check.py new file mode 100755 index 000000000..e00994b04 --- /dev/null +++ b/tests/modification-check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile +import shutil +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +def find_repo_path(): + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +# We do not use filecmp.cmp to take advantage of universal newlines +# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. +# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. +# But it will work correctly on Linux for any version, because every file tested will be LF. +def compare_files(path_1, path_2): + l1 = l2 = True + with open(path_1, 'r') as f1, open(path_2, 'r') as f2: + line = 1 + while l1 and l2: + line += 1 + l1 = f1.readline() + l2 = f2.readline() + if l1 != l2: + print('---') + print(( + 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' + .format(os.path.basename(path_1), os.path.basename(path_2), line))) + print('(1): {0}'.format(repr(l1))) + print('(2): {0}'.format(repr(l2))) + print('---') + return False + + return True + +def validate_scripts_content(repo_path, temp_cwd): + errors = False + + if not compare_files( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'letsencrypt-auto')): + print('Root certbot-auto and letsencrypt-auto differ.') + errors = True + else: + shutil.copyfile( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')) + shutil.copy(os.path.normpath(os.path.join( + repo_path, + 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) + + # Compare file against current version in the target branch + branch = os.environ.get('TRAVIS_BRANCH', 'master') + url = ( + 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' + .format(branch)) + urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) + + if compare_files( + os.path.join(temp_cwd, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were unchanged') + else: + # Compare file against the latest released version + latest_version = subprocess.check_output( + [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) + subprocess.call( + [sys.executable, 'fetch.py', '--le-auto-script', + 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) + if compare_files( + os.path.join(temp_cwd, 'letsencrypt-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were updated to the latest version.') + else: + print('Root *-auto have unexpected changes.') + errors = True + + return errors + +def main(): + repo_path = find_repo_path() + temp_cwd = tempfile.mkdtemp() + errors = False + + try: + errors = validate_scripts_content(repo_path, temp_cwd) + + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'original-lea') + ) + subprocess.call([sys.executable, os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/build.py'))]) + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'build-lea') + ) + shutil.copyfile( + os.path.join(temp_cwd, 'original-lea'), + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) + ) + + if not compare_files( + os.path.join(temp_cwd, 'original-lea'), + os.path.join(temp_cwd, 'build-lea')): + print('Script letsencrypt-auto-source/letsencrypt-auto ' + 'doesn\'t match output of build.py.') + errors = True + else: + print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') + finally: + shutil.rmtree(temp_cwd) + + return errors + +if __name__ == '__main__': + if main(): + sys.exit(1) diff --git a/tests/modification-check.sh b/tests/modification-check.sh deleted file mode 100755 index 0145b0228..000000000 --- a/tests/modification-check.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -e - -temp_dir=`mktemp -d` -trap "rm -rf $temp_dir" EXIT - -# cd to repo root -cd $(dirname $(dirname $(readlink -f $0))) -FLAG=false - -if ! cmp -s certbot-auto letsencrypt-auto; then - echo "Root certbot-auto and letsencrypt-auto differ." - FLAG=true -else - cp certbot-auto "$temp_dir/local-auto" - cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" - cd $temp_dir - - # Compare file against current version in the target branch - BRANCH=${TRAVIS_BRANCH:-master} - URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" - curl -sS $URL > certbot-auto - if cmp -s certbot-auto local-auto; then - echo "Root *-auto were unchanged." - else - # Compare file against the latest released version - python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" - if cmp -s letsencrypt-auto local-auto; then - echo "Root *-auto were updated to the latest version." - else - echo "Root *-auto have unexpected changes." - FLAG=true - fi - fi - cd ~- -fi - -# Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py - -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea -python letsencrypt-auto-source/build.py -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/build-lea -cp ${temp_dir}/original-lea letsencrypt-auto-source/letsencrypt-auto - -cd $temp_dir - -if ! cmp -s original-lea build-lea; then - echo "letsencrypt-auto-source/letsencrypt-auto doesn't match output of \ -build.py." - FLAG=true -else - echo "letsencrypt-auto-source/letsencrypt-auto matches output of \ -build.py." -fi - -rm -rf $temp_dir - -if $FLAG ; then - exit 1 -fi diff --git a/tools/_venv_common.py b/tools/_venv_common.py new file mode 100755 index 000000000..0c24664b3 --- /dev/null +++ b/tools/_venv_common.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import shutil +import glob +import time +import subprocess +import sys + +def subprocess_with_print(command): + print(command) + subprocess.call(command, shell=True) + +def get_venv_python(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_linux): + return python_linux + if os.path.isfile(python_windows): + return python_windows + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + +def main(venv_name, venv_args, args): + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + subprocess_with_print(' '.join([ + sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', + venv_name, venv_args])) + + python_executable = get_venv_python(venv_name) + + subprocess_with_print(' '.join([ + python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) + command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command.extend(args) + subprocess_with_print(' '.join(command)) + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + +if __name__ == '__main__': + main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh deleted file mode 100755 index 0f0ff7e28..000000000 --- a/tools/_venv_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -xe - -VENV_NAME=${VENV_NAME:-venv} - -# .egg-info directories tend to cause bizarre problems (e.g. `pip -e -# .` might unexpectedly install letshelp-certbot only, in case -# `python letshelp-certbot/setup.py build` has been called -# earlier) -rm -rf *.egg-info - -# virtualenv setup is NOT idempotent: shutil.Error: -# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and -# `venv/bin/python2` are the same file -mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true -virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS -. ./$VENV_NAME/bin/activate - -# Use pipstrap to update Python packaging tools to only update to a well tested -# version and to work around https://github.com/pypa/pip/issues/4817 on older -# systems. -python letsencrypt-auto-source/pieces/pipstrap.py -./tools/pip_install.sh "$@" - -set +x -echo "Please run the following command to activate developer environment:" -echo "source $VENV_NAME/bin/activate" diff --git a/tools/install_and_test.py b/tools/install_and_test.py new file mode 100755 index 000000000..b16181aa5 --- /dev/null +++ b/tools/install_and_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# pip installs the requested packages in editable mode and runs unit tests on +# them. Each package is installed and tested in the order they are provided +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.py for more information on the versions pinned +# to. +from __future__ import print_function + +import os +import sys +import tempfile +import shutil +import subprocess +import re + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + if os.environ.get('CERTBOT_NO_PIN') == '1': + command = [sys.executable, '-m', 'pip', '-q', '-e'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + + new_args = [] + for arg in args: + if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested.' + .format(arg))) + else: + new_args.append(arg) + + for requirement in new_args: + current_command = command[:] + current_command.append(requirement) + call_with_print(' '.join(current_command)) + pkg = re.sub(r'\[\w+\]', '', requirement) + + if pkg == '.': + pkg = 'certbot' + + temp_cwd = tempfile.mkdtemp() + try: + call_with_print(' '.join([ + sys.executable, '-m', 'pytest', '--numprocesses', 'auto', + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + finally: + shutil.rmtree(temp_cwd) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh deleted file mode 100755 index 819f683aa..000000000 --- a/tools/install_and_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -e -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. - -if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -q -e" -else - pip_install="$(dirname $0)/pip_install_editable.sh" -fi - -temp_cwd=$(mktemp -d) -trap "rm -rf $temp_cwd" EXIT - -set -x -for requirement in "$@" ; do - $pip_install $requirement - pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] - pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names - if [ $pkg = "." ]; then - pkg="certbot" - fi - cd "$temp_cwd" - pytest --numprocesses auto --quiet --pyargs $pkg - cd - -done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index c8fb95351..ad44a55d0 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys - def read_file(file_path): """Reads in a Python requirements file. @@ -32,17 +31,17 @@ def read_file(file_path): return d -def print_requirements(requirements): - """Prints requirements to stdout. +def output_requirements(requirements): + """Prepare print requirements to stdout. :param dict requirements: mapping from a project to its pinned version """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) + return '\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items())) -def merge_requirements_files(*files): +def main(*files): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier @@ -54,8 +53,9 @@ def merge_requirements_files(*files): d = {} for f in files: d.update(read_file(f)) - print_requirements(d) + return output_requirements(d) if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) + merged_requirements = main(*sys.argv[1:]) + print(merged_requirements) diff --git a/tools/pip_install.py b/tools/pip_install.py new file mode 100755 index 000000000..d09997bf5 --- /dev/null +++ b/tools/pip_install.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. + +from __future__ import print_function, absolute_import + +import subprocess +import os +import sys +import re +import shutil +import tempfile + +import merge_requirements as merge_module +import readlink + +def find_tools_path(): + return os.path.dirname(readlink.main(__file__)) + +def certbot_oldest_processing(tools_path, args, test_constraints): + if args[0] != '-e' or len(args) != 2: + raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' + 'with a single -e argument.') + # remove any extras such as [dev] + pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + # packages like acme don't have any local oldest requirements + if not os.path.isfile(requirements): + requirements = None + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + + return requirements + +def certbot_normal_processing(tools_path, test_constraints): + repo_path = os.path.dirname(tools_path) + certbot_requirements = os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + with open(certbot_requirements, 'r') as fd: + data = fd.readlines() + with open(test_constraints, 'w') as fd: + for line in data: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + fd.write('{0}{1}'.format(search.group(1), os.linesep)) + +def merge_requirements(tools_path, test_constraints, all_constraints): + merged_requirements = merge_module.main( + os.path.join(tools_path, 'dev_constraints.txt'), + test_constraints + ) + with open(all_constraints, 'w') as fd: + fd.write(merged_requirements) + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + tools_path = find_tools_path() + working_dir = tempfile.mkdtemp() + try: + test_constraints = os.path.join(working_dir, 'test_constraints.txt') + all_constraints = os.path.join(working_dir, 'all_constraints.txt') + + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) + + merge_requirements(tools_path, test_constraints, all_constraints) + if requirements: + call_with_print(' '.join([ + sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, + '--requirement', requirements])) + + command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] + command.extend(args) + call_with_print(' '.join(command)) + finally: + shutil.rmtree(working_dir) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index 78e2afa17..000000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the -# top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If -# CERTBOT_OLDEST is set, this script must be run with `-e ` and -# no other arguments. - -# get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -all_constraints=$(mktemp) -test_constraints=$(mktemp) -trap "rm -f $all_constraints $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - if [ "$1" != "-e" -o "$#" -ne "2" ]; then - echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." - exit 1 - fi - pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] - requirements="$pkg_dir/local-oldest-requirements.txt" - # packages like acme don't have any local oldest requirements - if [ ! -f "$requirements" ]; then - unset requirements - fi - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi - -"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ - "$test_constraints" > "$all_constraints" - -set -x - -# install the requested packages using the pinned requirements as constraints -if [ -n "$requirements" ]; then - pip install -q --constraint "$all_constraints" --requirement "$requirements" -fi -pip install -q --constraint "$all_constraints" "$@" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py new file mode 100755 index 000000000..bdbdcadc5 --- /dev/null +++ b/tools/pip_install_editable.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# pip installs packages in editable mode using certbot-auto's requirements file +# as constraints + +from __future__ import absolute_import + +import sys + +import pip_install + +def main(args): + new_args = [] + for arg in args: + new_args.append('-e') + new_args.append(arg) + pip_install.main(new_args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh deleted file mode 100755 index 6130bf6e7..000000000 --- a/tools/pip_install_editable.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -# pip installs packages in editable mode using certbot-auto's requirements file -# as constraints - -args="" -for requirement in "$@" ; do - args="$args -e $requirement" -done - -"$(dirname $0)/pip_install.sh" $args diff --git a/tools/readlink.py b/tools/readlink.py index 02c74c48d..0199ce184 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -7,7 +7,12 @@ platforms. """ from __future__ import print_function + import os import sys -print(os.path.realpath(sys.argv[1])) +def main(link): + return os.path.realpath(link) + +if __name__ == '__main__': + print(main(sys.argv[1])) diff --git a/tools/venv.py b/tools/venv.py new file mode 100755 index 000000000..849c49119 --- /dev/null +++ b/tools/venv.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + command_python2_st_code = subprocess.call( + 'command -v python2', shell=True, stdout=fnull, stderr=fnull) + if not command_python2_st_code: + return '--python python2' + + command_python27_st_code = subprocess.call( + 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) + if not command_python27_st_code: + return '--python python2.7' + + raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) + +def main(): + if os.name == 'nt': + raise ValueError('Certbot for Windows is not supported on Python 2.x.') + + venv_args = get_venv_args() + + _venv_common.main('venv', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv.sh b/tools/venv.sh deleted file mode 100755 index 5692f9ebf..000000000 --- a/tools/venv.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -xe -# Developer virtualenv setup for Certbot client - -if command -v python2; then - export VENV_ARGS="--python python2" -elif command -v python2.7; then - export VENV_ARGS="--python python2.7" -else - echo "Couldn't find python2 or python2.7 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tools/venv3.py b/tools/venv3.py new file mode 100755 index 000000000..15db9495a --- /dev/null +++ b/tools/venv3.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + where_python3_st_code = subprocess.call( + 'where python3', shell=True, stdout=fnull, stderr=fnull) + command_python3_st_code = subprocess.call( + 'command -v python3', shell=True, stdout=fnull, stderr=fnull) + + if not where_python3_st_code or not command_python3_st_code: + return '--python python3' + + raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) + +def main(): + venv_args = get_venv_args() + + _venv_common.main('venv3', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv3.sh b/tools/venv3.sh deleted file mode 100755 index 07512f370..000000000 --- a/tools/venv3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -xe -# Developer Python3 virtualenv setup for Certbot - -if command -v python3; then - export VENV_NAME="${VENV_NAME:-venv3}" - export VENV_ARGS="--python python3" -else - echo "Couldn't find python3 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tox-win.ini b/tox-win.ini deleted file mode 100644 index fe063c264..000000000 --- a/tox-win.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -skipsdist = True -envlist = py{34,35,36,37}-cover - -[testenv] -deps = -e acme[dev] - -e .[dev] -commands = pytest -n auto --pyargs acme - pytest -n auto --pyargs certbot - -[testenv:cover] -commands = pytest -n auto --cov acme --pyargs acme - pytest -n auto --cov certbot --cov-append --pyargs certbot diff --git a/tox.cover.py b/tox.cover.py new file mode 100755 index 000000000..8bbce2d09 --- /dev/null +++ b/tox.cover.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import argparse +import subprocess +import os +import sys + +DEFAULT_PACKAGES = [ + 'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns', + 'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy', + 'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns', + 'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53', + 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] + +COVER_THRESHOLDS = { + 'certbot': 98, + 'acme': 100, + 'certbot_apache': 100, + 'certbot_dns_cloudflare': 98, + 'certbot_dns_cloudxns': 99, + 'certbot_dns_digitalocean': 98, + 'certbot_dns_dnsimple': 98, + 'certbot_dns_dnsmadeeasy': 99, + 'certbot_dns_gehirn': 97, + 'certbot_dns_google': 99, + 'certbot_dns_linode': 98, + 'certbot_dns_luadns': 98, + 'certbot_dns_nsone': 99, + 'certbot_dns_ovh': 97, + 'certbot_dns_rfc2136': 99, + 'certbot_dns_route53': 92, + 'certbot_dns_sakuracloud': 97, + 'certbot_nginx': 97, + 'certbot_postfix': 100, + 'letshelp_certbot': 100 +} + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def cover(package): + threshold = COVER_THRESHOLDS.get(package) + + if not threshold: + raise ValueError('Unrecognized package: {0}'.format(package)) + + pkg_dir = package.replace('_', '-') + + if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested/covered.' + .format(pkg_dir))) + return + + subprocess.call([ + sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', + '--numprocesses', 'auto', '--pyargs', package]) + subprocess.call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', + '{0}/*'.format(pkg_dir), '--show-missing']) + +def main(): + description = """ +This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order +to generate separate stats for each package. It should be removed once those +packages are moved to a separate repo. + +Option -e makes sure we fail fast and don't submit to codecov.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--packages', nargs='+') + + args = parser.parse_args() + + packages = args.packages or DEFAULT_PACKAGES + + # --cov-append is on, make sure stats are correct + try: + os.remove('.coverage') + except OSError: + pass + + for package in packages: + cover(package) + +if __name__ == '__main__': + main() diff --git a/tox.cover.sh b/tox.cover.sh deleted file mode 100755 index c68e757de..000000000 --- a/tox.cover.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -xe - -# USAGE: ./tox.cover.sh [package] -# -# This script is used by tox.ini (and thus Travis CI) in order to -# generate separate stats for each package. It should be removed once -# those packages are moved to separate repo. -# -# -e makes sure we fail fast and don't submit to codecov - -if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" -else - pkgs="$@" -fi - -cover () { - if [ "$1" = "certbot" ]; then - min=98 - elif [ "$1" = "acme" ]; then - min=100 - elif [ "$1" = "certbot_apache" ]; then - min=100 - elif [ "$1" = "certbot_dns_cloudflare" ]; then - min=98 - elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=99 - elif [ "$1" = "certbot_dns_digitalocean" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsimple" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=99 - elif [ "$1" = "certbot_dns_gehirn" ]; then - min=97 - elif [ "$1" = "certbot_dns_google" ]; then - min=99 - elif [ "$1" = "certbot_dns_linode" ]; then - min=98 - elif [ "$1" = "certbot_dns_luadns" ]; then - min=98 - elif [ "$1" = "certbot_dns_nsone" ]; then - min=99 - elif [ "$1" = "certbot_dns_ovh" ]; then - min=97 - elif [ "$1" = "certbot_dns_rfc2136" ]; then - min=99 - elif [ "$1" = "certbot_dns_route53" ]; then - min=92 - elif [ "$1" = "certbot_dns_sakuracloud" ]; then - min=97 - elif [ "$1" = "certbot_nginx" ]; then - min=97 - elif [ "$1" = "certbot_postfix" ]; then - min=100 - elif [ "$1" = "letshelp_certbot" ]; then - min=100 - else - echo "Unrecognized package: $1" - exit 1 - fi - - pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" - coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing -} - -rm -f .coverage # --cov-append is on, make sure stats are correct -for pkg in $pkgs -do - cover $pkg -done diff --git a/tox.ini b/tox.ini index 9db06f78c..e38f1311e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,16 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},cover,lint +envlist = modification,py{34,35,36},py27-cover,lint [base] # pip installs the requested packages in editable mode -pip_install = {toxinidir}/tools/pip_install_editable.sh +pip_install = python {toxinidir}/tools/pip_install_editable.py # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. -install_and_test = {toxinidir}/tools/install_and_test.sh +install_and_test = python {toxinidir}/tools/install_and_test.py dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ @@ -38,7 +38,7 @@ all_packages = certbot-postfix \ letshelp-certbot install_packages = - {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} + python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} source_paths = acme/acme certbot @@ -64,7 +64,9 @@ source_paths = tests/lock_test.py [testenv] -passenv = TRAVIS +passenv = + TRAVIS + APPVEYOR commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -120,11 +122,17 @@ basepython = python2.7 commands = {[base]install_packages} -[testenv:cover] +[testenv:py27-cover] basepython = python2.7 commands = {[base]install_packages} - ./tox.cover.sh + python tox.cover.py + +[testenv:py37-cover] +basepython = python3.7 +commands = + {[base]install_packages} + python tox.cover.py [testenv:lint] basepython = python2.7 @@ -133,7 +141,7 @@ basepython = python2.7 # continue, but tox return code will reflect previous error commands = {[base]install_packages} - pylint --reports=n --rcfile=.pylintrc {[base]source_paths} + python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] basepython = python3 @@ -157,7 +165,7 @@ commands = # allow users to run the modification check by running `tox` [testenv:modification] commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py [testenv:apache_compat] commands = @@ -197,7 +205,7 @@ passenv = # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Trusty Docker image. commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = From 7352727a6507ee153dff485b5423940497186663 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 17:35:07 +0100 Subject: [PATCH 430/639] [URGENT] Fix the CI system (#6485) It is about the exit codes that are returned from the various scripts in tools during tox execution. Indeed, tox relies on the non-zero exit code from a given script to know that something failed during the execution. Previously, theses scripts were in bash, and a bash script returns an exit code that is the higher code returned from any of the command executed by the script. So if any command return a non-zero (in particular pylint or pytest), then the script return also non-zero. Now that these scripts are converted into python, pylint and pytest are executed via subprocess, that returns the exit code as variables. But if theses codes are not handled explicitly, the python script itself will return zero if no python exception occured. As a consequence currently, Certbot CI system is unable to detect any test error or lint error, because there is no exception in this case, only exit codes from the binaries executed. This PR fixes that, by handling correctly the exit code from the most critical scripts, install_and_test.py and tox.cover.py, but also all the scripts that I converted into Python and that could be executed in the context of a shell (via tox or directly for instance). --- tools/_venv_common.py | 20 +++++++++++++------- tools/install_and_test.py | 14 +++++++++----- tools/pip_install.py | 15 ++++++++++----- tools/pip_install_editable.py | 5 +++-- tools/venv.py | 5 +++-- tools/venv3.py | 5 +++-- 6 files changed, 41 insertions(+), 23 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 0c24664b3..cce88f826 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -11,7 +11,7 @@ import sys def subprocess_with_print(command): print(command) - subprocess.call(command, shell=True) + return subprocess.call(command, shell=True) def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') @@ -35,17 +35,19 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ + exit_code = 0 + + exit_code = subprocess_with_print(' '.join([ sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + venv_name, venv_args])) or exit_code python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ + exit_code = subprocess_with_print(' '.join([ python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code command.extend(args) - subprocess_with_print(' '.join(command)) + exit_code = subprocess_with_print(' '.join(command)) or exit_code if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -63,5 +65,9 @@ def main(venv_name, venv_args, args): else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + return exit_code + if __name__ == '__main__': - main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) + sys.exit(main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:])) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b16181aa5..149ffc776 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -19,7 +19,7 @@ SKIP_PROJECTS_ON_WINDOWS = [ def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': @@ -37,10 +37,12 @@ def main(args): else: new_args.append(arg) + exit_code = 0 + for requirement in new_args: current_command = command[:] current_command.append(requirement) - call_with_print(' '.join(current_command)) + exit_code = call_with_print(' '.join(current_command)) or exit_code pkg = re.sub(r'\[\w+\]', '', requirement) if pkg == '.': @@ -48,11 +50,13 @@ def main(args): temp_cwd = tempfile.mkdtemp() try: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pytest', '--numprocesses', 'auto', - '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code finally: shutil.rmtree(temp_cwd) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install.py b/tools/pip_install.py index d09997bf5..273ce5ec2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,11 +59,14 @@ def merge_requirements(tools_path, test_constraints, all_constraints): def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + + exit_code = 0 + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -76,15 +79,17 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + '--requirement', requirements])) or exit_code command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] command.extend(args) - call_with_print(' '.join(command)) + exit_code = call_with_print(' '.join(command)) or exit_code finally: shutil.rmtree(working_dir) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index bdbdcadc5..35cc2264d 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -13,7 +13,8 @@ def main(args): for arg in args: new_args.append('-e') new_args.append(arg) - pip_install.main(new_args) + + return pip_install.main(new_args) if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/venv.py b/tools/venv.py index 849c49119..2cc43251d 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -52,7 +53,7 @@ def main(): venv_args = get_venv_args() - _venv_common.main('venv', venv_args, REQUIREMENTS) + return _venv_common.main('venv', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/tools/venv3.py b/tools/venv3.py index 15db9495a..1bacc9c9a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -47,7 +48,7 @@ def get_venv_args(): def main(): venv_args = get_venv_args() - _venv_common.main('venv3', venv_args, REQUIREMENTS) + return _venv_common.main('venv3', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) From ee6f20d93d8d24662cab2c41d74ccf8f049d08f3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Nov 2018 15:39:24 -0800 Subject: [PATCH 431/639] Clarify letsencrypt-auto to certbot-auto message --- certbot/cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 99bf33180..2e9880505 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -172,10 +172,11 @@ def possible_deprecation_warning(config): # need warnings return if "CERTBOT_AUTO" not in os.environ: - logger.warning("You are running with an old copy of letsencrypt-auto that does " - "not receive updates, and is less reliable than more recent versions. " - "We recommend upgrading to the latest certbot-auto script, or using native " - "OS packages.") + logger.warning("You are running with an old copy of letsencrypt-auto" + " that does not receive updates, and is less reliable than more" + " recent versions. The letsencrypt client has also been renamed" + " to Certbot. We recommend upgrading to the latest certbot-auto" + " script, or using native OS packages.") logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ) From ad885afdb8f51568dd247f1df5e9e1caf99ade12 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 10 Nov 2018 01:17:17 +0100 Subject: [PATCH 432/639] Correct venv3 detection on windows (#6490) A little typo in the _venv_common.py block the script to finish correctly once the virtual environment has been setup on Windows. This PR fixes that. --- tools/_venv_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index cce88f826..b180518f9 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -55,7 +55,7 @@ def main(venv_name, venv_args, args): print('Please run the following command to activate developer environment:') print('source {0}/bin/activate'.format(venv_name)) print('-------------------------------------------------------------------') - elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') From b3d2ac5161b44d2f09c211048056248be224100a Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 14 Nov 2018 22:57:40 +0100 Subject: [PATCH 433/639] Fail-fast in test/cover/lint scripts (#6487) After #6485 and #6435, it appears that there is no good reason to not fail fast when test, cover or linting scripts are executed. This PR ensures to fail fast by invoking commands throught subprocess.check_call instead of subprocess.call, and by removing the handling of non-zero exit code at the end of theses scripts. As now coverage on Windows is executed with thresholds, I added specific thresholds for this platform. Because some portions of code that are done for Unix platform will not be executed on Windows. Note that coverage reports from Travis and AppVeyor are accumulated on Codecov. So if a file is covered up to 50 % on Linux, and all other parts are covered on Windows, then coverage is 100 % for Codecov. Note: that PR also fixes the ability of coverage tests to fail if thresholds are exceeded. * Use check_call to fail fast in all scripts related to tests/lint/coverage/deploy * Make specific coverage threshold for windows --- tests/modification-check.py | 4 +-- tools/_venv_common.py | 24 +++++++---------- tools/install_and_test.py | 14 ++++------ tools/pip_install.py | 14 ++++------ tools/pip_install_editable.py | 4 +-- tools/venv.py | 4 +-- tools/venv3.py | 4 +-- tox.cover.py | 49 ++++++++++++++++------------------- 8 files changed, 51 insertions(+), 66 deletions(-) diff --git a/tests/modification-check.py b/tests/modification-check.py index e00994b04..8abc0fbfe 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -70,7 +70,7 @@ def validate_scripts_content(repo_path, temp_cwd): # Compare file against the latest released version latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) - subprocess.call( + subprocess.check_call( [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( @@ -95,7 +95,7 @@ def main(): os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), os.path.join(temp_cwd, 'original-lea') ) - subprocess.call([sys.executable, os.path.normpath(os.path.join( + subprocess.check_call([sys.executable, os.path.normpath(os.path.join( repo_path, 'letsencrypt-auto-source/build.py'))]) shutil.copyfile( os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), diff --git a/tools/_venv_common.py b/tools/_venv_common.py index b180518f9..6134bd29d 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -11,13 +11,13 @@ import sys def subprocess_with_print(command): print(command) - return subprocess.call(command, shell=True) + subprocess.check_call(command, shell=True) def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') - python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_linux): return python_linux + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): return python_windows @@ -35,19 +35,17 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - exit_code = 0 - - exit_code = subprocess_with_print(' '.join([ + subprocess_with_print(' '.join([ sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) or exit_code + venv_name, venv_args])) python_executable = get_venv_python(venv_name) - exit_code = subprocess_with_print(' '.join([ + subprocess_with_print(' '.join([ python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code + command = [python_executable, os.path.normpath('./tools/pip_install.py')] command.extend(args) - exit_code = subprocess_with_print(' '.join(command)) or exit_code + subprocess_with_print(' '.join(command)) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -65,9 +63,7 @@ def main(venv_name, venv_args, args): else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) - return exit_code - if __name__ == '__main__': - sys.exit(main(os.environ.get('VENV_NAME', 'venv'), - os.environ.get('VENV_ARGS', ''), - sys.argv[1:])) + main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:]) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 149ffc776..2842ec069 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -19,7 +19,7 @@ SKIP_PROJECTS_ON_WINDOWS = [ def call_with_print(command, cwd=None): print(command) - return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': @@ -37,12 +37,10 @@ def main(args): else: new_args.append(arg) - exit_code = 0 - for requirement in new_args: current_command = command[:] current_command.append(requirement) - exit_code = call_with_print(' '.join(current_command)) or exit_code + call_with_print(' '.join(current_command)) pkg = re.sub(r'\[\w+\]', '', requirement) if pkg == '.': @@ -50,13 +48,11 @@ def main(args): temp_cwd = tempfile.mkdtemp() try: - exit_code = call_with_print(' '.join([ + call_with_print(' '.join([ sys.executable, '-m', 'pytest', '--numprocesses', 'auto', - '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) finally: shutil.rmtree(temp_cwd) - return exit_code - if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/pip_install.py b/tools/pip_install.py index 273ce5ec2..2c4a47c21 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,14 +59,12 @@ def merge_requirements(tools_path, test_constraints, all_constraints): def call_with_print(command, cwd=None): print(command) - return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() - exit_code = 0 - try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -79,17 +77,15 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - exit_code = call_with_print(' '.join([ + call_with_print(' '.join([ sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) or exit_code + '--requirement', requirements])) command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] command.extend(args) - exit_code = call_with_print(' '.join(command)) or exit_code + call_with_print(' '.join(command)) finally: shutil.rmtree(working_dir) - return exit_code - if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index 35cc2264d..8eaf3a9fa 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -14,7 +14,7 @@ def main(args): new_args.append('-e') new_args.append(arg) - return pip_install.main(new_args) + pip_install.main(new_args) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/venv.py b/tools/venv.py index 2cc43251d..8b6f92a56 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -53,7 +53,7 @@ def main(): venv_args = get_venv_args() - return _venv_common.main('venv', venv_args, REQUIREMENTS) + _venv_common.main('venv', venv_args, REQUIREMENTS) if __name__ == '__main__': - sys.exit(main()) + main() diff --git a/tools/venv3.py b/tools/venv3.py index 1bacc9c9a..9710806c5 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -48,7 +48,7 @@ def get_venv_args(): def main(): venv_args = get_venv_args() - return _venv_common.main('venv3', venv_args, REQUIREMENTS) + _venv_common.main('venv3', venv_args, REQUIREMENTS) if __name__ == '__main__': - sys.exit(main()) + main() diff --git a/tox.cover.py b/tox.cover.py index 8bbce2d09..ad6bb1a39 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -12,36 +12,33 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] COVER_THRESHOLDS = { - 'certbot': 98, - 'acme': 100, - 'certbot_apache': 100, - 'certbot_dns_cloudflare': 98, - 'certbot_dns_cloudxns': 99, - 'certbot_dns_digitalocean': 98, - 'certbot_dns_dnsimple': 98, - 'certbot_dns_dnsmadeeasy': 99, - 'certbot_dns_gehirn': 97, - 'certbot_dns_google': 99, - 'certbot_dns_linode': 98, - 'certbot_dns_luadns': 98, - 'certbot_dns_nsone': 99, - 'certbot_dns_ovh': 97, - 'certbot_dns_rfc2136': 99, - 'certbot_dns_route53': 92, - 'certbot_dns_sakuracloud': 97, - 'certbot_nginx': 97, - 'certbot_postfix': 100, - 'letshelp_certbot': 100 + 'certbot': {'linux': 98, 'windows': 94}, + 'acme': {'linux': 100, 'windows': 99}, + 'certbot_apache': {'linux': 100, 'windows': 100}, + 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, + 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, + 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsimple': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99}, + 'certbot_dns_gehirn': {'linux': 97, 'windows': 97}, + 'certbot_dns_google': {'linux': 99, 'windows': 99}, + 'certbot_dns_linode': {'linux': 98, 'windows': 98}, + 'certbot_dns_luadns': {'linux': 98, 'windows': 98}, + 'certbot_dns_nsone': {'linux': 99, 'windows': 99}, + 'certbot_dns_ovh': {'linux': 97, 'windows': 97}, + 'certbot_dns_rfc2136': {'linux': 99, 'windows': 99}, + 'certbot_dns_route53': {'linux': 92, 'windows': 92}, + 'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97}, + 'certbot_nginx': {'linux': 97, 'windows': 97}, + 'certbot_postfix': {'linux': 100, 'windows': 100}, + 'letshelp_certbot': {'linux': 100, 'windows': 100} } SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] def cover(package): - threshold = COVER_THRESHOLDS.get(package) - - if not threshold: - raise ValueError('Unrecognized package: {0}'.format(package)) + threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] pkg_dir = package.replace('_', '-') @@ -51,10 +48,10 @@ def cover(package): .format(pkg_dir))) return - subprocess.call([ + subprocess.check_call([ sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', '--numprocesses', 'auto', '--pyargs', package]) - subprocess.call([ + subprocess.check_call([ sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) From 5073090a20fa59fae45b4d90bfb41635bc181911 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 16 Nov 2018 00:17:36 +0100 Subject: [PATCH 434/639] Update tools/venv3.py to support py launcher on Windows (#6493) Following some inconsistencies occurred during by developments, and in the light of #6508, it decided to wrote a PR that will take fully advantage of the conversion from bash to python to the development setup tools. This PR adresses several issues when trying to use the development setup tools (`tools/venv.py` and `tools/venv3.py`: * on Windows, `python` executable is not always in PATH (default behavior) * even if the option is checked, the `python` executable is not associated to the usually symlink `python3` on Windows * on Windows again, really powerful introspection of the available Python environments can be done with `py`, the Windows Python launcher * in general for all systems, `tools/venv.py` and `tools/venv3.py` ensures that the respective Python major version will be used to setup the virtual environment if available. * finally, the best and first candidate to test should be the Python executable used to launch the `tools/venv*.py` script. It was not relevant before because it was shell scripts, but do it is. The logic is shared in `_venv_common.py`, and will be called appropriately for both scripts. In priority decreasing order, python executable will be search and tested: * from the current Python executable, as exposed by `sys.executable` * from any python or pythonX (X as a python version like 2, 3 or 2.7 or 3.4) executable available in PATH * from the Windows Python launched `py` if available Individual changes were: * Update tools/venv3.py to support py launcher on Windows * Fix typo in help message * More explicit calls with space protection * Complete refactoring to take advantage of the python runtime, and control of the compatible version to use. --- tools/_venv_common.py | 104 ++++++++++++++++++++++++++++++++++++++---- tools/pip_install.py | 17 ++++--- tools/venv.py | 22 +-------- tools/venv3.py | 22 +-------- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 6134bd29d..c44d05bf7 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -8,11 +8,94 @@ import glob import time import subprocess import sys +import re + +VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') + + +class PythonExecutableNotFoundError(Exception): + pass + + +def find_python_executable(python_major): + # type: (int) -> str + """ + Find the relevant python executable that is of the given python major version. + Will test, in decreasing priority order: + * the current Python interpreter + * 'pythonX' executable in PATH (with X the given major version) if available + * 'python' executable in PATH if available + * Windows Python launcher 'py' executable in PATH if available + Incompatible python versions for Certbot will be evicted (eg. Python < 3.5 on Windows) + :param int python_major: the Python major version to target (2 or 3) + :rtype: str + :return: the relevant python executable path + :raise RuntimeError: if no relevant python executable path could be found + """ + python_executable_path = None + + # First try, current python executable + if _check_version('{0}.{1}.{2}'.format( + sys.version_info[0], sys.version_info[1], sys.version_info[2]), python_major): + return sys.executable + + # Second try, with python executables in path + versions_to_test = ['2.7', '2', ''] if python_major == 2 else ['3', ''] + for one_version in versions_to_test: + try: + one_python = 'python{0}'.format(one_version) + output = subprocess.check_output([one_python, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output.strip().split()[1], python_major): + return subprocess.check_output([one_python, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + # Last try, with Windows Python launcher + try: + env_arg = '-{0}'.format(python_major) + output_version = subprocess.check_output(['py', env_arg, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output_version.strip().split()[1], python_major): + return subprocess.check_output(['py', env_arg, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + if not python_executable_path: + raise RuntimeError('Error, no compatible Python {0} executable for Certbot could be found.' + .format(python_major)) + + +def _check_version(version_str, major_version): + search = VERSION_PATTERN.search(version_str) + + if not search: + return False + + version = (int(search.group(1)), int(search.group(2))) + + minimal_version_supported = (2, 7) + if major_version == 3 and os.name == 'nt': + minimal_version_supported = (3, 5) + elif major_version == 3: + minimal_version_supported = (3, 4) + + if version >= minimal_version_supported: + return True + + print('Incompatible python version for Certbot found: {0}'.format(version_str)) + return False + def subprocess_with_print(command): print(command) subprocess.check_call(command, shell=True) + def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): @@ -25,6 +108,7 @@ def get_venv_python(venv_path): 'Error, could not find python executable in venv path {0}: is it a valid venv ?' .format(venv_path))) + def main(venv_name, venv_args, args): for path in glob.glob('*.egg-info'): if os.path.isdir(path): @@ -35,17 +119,18 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ - sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' + .format(sys.executable, venv_name, venv_args)) python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ - python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] - command.extend(args) - subprocess_with_print(' '.join(command)) + subprocess_with_print('"{0}" {1}'.format( + python_executable, + os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) + subprocess_with_print('"{0}" {1} {2}'.format( + python_executable, + os.path.normpath('./tools/pip_install.py'), + ' '.join(args))) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -57,12 +142,13 @@ def main(venv_name, venv_args, args): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') - print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) print('---------------------------------------------------------------------------') else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + if __name__ == '__main__': main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), diff --git a/tools/pip_install.py b/tools/pip_install.py index 2c4a47c21..8878674c9 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -20,9 +20,11 @@ import tempfile import merge_requirements as merge_module import readlink + def find_tools_path(): return os.path.dirname(readlink.main(__file__)) + def certbot_oldest_processing(tools_path, args, test_constraints): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' @@ -37,6 +39,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): return requirements + def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( @@ -49,6 +52,7 @@ def certbot_normal_processing(tools_path, test_constraints): if search: fd.write('{0}{1}'.format(search.group(1), os.linesep)) + def merge_requirements(tools_path, test_constraints, all_constraints): merged_requirements = merge_module.main( os.path.join(tools_path, 'dev_constraints.txt'), @@ -57,10 +61,12 @@ def merge_requirements(tools_path, test_constraints, all_constraints): with open(all_constraints, 'w') as fd: fd.write(merged_requirements) + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() @@ -77,15 +83,14 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ - sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + call_with_print('"{0}" -m pip install -q --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) - command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] - command.extend(args) - call_with_print(' '.join(command)) + call_with_print('"{0}" -m pip install -q --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: shutil.rmtree(working_dir) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/tools/venv.py b/tools/venv.py index 8b6f92a56..93b012e76 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -1,11 +1,6 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - import os -import subprocess -import sys import _venv_common @@ -33,27 +28,14 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - command_python2_st_code = subprocess.call( - 'command -v python2', shell=True, stdout=fnull, stderr=fnull) - if not command_python2_st_code: - return '--python python2' - - command_python27_st_code = subprocess.call( - 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) - if not command_python27_st_code: - return '--python python2.7' - - raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) def main(): if os.name == 'nt': raise ValueError('Certbot for Windows is not supported on Python 2.x.') - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(2)) _venv_common.main('venv', venv_args, REQUIREMENTS) + if __name__ == '__main__': main() diff --git a/tools/venv3.py b/tools/venv3.py index 9710806c5..c2374ba5a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -1,12 +1,5 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - -import os -import subprocess -import sys - import _venv_common REQUIREMENTS = [ @@ -33,22 +26,11 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - where_python3_st_code = subprocess.call( - 'where python3', shell=True, stdout=fnull, stderr=fnull) - command_python3_st_code = subprocess.call( - 'command -v python3', shell=True, stdout=fnull, stderr=fnull) - - if not where_python3_st_code or not command_python3_st_code: - return '--python python3' - - raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) def main(): - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(3)) _venv_common.main('venv3', venv_args, REQUIREMENTS) + if __name__ == '__main__': main() From 7fe64c3b9bdc01bafcbeeea874a29c23c3cf4064 Mon Sep 17 00:00:00 2001 From: Ye Wang Date: Fri, 16 Nov 2018 12:37:06 -0500 Subject: [PATCH 435/639] Add clarification of what means in the case of creating a SAN cert. --- docs/using.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 1fa13e022..8adf00d4d 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -696,7 +696,9 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain``. Rather than copying, please point +``/etc/letsencrypt/live/$domain`` In the case of creating a SAN certificate +with multiple alternative names, ``$domain`` is the first domain passed in +via -d parameter. Rather than copying, please point your (web) server configuration directly to those files (or create symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest necessary files. From ca12921a60eee87eba9eba275390171810b34258 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Nov 2018 14:28:29 -0800 Subject: [PATCH 436/639] Don't mention _venv_common.sh in certbot-auto. This wasn't always the case, but nowadays, _venv_common is a developer tool and has nothing to do with certbot-auto. --- letsencrypt-auto-source/letsencrypt-auto | 3 +-- letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 12be26e19..6c98b71d4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -593,8 +593,7 @@ BootstrapArchCommon() { # - ArchLinux (x86_64) # # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.py + # only "virtualenv2" binary, not "virtualenv". deps=" python2 diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index c55527590..3be78d3f8 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -7,8 +7,7 @@ BootstrapArchCommon() { # - ArchLinux (x86_64) # # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.py + # only "virtualenv2" binary, not "virtualenv". deps=" python2 From 4e1c22779e95a9c86c70640309482b4f5113a4d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 19 Nov 2018 11:47:14 -0800 Subject: [PATCH 437/639] Fix up environment variable use in venv creation scripts (#6518) This PR has the value of VENV_NAME override any value set in the tools/venv* scripts. I also removed the use of VENV_ARGS. This was used in _venv_common.sh as a means of passing arguments for virtualenv between the scripts, however, there is no other use of the variable in this repository and passing the arguments through a function call is much more natural in Python. * Respect VENV_NAME in tools/venv*. * Stop using VENV_ARGS * Remove VENV_NAME_ENV_VAR and add docstrings. --- tools/_venv_common.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index c44d05bf7..bddda6302 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -1,4 +1,14 @@ #!/usr/bin/env python +"""Aids in creating a developer virtual environment for Certbot. + +When this module is run as a script, it takes the arguments that should +be passed to pip to install the Certbot packages as command line +arguments. The virtual environment will be created with the name "venv" +in the current working directory and will use the default version of +Python for the virtualenv executable in your PATH. You can change the +name of the virtual environment by setting the environment variable +VENV_NAME. +""" from __future__ import print_function @@ -110,12 +120,27 @@ def get_venv_python(venv_path): def main(venv_name, venv_args, args): + """Creates a virtual environment and installs packages. + + :param str venv_name: The name or path at where the virtual + environment should be created. + :param str venv_args: Command line arguments for virtualenv + :param str args: Command line arguments that should be given to pip + to install packages + """ + for path in glob.glob('*.egg-info'): if os.path.isdir(path): shutil.rmtree(path) else: os.remove(path) + env_venv_name = os.environ.get('VENV_NAME') + if env_venv_name: + print('Creating venv at {0}' + ' as specified in VENV_NAME'.format(env_venv_name)) + venv_name = env_venv_name + if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) @@ -150,6 +175,6 @@ def main(venv_name, venv_args, args): if __name__ == '__main__': - main(os.environ.get('VENV_NAME', 'venv'), - os.environ.get('VENV_ARGS', ''), + main('venv', + '', sys.argv[1:]) From 78cf8ec4de0a4030ac4c2dd45113e456acb1621f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 19 Nov 2018 23:28:59 +0100 Subject: [PATCH 438/639] Protect certbot-auto against automated downgrades (#6448) With current code, the certbot-auto self-upgrade process can make it actually to downgrade itself, because the comparison done is an equality test between local certbot-auto version and the remote one. This is a flaw for attackers, that could make certbot-auto break itself by falsely advertising it about an old version as the latest one available. A function is added to make a more advanced comparison between version. Certbot-auto will upgrade itself only if the local version is strictly inferior to the latest one available. For instance, a version 0.28.0 will not upgrade itself if the latest one available on internet is 0.27.1. Similarly, non-official versions like 0.28.0.dev0 will never trigger a self-upgrade, to help development workflows. This implementation relies only on the Python distribution installed by certbot-auto (supporting 2.7+) and basic shell operations, to be compatible with any UNIX-based system. * Check version with protection again downgrade * Create a stable version of letsencrypt-auto to use correctly self-upgrade functionality * Update letsencrypt-auto-source/letsencrypt-auto.template --- letsencrypt-auto-source/letsencrypt-auto | 36 ++++++++++++++++++- .../letsencrypt-auto.template | 36 ++++++++++++++++++- letsencrypt-auto-source/tests/auto_test.py | 15 +++++++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 12be26e19..9973607be 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -912,6 +912,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -1640,7 +1669,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 6d2977832..f431e32e4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -451,6 +451,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -635,7 +664,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index c5109e208..16c478f20 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -27,6 +27,19 @@ def tests_dir(): return dirname(abspath(__file__)) +def copy_stable(src, dst): + """ + Copy letsencrypt-auto, and replace its current version to its equivalent stable one. + This is needed to test correctly the self-upgrade functionality. + """ + copy(src, dst) + with open(dst, 'r') as file: + filedata = file.read() + filedata = re.sub(r'LE_AUTO_VERSION="(.*)\.dev0"', r'LE_AUTO_VERSION="\1"', filedata) + with open(dst, 'w') as file: + file.write(filedata) + + sys.path.insert(0, dirname(tests_dir())) from build import build as build_le_auto @@ -343,7 +356,7 @@ class AutoTests(TestCase): 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), 'v99.9.9/letsencrypt-auto.sig': signed('something else')} with serving(resources) as base_url: - copy(LE_AUTO_PATH, le_auto_path) + copy_stable(LE_AUTO_PATH, le_auto_path) try: out, err = run_le_auto(le_auto_path, venv_dir, base_url) except CalledProcessError as exc: From 1dd7db12e05c7a52f1b1092d76a14caf0bf4c4cc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 20 Nov 2018 00:38:37 +0100 Subject: [PATCH 439/639] Workaround for old pip versions that are not exposed as importable modules. (#6500) Fallback to pipstrap 1.5.0. Manipulate PATH variable on higher level to activate the virtual environment before calling pipstrap. --- letsencrypt-auto-source/letsencrypt-auto | 13 +++----- letsencrypt-auto-source/pieces/pipstrap.py | 14 +++------ tools/_venv_common.py | 36 ++++++++++++---------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9973607be..1417811f3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1244,13 +1244,11 @@ UNLIKELY_EOF cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -1289,7 +1287,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1301,7 +1299,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1377,10 +1375,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -1394,7 +1390,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1407,7 +1403,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1426,6 +1422,7 @@ def main(): if __name__ == '__main__': exit(main()) + UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index f21d36657..727040c3c 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,12 +1,10 @@ #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -45,7 +43,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +55,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -133,10 +131,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -150,7 +146,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +159,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +177,4 @@ def main(): if __name__ == '__main__': - exit(main()) \ No newline at end of file + exit(main()) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index bddda6302..2a48b0d2e 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -19,6 +19,7 @@ import time import subprocess import sys import re +import shlex VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') @@ -101,18 +102,18 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(command): - print(command) - subprocess.check_call(command, shell=True) +def subprocess_with_print(cmd, env=os.environ, shell=False): + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_python(venv_path): +def get_venv_bin_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return python_linux + return os.path.abspath(os.path.dirname(python_linux)) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return python_windows + return os.path.abspath(os.path.dirname(python_windows)) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -144,18 +145,19 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' - .format(sys.executable, venv_name, venv_args)) + command = [sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', venv_name] + command.extend(shlex.split(venv_args)) + subprocess_with_print(command) - python_executable = get_venv_python(venv_name) - - subprocess_with_print('"{0}" {1}'.format( - python_executable, - os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) - subprocess_with_print('"{0}" {1} {2}'.format( - python_executable, - os.path.normpath('./tools/pip_install.py'), - ' '.join(args))) + # We execute the two following commands in the context of the virtual environment, to install + # the packages in it. To do so, we append the venv binary to the PATH that will be used for + # these commands. With this trick, correct python executable will be selected. + new_environ = os.environ.copy() + new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) + subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), + env=new_environ, shell=True) + subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), + env=new_environ, shell=True) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific From fce5af50fdb0736ad7c19efcf2031b84dbda636d Mon Sep 17 00:00:00 2001 From: Benjamin Akhras Date: Tue, 20 Nov 2018 21:48:20 +0100 Subject: [PATCH 440/639] Fixed Typo in the examples section since .ini files are not supported. --- certbot-dns-google/certbot_dns_google/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index f19266737..b88260b07 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -98,7 +98,7 @@ Examples certbot certonly \\ --dns-google \\ - --dns-google-credentials ~/.secrets/certbot/google.ini \\ + --dns-google-credentials ~/.secrets/certbot/google.json \\ --dns-google-propagation-seconds 120 \\ -d example.com From a23d76beb0e2c9539670766045314a5d50f582a2 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 20 Nov 2018 23:06:09 +0100 Subject: [PATCH 441/639] [Windows] Change default paths for Certbot when run on Windows (#6416) Defaults path of Certbot are the following: config: /etc/letsencrypt workdir: /var/letsencrypt/lib logs: /var/letsencrypt/log On Windows, this translate into: config: C:\etc\letsencrypt workdir: C:\var\letsencrypt\lib logs: C:\var\letsencrypt\log As Windows does not follow the standard POSIX filesystem layout, theses paths do not have a lot of sense in this case. This PR sets the following default paths when Certbot is run on Windows: config: C:\Certbot workdir: C:\Certbot\lib logs: C:\Certbot\log Better to decide the default paths for Certbot before users start to run it on Windows, to avoid future migration procedures. --- certbot/compat.py | 27 +++++++++++++++++++++++++++ certbot/constants.py | 10 +++++----- certbot/display/ops.py | 5 ++++- certbot/tests/main_test.py | 7 ++++--- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/certbot/compat.py b/certbot/compat.py index d42febe81..7e509206f 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -172,3 +172,30 @@ def compare_file_modes(mode1, mode2): # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights. return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE) + +WINDOWS_DEFAULT_FOLDERS = { + 'config': 'C:\\Certbot', + 'work': 'C:\\Certbot\\lib', + 'logs': 'C:\\Certbot\\log', +} +LINUX_DEFAULT_FOLDERS = { + 'config': '/etc/letsencrypt', + 'work': '/var/letsencrypt/lib', + 'logs': '/var/letsencrypt/log', +} + +def get_default_folder(folder_type): + """ + Return the relevant default folder for the current OS + + :param str folder_type: The type of folder to retrieve (config, work or logs) + + :returns: The relevant default folder. + :rtype: str + + """ + if 'fcntl' in sys.modules: + # Linux specific + return LINUX_DEFAULT_FOLDERS[folder_type] + # Windows specific + return WINDOWS_DEFAULT_FOLDERS[folder_type] diff --git a/certbot/constants.py b/certbot/constants.py index a2de2d27a..5bf68589d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -4,7 +4,7 @@ import os import pkg_resources from acme import challenges - +from certbot import compat SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins" """Setuptools entry point group name for plugins.""" @@ -14,7 +14,7 @@ OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT = "letsencrypt.plugins" CLI_DEFAULTS = dict( config_files=[ - "/etc/letsencrypt/cli.ini", + os.path.join(compat.get_default_folder('config'), 'cli.ini'), # http://freedesktop.org/wiki/Software/xdg-user-dirs/ os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), @@ -85,9 +85,9 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", key_path=None, - config_dir="/etc/letsencrypt", - work_dir="/var/lib/letsencrypt", - logs_dir="/var/log/letsencrypt", + config_dir=compat.get_default_folder('config'), + work_dir=compat.get_default_folder('work'), + logs_dir=compat.get_default_folder('logs'), server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 1e15a8474..3dae1070b 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -4,9 +4,11 @@ import os import zope.component +from certbot import compat from certbot import errors from certbot import interfaces from certbot import util + from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -33,7 +35,8 @@ def get_email(invalid=False, optional=True): unsafe_suggestion = ("\n\nIf you really want to skip this, you can run " "the client with --register-unsafely-without-email " "but make sure you then backup your account key from " - "/etc/letsencrypt/accounts\n\n") + "{0}\n\n".format(os.path.join( + compat.get_default_folder('config'), 'accounts'))) if optional: if invalid: msg += unsafe_suggestion diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8d6a3e7ae..6c35f1fdb 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -944,8 +944,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.crypto_util.notAfter') @test_util.patch_get_utility() def test_certonly_new_request_success(self, mock_get_utility, mock_notAfter): - cert_path = '/etc/letsencrypt/live/foo.bar' - key_path = '/etc/letsencrypt/live/baz.qux' + cert_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/foo.bar')) + key_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/baz.qux')) date = '1970-01-01' mock_notAfter().date.return_value = date @@ -975,7 +975,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met reuse_key=False): # pylint: disable=too-many-locals,too-many-arguments,too-many-branches cert_path = test_util.vector_path('cert_512.pem') - chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' + chain_path = os.path.normpath(os.path.join(self.config.config_dir, + 'live/foo.bar/fullchain.pem')) mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path, cert_path=cert_path, fullchain_path=chain_path) mock_lineage.should_autorenew.return_value = due_for_renewal From ca42945264c9a5d0663bae89949b0aa062895eba Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 20 Nov 2018 18:39:12 -0500 Subject: [PATCH 442/639] Fix test_sdists test farm test (#6524) * Switch to using _venv_common.py in test_sdists.sh * Upgrade setuptools in _venv_common.py * Upgrade setuptools before running pip_install --- tests/letstest/scripts/test_sdists.sh | 2 +- tools/_venv_common.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index f18a64065..0b9a91ffd 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -10,7 +10,7 @@ VERSION=$(letsencrypt-auto-source/version.py) export VENV_ARGS="-p $PYTHON" # setup venv -tools/_venv_common.sh --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt +tools/_venv_common.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate # build sdists diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 2a48b0d2e..ecd438f94 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -149,13 +149,15 @@ def main(venv_name, venv_args, args): command.extend(shlex.split(venv_args)) subprocess_with_print(command) - # We execute the two following commands in the context of the virtual environment, to install + # We execute the following commands in the context of the virtual environment, to install # the packages in it. To do so, we append the venv binary to the PATH that will be used for # these commands. With this trick, correct python executable will be selected. new_environ = os.environ.copy() new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), env=new_environ, shell=True) + subprocess_with_print("python -m pip install --upgrade 'setuptools>=30.3'", + env=new_environ, shell=True) subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), env=new_environ, shell=True) From e8e3534335d09288ee84e93c0a3e9c044387c593 Mon Sep 17 00:00:00 2001 From: schoen Date: Tue, 20 Nov 2018 20:55:51 -0800 Subject: [PATCH 443/639] Add a random sleep for noninteractive renewals (#6393) * WIP on adding a random sleep for noninteractive renewal * Update changelog * Log the fact that we're randomly sleeping * stdin may better define interactivity than stdout * Try mocking time.sleep for all tests * Move mocked sleep elsewhere * mock the right object * Somewhat ugly synthetic PTY trick * Move set -u down below self-exec * Revert "Move set -u down below self-exec" This reverts commit 6bde65a7384131052a5c90abaca5028615fc5186. * Revert "Somewhat ugly synthetic PTY trick" This reverts commit 89c704a4be5722dc0babaf3358be1cdcfea240e3. * Log specific duration of random sleep * Test coverage for random sleep() logic in main.py --- CHANGELOG.md | 4 +++- certbot/main.py | 12 ++++++++++++ certbot/tests/main_test.py | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a25890929..2bb08d4e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added -* +* Noninteractive renewals with `certbot renew` (those not started from a + terminal) now randomly sleep 1-480 seconds before beginning work in + order to spread out load spikes on the server side. ### Changed diff --git a/certbot/main.py b/certbot/main.py index 5d5251dd2..692dafeed 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,9 @@ from __future__ import print_function import functools import logging.handlers import os +import random import sys +import time import configobj import josepy as jose @@ -1243,6 +1245,16 @@ def renew(config, unused_plugins): :rtype: None """ + if not sys.stdin.isatty(): + # Noninteractive renewals include a random delay in order to spread + # out the load on the certificate authority servers, even if many + # users all pick the same time for renewals. This delay precedes + # running any hooks, so that side effects of the hooks (such as + # shutting down a web service) aren't prolonged unnecessarily. + sleep_time = random.randint(1, 60*8) + logger.info("Non-interactive renewal: random delay of %s seconds", sleep_time) + time.sleep(sleep_time) + try: renewal.handle_renewal_request(config) finally: diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 6c35f1fdb..2e53606a2 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -520,6 +520,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met '--work-dir', self.config.work_dir, '--logs-dir', self.config.logs_dir, '--text'] + self.mock_sleep = mock.patch('time.sleep').start() + def tearDown(self): # Reset globals in cli reload_module(cli) @@ -1093,6 +1095,26 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--reuse-key"] self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + @mock.patch('sys.stdin') + def test_noninteractive_renewal_delay(self, stdin): + stdin.isatty.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, should_renew=True) + self.assertEqual(self.mock_sleep.call_count, 1) + # in main.py: + # sleep_time = random.randint(1, 60*8) + sleep_call_arg = self.mock_sleep.call_args[0][0] + self.assertTrue(1 <= sleep_call_arg <= 60*8) + + @mock.patch('sys.stdin') + def test_interactive_no_renewal_delay(self, stdin): + stdin.isatty.return_value = True + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, should_renew=True) + self.assertEqual(self.mock_sleep.call_count, 0) + @mock.patch('certbot.renewal.should_renew') def test_renew_skips_recent_certs(self, should_renew): should_renew.return_value = False From 41bf9c70f67ccbebe2d318d49e79da974ad96c7e Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 21 Nov 2018 23:49:04 +0100 Subject: [PATCH 444/639] Update pinned version of cffi to 1.11.5 (#6512) Current pinned version of cffi is 1.10.0. This version does not provide pre-compiled wheels for latest Python versions on Windows. This implies on this plateform, when certbot is installed, to compile cffi from sources. But for that, the computer will need to have the Visual C compiler available locally. This environnement is really heavy to setup, and totally outside of the scope. This PR updates cffi to version 1.11.5, that has the required wheels, and makes certbot installable without a full .NET dev profile. --- letsencrypt-auto-source/letsencrypt-auto | 70 +++++++++---------- .../pieces/dependency-requirements.txt | 70 +++++++++---------- 2 files changed, 66 insertions(+), 74 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 1417811f3..891aab85e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1046,43 +1046,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index ae6079d96..eee0dd329 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -21,43 +21,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse From 3edb36c4cca925fec02c6133e9af7ee0dde324ae Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 22 Nov 2018 03:43:25 +0000 Subject: [PATCH 445/639] Revert acme/acme/client.py --- acme/acme/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 54ae58764..adc8ad9e3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -32,8 +32,11 @@ logger = logging.getLogger(__name__) # for SSL, which does allow these options to be configured. # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning if sys.version_info < (2, 7, 9): # pragma: no cover - import urllib3.contrib.pyopenssl # pylint: disable=import-error - urllib3.contrib.pyopenssl.inject_into_urllib3() + try: + requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore + except AttributeError: + import urllib3.contrib.pyopenssl # pylint: disable=import-error + urllib3.contrib.pyopenssl.inject_into_urllib3() DEFAULT_NETWORK_TIMEOUT = 45 From ff66b641e3b58eb732104d4dc9a2993417fefd6c Mon Sep 17 00:00:00 2001 From: schoen Date: Mon, 26 Nov 2018 11:46:57 -0800 Subject: [PATCH 446/639] Re-adding period --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 8adf00d4d..20c2c0a36 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -696,7 +696,7 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain`` In the case of creating a SAN certificate +``/etc/letsencrypt/live/$domain``. In the case of creating a SAN certificate with multiple alternative names, ``$domain`` is the first domain passed in via -d parameter. Rather than copying, please point your (web) server configuration directly to those files (or create From f65cb070b3b75a04580f6d53561f88cfbd2a42ff Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 26 Nov 2018 17:48:59 -0500 Subject: [PATCH 447/639] Automation for changelog changes during release (#6489) * Automation for changelog changes during release * Update changelog during release before modifying version numbers * don't link to the GitHub repo * no need to sign the commit bumping version numbers * simplify tail call --- tools/_changelog_top.txt | 21 +++++++++++++++++++++ tools/_release.sh | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tools/_changelog_top.txt diff --git a/tools/_changelog_top.txt b/tools/_changelog_top.txt new file mode 100644 index 000000000..6983b3a43 --- /dev/null +++ b/tools/_changelog_top.txt @@ -0,0 +1,21 @@ +## nextversion - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. diff --git a/tools/_release.sh b/tools/_release.sh index dab4eec3a..97e6e9c63 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -65,12 +65,19 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" +# Update changelog +sed -i "s/master/$(date +'%Y-%m-%d')/" CHANGELOG.md +git add CHANGELOG.md +git diff --cached +git commit -m "Update changelog for $version release" + for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . do sed -i 's/\.dev0//' "$pkg_dir/setup.py" git add "$pkg_dir/setup.py" done + SetVersion() { ver="$1" # bumping Certbot's version number is done differently @@ -232,6 +239,19 @@ echo tar cJvf $name.$rev.tar.xz $name.$rev echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz cd ~- +# Add master section to CHANGELOG.md +header=$(head -n 4 CHANGELOG.md) +body=$(sed s/nextversion/$nextversion/ tools/_changelog_top.txt) +footer=$(tail -n +5 CHANGELOG.md) +echo "$header + +$body + +$footer" > CHANGELOG.md +git add CHANGELOG.md +git diff --cached +git commit -m "Add contents to CHANGELOG.md for next version" + echo "New root: $root" echo "Test commands (in the letstest repo):" echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' From 1c23fea076caf7462096e35600058eaf200ca695 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 27 Nov 2018 17:24:34 -0800 Subject: [PATCH 448/639] ignore erroneously no-member lint error --- acme/acme/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index adc8ad9e3..af52b44ed 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning if sys.version_info < (2, 7, 9): # pragma: no cover try: + # pylint: disable=no-member requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore except AttributeError: import urllib3.contrib.pyopenssl # pylint: disable=import-error From 7527a6c95908af7862d8bb2d91ddde2755e8ce16 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 28 Nov 2018 19:01:05 -0800 Subject: [PATCH 449/639] Remove "beta" label from Apache plugin. (#6537) --- certbot-apache/certbot_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index f431b9dab..16de3a3d8 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -91,7 +91,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ - description = "Apache Web Server plugin - Beta" + description = "Apache Web Server plugin" OS_DEFAULTS = dict( server_root="/etc/apache2", From 7d0ac4713936193d35cc9e99f4dd99500d1ab718 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 29 Nov 2018 09:33:05 -0800 Subject: [PATCH 450/639] Change default privkey permissions while preserving group permissions (#6480) Fixes #1473. writes privkey.pem to 0600 by default for new lineages on renewals where a new privkey is generated, preserves group mode and gid Things this PR does not do: we talked about forcing 0600 on privkeys when a Certbot upgrade is detected. Instead, this PR only creates new lineages with the more restrictive permission to prevent renewal breakages. this doesn't solve many of the problems mentioned in #1473 that are not directly related to the title issue! * safe_open on archive keyfiles * keep group from current lineage * clean up integration test * safe_open can follow symlinks * fix tests on windows, maybe * Address Brad's comments * Revert changes to safe_open * Test chown is called when saving new key * Reorder chown operation * Changelog and documentation * Fix documentation style --- CHANGELOG.md | 4 ++- certbot/storage.py | 22 ++++++++++---- certbot/tests/storage_test.py | 45 ++++++++++++++++++++++++++++ docs/using.rst | 4 +++ tests/certbot-boulder-integration.sh | 30 +++++++++++++++++++ 5 files changed, 98 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cec3eded3..4b10a5805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,9 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed -* +* Private key permissioning changes: Renewal preserves existing group mode + & gid of previous private key material. Private keys for new + lineages (i.e. new certs, not renewed) default to 0o600. ### Fixed diff --git a/certbot/storage.py b/certbot/storage.py index 664ff172a..c90a95a73 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) ALL_FOUR = ("cert", "privkey", "chain", "fullchain") README = "README" CURRENT_VERSION = util.get_strict_version(certbot.__version__) +BASE_PRIVKEY_MODE = 0o600 def renewal_conf_files(config): @@ -1050,13 +1051,14 @@ class RenewableCert(object): # Put the data into the appropriate files on disk target = dict([(kind, os.path.join(live_dir, kind + ".pem")) for kind in ALL_FOUR]) + archive_target = dict([(kind, os.path.join(archive, kind + "1.pem")) + for kind in ALL_FOUR]) for kind in ALL_FOUR: - os.symlink(os.path.join(_relpath_from_file(archive, target[kind]), kind + "1.pem"), - target[kind]) + os.symlink(_relpath_from_file(archive_target[kind], target[kind]), target[kind]) with open(target["cert"], "wb") as f: logger.debug("Writing certificate to %s.", target["cert"]) f.write(cert) - with open(target["privkey"], "wb") as f: + with util.safe_open(archive_target["privkey"], "wb", chmod=BASE_PRIVKEY_MODE) as f: logger.debug("Writing private key to %s.", target["privkey"]) f.write(privkey) # XXX: Let's make sure to get the file permissions right here @@ -1120,14 +1122,15 @@ class RenewableCert(object): os.path.join(self.archive_dir, "{0}{1}.pem".format(kind, target_version))) for kind in ALL_FOUR]) + old_privkey = os.path.join( + self.archive_dir, "privkey{0}.pem".format(prior_version)) + # Distinguish the cases where the privkey has changed and where it # has not changed (in the latter case, making an appropriate symlink # to an earlier privkey version) if new_privkey is None: # The behavior below keeps the prior key by creating a new # symlink to the old key or the target of the old key symlink. - old_privkey = os.path.join( - self.archive_dir, "privkey{0}.pem".format(prior_version)) if os.path.islink(old_privkey): old_privkey = os.readlink(old_privkey) else: @@ -1135,9 +1138,16 @@ class RenewableCert(object): logger.debug("Writing symlink to old private key, %s.", old_privkey) os.symlink(old_privkey, target["privkey"]) else: - with open(target["privkey"], "wb") as f: + with util.safe_open(target["privkey"], "wb", chmod=BASE_PRIVKEY_MODE) as f: logger.debug("Writing new private key to %s.", target["privkey"]) f.write(new_privkey) + # If the previous privkey in this lineage has an existing gid or group mode > 0, + # let's keep those. + group_mode = stat.S_IMODE(os.stat(old_privkey).st_mode) & \ + (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) + mode = BASE_PRIVKEY_MODE | group_mode + os.chown(target["privkey"], -1, os.stat(old_privkey).st_gid) + os.chmod(target["privkey"], mode) # Save everything else with open(target["cert"], "wb") as f: diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 484ad7e16..5fd7746ed 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -13,6 +13,7 @@ import six import certbot from certbot import cli +from certbot import compat from certbot import errors from certbot.storage import ALL_FOUR @@ -91,6 +92,8 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): link) with open(link, "wb") as f: f.write(kind.encode('ascii') if value is None else value) + if kind == "privkey": + os.chmod(link, 0o600) def _write_out_ex_kinds(self): for kind in ALL_FOUR: @@ -543,6 +546,47 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.exists(temp_config_file)) + @test_util.broken_on_windows + @mock.patch("certbot.storage.relevant_values") + def test_save_successor_maintains_group_mode(self, mock_rv): + # Mock relevant_values() to claim that all values are relevant here + # (to avoid instantiating parser) + mock_rv.side_effect = lambda x: x + for kind in ALL_FOUR: + self._write_out_kind(kind, 1) + self.test_rc.update_all_links_to(1) + self.assertTrue(compat.compare_file_modes( + os.stat(self.test_rc.version("privkey", 1)).st_mode, 0o600)) + os.chmod(self.test_rc.version("privkey", 1), 0o444) + # If no new key, permissions should be the same (we didn't write any keys) + self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) + self.assertTrue(compat.compare_file_modes( + os.stat(self.test_rc.version("privkey", 2)).st_mode, 0o444)) + # If new key, permissions should be rest to 600 + preserved group + self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) + self.assertTrue(compat.compare_file_modes( + os.stat(self.test_rc.version("privkey", 3)).st_mode, 0o640)) + # If permissions reverted, next renewal will also revert permissions of new key + os.chmod(self.test_rc.version("privkey", 3), 0o404) + self.test_rc.save_successor(3, b"newcert", b"new_privkey", b"new chain", self.config) + self.assertTrue(compat.compare_file_modes( + os.stat(self.test_rc.version("privkey", 4)).st_mode, 0o600)) + + @test_util.broken_on_windows + @mock.patch("certbot.storage.relevant_values") + @mock.patch("certbot.storage.os.chown") + def test_save_successor_maintains_gid(self, mock_chown, mock_rv): + # Mock relevant_values() to claim that all values are relevant here + # (to avoid instantiating parser) + mock_rv.side_effect = lambda x: x + for kind in ALL_FOUR: + self._write_out_kind(kind, 1) + self.test_rc.update_all_links_to(1) + self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) + self.assertFalse(mock_chown.called) + self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) + self.assertTrue(mock_chown.called) + def _test_relevant_values_common(self, values): defaults = dict((option, cli.flag_default(option)) for option in ("authenticator", "installer", @@ -629,6 +673,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.config.live_dir, "README"))) self.assertTrue(os.path.exists(os.path.join( self.config.live_dir, "the-lineage.com", "README"))) + self.assertTrue(compat.compare_file_modes(os.stat(result.key_path).st_mode, 0o600)) with open(result.fullchain, "rb") as f: self.assertEqual(f.read(), b"cert" + b"chain") # Let's do it again and make sure it makes a different lineage diff --git a/docs/using.rst b/docs/using.rst index 20c2c0a36..bd6bb5f8f 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -717,6 +717,10 @@ The following files are available: put it into a safe, however - your server still needs to access this file in order for SSL/TLS to work. + .. note:: As of Certbot version 0.29.0, private keys for new certificate + default to ``0600``. Any changes to the group mode or group owner (gid) + of this file will be preserved on renewals. + This is what Apache needs for `SSLCertificateKeyFile `_, and Nginx for `ssl_certificate_key diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 73e668e28..709daa217 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -280,7 +280,29 @@ CheckCertCount() { fi } +CheckGroup() { + group_mode() { echo $((0`stat -c %a $1` & 070)); } + group_owner() { echo `stat -c %G $1`; } + if [ `group_mode $1` -ne `group_mode $2` ] ; then + echo "Expected group permission `group_mode $1`, got `group_mode $2` on file $2" + exit 1 + fi + if [ `group_owner $1` != `group_owner $2` ] ; then + echo "Expected group owner `group_owner $1`, got `group_owner $2` on file $2" + exit 1 + fi +} + +CheckOthersPermission() { + other_permission=$((0`stat -c %a $1` & 07)) + if [ $other_permission -ne 0 ] ; then + echo "Expected file $1 not to be accessible by others" + exit 1 + fi +} + CheckCertCount "le.wtf" 1 + # This won't renew (because it's not time yet) common_no_force_renew renew CheckCertCount "le.wtf" 1 @@ -294,6 +316,11 @@ rm -rf "$renewal_hooks_root" common renew --cert-name le.wtf --authenticator manual CheckCertCount "le.wtf" 2 +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey1.pem" +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey2.pem" +CheckGroup "${root}/conf/archive/le.wtf/privkey1.pem" "${root}/conf/archive/le.wtf/privkey2.pem" +chmod 0444 "${root}/conf/archive/le.wtf/privkey2.pem" + # test renewal with no executables in hook directories for hook_dir in $renewal_hooks_dirs; do touch "$hook_dir/file" @@ -310,6 +337,9 @@ CreateDirHooks sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks CheckCertCount "le.wtf" 3 +CheckGroup "${root}/conf/archive/le.wtf/privkey2.pem" "${root}/conf/archive/le.wtf/privkey3.pem" +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey3.pem" + if [ -s "$HOOK_DIRS_TEST" ]; then echo "Directory hooks were executed with --no-directory-hooks!" >&2 exit 1 From 8b5ac9e2573de9dfd4182bdcd98d7262a2bdc8f0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 29 Nov 2018 15:42:08 -0800 Subject: [PATCH 451/639] Ask people not to rewrite commits. (#6538) --- docs/contributing.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ead4d7e2b..f05b0fa9a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -359,7 +359,10 @@ Steps: 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. -5. Submit the PR. +5. Submit the PR. Once your PR is open, please do not force push to the branch + containing your pull request to squash or amend commits. We use `squash + merges `_ on PRs and + rewriting commits makes changes harder to track between reviews. 6. Did your tests pass on Travis? If they didn't, fix any errors. Asking for help From 0b5468e992ab57fa028ddf33ca2351cb37c8e1ee Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 30 Nov 2018 01:42:06 +0100 Subject: [PATCH 452/639] Implement POST-as-GET requests (#6522) * Setup an integration tests env against Pebble, that enforce post-as-get * Implement POST-as-GET requests, with fallback to GET. * Fix unit tests * Fix coverage. * Fix or ignore lint errors * Corrections after review * Correct test * Try a simple delegate approach * Add a test * Simplify test mocking * Clean comment --- acme/acme/client.py | 92 ++++++++++++++++++++--------- acme/acme/client_test.py | 30 +++++++++- tests/certbot-pebble-integration.sh | 16 +++++ tests/pebble-fetch.sh | 41 +++++++++++++ 4 files changed, 150 insertions(+), 29 deletions(-) create mode 100755 tests/certbot-pebble-integration.sh create mode 100755 tests/pebble-fetch.sh diff --git a/acme/acme/client.py b/acme/acme/client.py index af52b44ed..88938e999 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -200,22 +200,6 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes return datetime.datetime.now() + datetime.timedelta(seconds=seconds) - def poll(self, authzr): - """Poll Authorization Resource for status. - - :param authzr: Authorization Resource - :type authzr: `.AuthorizationResource` - - :returns: Updated Authorization Resource and HTTP response. - - :rtype: (`.AuthorizationResource`, `requests.Response`) - - """ - response = self.net.get(authzr.uri) - updated_authzr = self._authzr_from_response( - response, authzr.body.identifier, authzr.uri) - return updated_authzr, response - def _revoke(self, cert, rsn, url): """Revoke certificate. @@ -237,6 +221,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes raise errors.ClientError( 'Successful revocation must return HTTP OK status') + class Client(ClientBase): """ACME client for a v1 API. @@ -389,6 +374,22 @@ class Client(ClientBase): body=jose.ComparableX509(OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_ASN1, response.content))) + def poll(self, authzr): + """Poll Authorization Resource for status. + + :param authzr: Authorization Resource + :type authzr: `.AuthorizationResource` + + :returns: Updated Authorization Resource and HTTP response. + + :rtype: (`.AuthorizationResource`, `requests.Response`) + + """ + response = self.net.get(authzr.uri) + updated_authzr = self._authzr_from_response( + response, authzr.body.identifier, authzr.uri) + return updated_authzr, response + def poll_and_request_issuance( self, csr, authzrs, mintime=5, max_attempts=10): """Poll and request issuance. @@ -652,13 +653,29 @@ class ClientV2(ClientBase): body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) + authorizations.append(self._authzr_from_response(self._post_as_get(url), uri=url)) return messages.OrderResource( body=body, uri=response.headers.get('Location'), authorizations=authorizations, csr_pem=csr_pem) + def poll(self, authzr): + """Poll Authorization Resource for status. + + :param authzr: Authorization Resource + :type authzr: `.AuthorizationResource` + + :returns: Updated Authorization Resource and HTTP response. + + :rtype: (`.AuthorizationResource`, `requests.Response`) + + """ + response = self._post_as_get(authzr.uri) + updated_authzr = self._authzr_from_response( + response, authzr.body.identifier, authzr.uri) + return updated_authzr, response + def poll_and_finalize(self, orderr, deadline=None): """Poll authorizations and finalize the order. @@ -682,7 +699,7 @@ class ClientV2(ClientBase): responses = [] for url in orderr.body.authorizations: while datetime.datetime.now() < deadline: - authzr = self._authzr_from_response(self.net.get(url), uri=url) + authzr = self._authzr_from_response(self._post_as_get(url), uri=url) if authzr.body.status != messages.STATUS_PENDING: responses.append(authzr) break @@ -717,12 +734,12 @@ class ClientV2(ClientBase): self._post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) - response = self.net.get(orderr.uri) + response = self._post_as_get(orderr.uri) body = messages.Order.from_json(response.json()) if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self.net.get(body.certificate, + certificate_response = self._post_as_get(body.certificate, content_type=DER_CONTENT_TYPE).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() @@ -740,6 +757,32 @@ class ClientV2(ClientBase): """ return self._revoke(cert, rsn, self.directory['revokeCert']) + def _post_as_get(self, *args, **kwargs): + """ + Send GET request using the POST-as-GET protocol if needed. + The request will be first issued using POST-as-GET for ACME v2. If the ACME CA servers do + not support this yet and return an error, request will be retried using GET. + For ACME v1, only GET request will be tried, as POST-as-GET is not supported. + :param args: + :param kwargs: + :return: + """ + if self.acme_version >= 2: + # We add an empty payload for POST-as-GET requests + new_args = args[:1] + (None,) + args[1:] + try: + return self._post(*new_args, **kwargs) # pylint: disable=star-args + except messages.Error as error: + if error.code == 'malformed': + logger.debug('Error during a POST-as-GET request, ' + 'your ACME CA may not support it:\n%s', error) + logger.debug('Retrying request with GET.') + else: # pragma: no cover + raise + + # If POST-as-GET is not supported yet, we use a GET instead. + return self.net.get(*args, **kwargs) + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but @@ -769,12 +812,7 @@ class BackwardsCompatibleClientV2(object): self.client = ClientV2(directory, net=net) def __getattr__(self, name): - if name in vars(self.client): - return getattr(self.client, name) - elif name in dir(ClientBase): - return getattr(self.client, name) - else: - raise AttributeError() + return getattr(self.client, name) def new_account_and_tos(self, regr, check_tos_cb=None): """Combined register and agree_tos for V1, new_account for V2 @@ -944,7 +982,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes :rtype: `josepy.JWS` """ - jobj = obj.json_dumps(indent=2).encode() + jobj = obj.json_dumps(indent=2).encode() if obj else b'' logger.debug('JWS payload:\n%s', jobj) kwargs = { "alg": self.alg, diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 18b61d537..8b6cda47e 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +# pylint: disable=too-many-lines import copy import datetime import json @@ -730,9 +731,10 @@ class ClientV2Test(ClientTestBase): authz_response2 = self.response authz_response2.json.return_value = self.authz2.to_json() authz_response2.headers['Location'] = self.authzr2.uri - self.net.get.side_effect = (authz_response, authz_response2) - self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + with mock.patch('acme.client.ClientV2._post_as_get') as mock_post_as_get: + mock_post_as_get.side_effect = (authz_response, authz_response2) + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) @mock.patch('acme.client.datetime') def test_poll_and_finalize(self, mock_datetime): @@ -821,6 +823,30 @@ class ClientV2Test(ClientTestBase): self.response.json.return_value = self.regr.body.update( contact=()).to_json() + def test_post_as_get(self): + with mock.patch('acme.client.ClientV2._authzr_from_response') as mock_client: + mock_client.return_value = self.authzr2 + + self.client.poll(self.authzr2) # pylint: disable=protected-access + + self.client.net.post.assert_called_once_with( + self.authzr2.uri, None, acme_version=2, + new_nonce_url='https://www.letsencrypt-demo.org/acme/new-nonce') + self.client.net.get.assert_not_called() + + class FakeError(messages.Error): # pylint: disable=too-many-ancestors + """Fake error to reproduce a malformed request ACME error""" + def __init__(self): # pylint: disable=super-init-not-called + pass + @property + def code(self): + return 'malformed' + self.client.net.post.side_effect = FakeError() + + self.client.poll(self.authzr2) # pylint: disable=protected-access + + self.client.net.get.assert_called_once_with(self.authzr2.uri) + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/tests/certbot-pebble-integration.sh b/tests/certbot-pebble-integration.sh new file mode 100755 index 000000000..8711f72c1 --- /dev/null +++ b/tests/certbot-pebble-integration.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Simple integration test. Make sure to activate virtualenv beforehand +# (source venv/bin/activate) and that you are running Pebble test +# instance (see ./pebble-fetch.sh). + +cleanup_and_exit() { + EXIT_STATUS=$? + unset SERVER + exit $EXIT_STATUS +} + +trap cleanup_and_exit EXIT + +export SERVER=https://localhost:14000/dir + +./tests/certbot-boulder-integration.sh diff --git a/tests/pebble-fetch.sh b/tests/pebble-fetch.sh new file mode 100755 index 000000000..b0ba08961 --- /dev/null +++ b/tests/pebble-fetch.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Download and run Pebble instance for integration testing +set -xe + +PEBBLE_VERSION=2018-11-02 + +# We reuse the same GOPATH-style directory than for Boulder. +# Pebble does not need it, but it will make the installation consistent with Boulder's one. +export GOPATH=${GOPATH:-$HOME/gopath} +PEBBLEPATH=${PEBBLEPATH:-$GOPATH/src/github.com/letsencrypt/pebble} + +mkdir -p ${PEBBLEPATH} + +cat << UNLIKELY_EOF > "$PEBBLEPATH/docker-compose.yml" +version: '3' + +services: + pebble: + image: letsencrypt/pebble:${PEBBLE_VERSION} + command: pebble -strict ${PEBBLE_STRICT:-false} -dnsserver 10.77.77.1 + ports: + - 14000:14000 + environment: + - PEBBLE_VA_NOSLEEP=1 +UNLIKELY_EOF + +docker-compose -f "$PEBBLEPATH/docker-compose.yml" up -d pebble + +set +x # reduce verbosity while waiting for boulder +for n in `seq 1 150` ; do + if curl -k https://localhost:14000/dir 2>/dev/null; then + break + else + sleep 1 + fi +done + +if ! curl -k https://localhost:14000/dir 2>/dev/null; then + echo "timed out waiting for pebble to start" + exit 1 +fi From 73458daac92c56bb211517a333e52fc1fef09715 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 30 Nov 2018 14:44:47 -0800 Subject: [PATCH 453/639] whitelist docker-compose (#6516) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e38f1311e..61cee8eff 100644 --- a/tox.ini +++ b/tox.ini @@ -241,5 +241,5 @@ passenv = DOCKER_* commands = docker-compose run --rm --service-ports development bash -c 'tox -e lint' whitelist_externals = - docker + docker-compose passenv = DOCKER_* From d461655ec60c172544ddcdccde972d4bbd4f6ceb Mon Sep 17 00:00:00 2001 From: hannay Mohanna Date: Fri, 30 Nov 2018 17:59:19 -0600 Subject: [PATCH 454/639] removed unused sys imports (#6546) --- certbot-dns-gehirn/setup.py | 2 -- certbot-dns-linode/setup.py | 2 -- certbot-dns-ovh/setup.py | 2 -- certbot-dns-sakuracloud/setup.py | 2 -- 4 files changed, 8 deletions(-) diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 94ca74761..97d7ca699 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 4c2571f96..5cff50196 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 68ede8006..a0e62caa5 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 5af4c8a00..2222999d5 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages From 11dae13c2f95a36e3d0715b8626a6e7663e33a41 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 30 Nov 2018 17:40:30 -0800 Subject: [PATCH 455/639] Fix oldest Certbot version that works with Apache. (#6549) --- certbot-apache/local-oldest-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index e70ac0c7f..fd8869f7c 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.25.0 --e .[dev] +certbot[dev]==0.26.0 From 37d4b983c8abea179874b7d7c83d7dc8f4e41d0b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 30 Nov 2018 17:40:55 -0800 Subject: [PATCH 456/639] Don't use pytest.ini during the release. (#6550) Currently the release script in master fails for a few reasons. First, it's trying to use --numprocesses which comes from a pytest plugin that we're not installing in the release script. Second, many new warnings are raised when we're not using pinned versions of our dependencies. I'm not sure I agree, but one could argue that we should fix these issues and use the file during the release. I'm particularly hesitant for us to do this when it comes to warnings. We currently do not pin our dependencies in the release script. Do we really want to stop the release because a new package was released and is warning about something? One could argue we do because these warnings may be visible to the user, but they very rarely are and I think this makes the release process much too painful. I especially do not think we should block the release on this now as we are not up to date on the warnings raised by the latest versions of our packages so there is a lot to work through. * Don't use pytest.ini during the release. * State that pytest.ini isn't used in release script. --- pytest.ini | 5 ++++- tools/_release.sh | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index de1ecf267..9a5807f34 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,6 @@ +# This file isn't used while testing packages in tools/_release.sh so any +# settings we want to also change there must be added to the release script +# directly. [pytest] addopts = --numprocesses auto --pyargs # ResourceWarnings are ignored as errors, since they're raised at close @@ -6,4 +9,4 @@ addopts = --numprocesses auto --pyargs filterwarnings = error ignore:decodestring:DeprecationWarning - ignore:TLS-SNI-01:DeprecationWarning \ No newline at end of file + ignore:TLS-SNI-01:DeprecationWarning diff --git a/tools/_release.sh b/tools/_release.sh index 97e6e9c63..dd08974ae 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -169,7 +169,8 @@ pip freeze | tee $kgs pip install pytest for module in $subpkgs_modules ; do echo testing $module - pytest --pyargs $module + # use an empty configuration file rather than the one in the repo root + pytest -c <(echo '') --pyargs $module done cd ~- From 1f4297d0edc46801176bc072276db59126483f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20K=C3=A4stel?= Date: Tue, 4 Dec 2018 00:27:35 +0100 Subject: [PATCH 457/639] WIP External Account Binding (#6059) * Rough draft of External Account Binding. * Remove parameter --eab and namespace kid and hmac. Also add parameters to "register" subcommand. * Refactor as much as possible of the EAB functionality into ExternalAccountBinding class. * Remove debug line. * Added external account binding to Directory.Meta. * Rename to account_public_key, hmac_key and make some non-optional. Rename command line argument to --eab-hmac-key. * Error out when the server requires External Account Binding and the user has not supplied kid and hmac key. * Remove whitespace. * Refactor a bit to make it possible to set the url argument. * Move from_data method into client. * Revert "Move from_data method into client." This reverts commit 8963fae * Refactored to use json field on Registration. * Inherit from object according to Google Python Style Guide. * Move to two separate ifs. * Get tests to pass after External Account Binding additions. * messages.py back to 100% test coverage with some EAB tests. * .encode() this JSON key. * Set eab parameter default values to None. * * Remove unnecessary public key mock on most of the test. * Restructure the directory mock to be able to mock both True and False for externalAccountRequired easily. * Add EAB client tests. * Move external_account_required check into BackwardsCompatibleClientV2 to be able to mock it. * Update versions. * Try 0.29.0. * Revert "Try 0.29.0." This reverts commit 5779509 * Try 0.29.0 again. * Try this. * Fix pylint failures. * Add tests for external_account_required method. * Test not needed, avoid: ************* Module acme.client_test C: 1, 0: Too many lines in module (1258/1250) (too-many-lines) * Move real external_account_required method into ClientV2 and pass through to it in BackwardsCompatibleClientV2. * Handle missing meta key in server ACME directory. * Add docstring for BackwardsCompatibleClientV2.external_account_required(). * Add tests for BackwardsCompatibleClientV2.external_account_required(). * Fix coverage for ACMEv1 code in BackwardsCompatibleClientV2. * Disable pylint too-many-lines check for client_test.py. * Fix versions. * Remove whitespace that accidently snuck into an earlier commit. * Remove these two stray whitespaces also. * And the last couple of whitespaces. * Add External Account Binding to changelog. * Add dev0 suffix to setup.py. Co-Authored-By: robaman * Set to "-e acme[dev]" again. Co-Authored-By: robaman --- CHANGELOG.md | 2 + acme/acme/client.py | 16 ++++++++ acme/acme/client_test.py | 48 +++++++++++++++++++++++ acme/acme/messages.py | 29 +++++++++++++- acme/acme/messages_test.py | 34 +++++++++++++++++ certbot/cli.py | 12 ++++++ certbot/client.py | 22 ++++++++++- certbot/constants.py | 2 + certbot/tests/client_test.py | 72 ++++++++++++++++++++++++++++++++++- local-oldest-requirements.txt | 2 +- setup.py | 2 +- 11 files changed, 234 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b10a5805..03f93a881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Noninteractive renewals with `certbot renew` (those not started from a terminal) now randomly sleep 1-480 seconds before beginning work in order to spread out load spikes on the server side. +* Added External Account Binding support in cli and acme library. + Command line arguments --eab-kid and --eab-hmac-key added. ### Changed diff --git a/acme/acme/client.py b/acme/acme/client.py index 88938e999..41338e17e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -757,6 +757,13 @@ class ClientV2(ClientBase): """ return self._revoke(cert, rsn, self.directory['revokeCert']) + def external_account_required(self): + """Checks if ACME server requires External Account Binding authentication.""" + if hasattr(self.directory, 'meta') and self.directory.meta.external_account_required: + return True + else: + return False + def _post_as_get(self, *args, **kwargs): """ Send GET request using the POST-as-GET protocol if needed. @@ -919,6 +926,15 @@ class BackwardsCompatibleClientV2(object): else: return 1 + def external_account_required(self): + """Checks if the server requires an external account for ACMEv2 servers. + + Always return False for ACMEv1 servers, as it doesn't use External Account Binding.""" + if self.acme_version == 1: + return False + else: + return self.client.external_account_required() + class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 8b6cda47e..33ae3886b 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -284,6 +284,37 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.update_registration(mock.sentinel.regr, None) mock_client().update_registration.assert_called_once_with(mock.sentinel.regr, None) + # newNonce present means it will pick acme_version 2 + def test_external_account_required_true(self): + self.response.json.return_value = messages.Directory({ + 'newNonce': 'http://letsencrypt-test.com/acme/new-nonce', + 'meta': messages.Directory.Meta(external_account_required=True), + }).to_json() + + client = self._init() + + self.assertTrue(client.external_account_required()) + + # newNonce present means it will pick acme_version 2 + def test_external_account_required_false(self): + self.response.json.return_value = messages.Directory({ + 'newNonce': 'http://letsencrypt-test.com/acme/new-nonce', + 'meta': messages.Directory.Meta(external_account_required=False), + }).to_json() + + client = self._init() + + self.assertFalse(client.external_account_required()) + + def test_external_account_required_false_v1(self): + self.response.json.return_value = messages.Directory({ + 'meta': messages.Directory.Meta(external_account_required=False), + }).to_json() + + client = self._init() + + self.assertFalse(client.external_account_required()) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -823,6 +854,23 @@ class ClientV2Test(ClientTestBase): self.response.json.return_value = self.regr.body.update( contact=()).to_json() + def test_external_account_required_true(self): + self.client.directory = messages.Directory({ + 'meta': messages.Directory.Meta(external_account_required=True) + }) + + self.assertTrue(self.client.external_account_required()) + + def test_external_account_required_false(self): + self.client.directory = messages.Directory({ + 'meta': messages.Directory.Meta(external_account_required=False) + }) + + self.assertFalse(self.client.external_account_required()) + + def test_external_account_required_default(self): + self.assertFalse(self.client.external_account_required()) + def test_post_as_get(self): with mock.patch('acme.client.ClientV2._authzr_from_response') as mock_client: mock_client.return_value = self.authzr2 diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 7e86b0c3b..4400a6c31 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,6 +1,7 @@ """ACME protocol messages.""" import collections import six +import json import josepy as jose @@ -8,6 +9,7 @@ from acme import challenges from acme import errors from acme import fields from acme import util +from acme import jws OLD_ERROR_PREFIX = "urn:acme:error:" ERROR_PREFIX = "urn:ietf:params:acme:error:" @@ -27,6 +29,7 @@ ERROR_CODES = { 'tls': 'The server experienced a TLS error during domain verification', 'unauthorized': 'The client lacks sufficient authorization', 'unknownHost': 'The server could not resolve a domain name', + 'externalAccountRequired': 'The server requires external account binding', } ERROR_TYPE_DESCRIPTIONS = dict( @@ -176,6 +179,7 @@ class Directory(jose.JSONDeSerializable): _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) caa_identities = jose.Field('caaIdentities', omitempty=True) + external_account_required = jose.Field('externalAccountRequired', omitempty=True) def __init__(self, **kwargs): kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) @@ -258,6 +262,24 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" +class ExternalAccountBinding(object): + """ACME External Account Binding""" + + @classmethod + def from_data(cls, account_public_key, kid, hmac_key, directory): + """Create External Account Binding Resource from contact details, kid and hmac.""" + + key_json = json.dumps(account_public_key.to_partial_json()).encode() + decoded_hmac_key = jose.b64.b64decode(hmac_key) + url = directory["newAccount"] + + eab = jws.JWS.sign(key_json, jose.jwk.JWKOct(key=decoded_hmac_key), + jose.jwa.HS256, None, + url, kid) + + return eab.to_partial_json() + + class Registration(ResourceBody): """Registration Resource Body. @@ -275,12 +297,13 @@ class Registration(ResourceBody): status = jose.Field('status', omitempty=True) terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) only_return_existing = jose.Field('onlyReturnExisting', omitempty=True) + external_account_binding = jose.Field('externalAccountBinding', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' @classmethod - def from_data(cls, phone=None, email=None, **kwargs): + def from_data(cls, phone=None, email=None, external_account_binding=None, **kwargs): """Create registration resource from contact details.""" details = list(kwargs.pop('contact', ())) if phone is not None: @@ -288,6 +311,10 @@ class Registration(ResourceBody): if email is not None: details.extend([cls.email_prefix + mail for mail in email.split(',')]) kwargs['contact'] = tuple(details) + + if external_account_binding: + kwargs['external_account_binding'] = external_account_binding + return cls(**kwargs) def _filter_contact(self, prefix): diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 876fbe825..7efaaa1a3 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -174,6 +174,24 @@ class DirectoryTest(unittest.TestCase): self.assertTrue(result) +class ExternalAccountBindingTest(unittest.TestCase): + def setUp(self): + from acme.messages import Directory + self.key = jose.jwk.JWKRSA(key=KEY.public_key()) + self.kid = "kid-for-testing" + self.hmac_key = "hmac-key-for-testing" + self.dir = Directory({ + 'newAccount': 'http://url/acme/new-account', + }) + + def test_from_data(self): + from acme.messages import ExternalAccountBinding + eab = ExternalAccountBinding.from_data(self.key, self.kid, self.hmac_key, self.dir) + + self.assertEqual(len(eab), 3) + self.assertEqual(sorted(eab.keys()), sorted(['protected', 'payload', 'signature'])) + + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" @@ -205,6 +223,22 @@ class RegistrationTest(unittest.TestCase): 'mailto:admin@foo.com', )) + def test_new_registration_from_data_with_eab(self): + from acme.messages import NewRegistration, ExternalAccountBinding, Directory + key = jose.jwk.JWKRSA(key=KEY.public_key()) + kid = "kid-for-testing" + hmac_key = "hmac-key-for-testing" + directory = Directory({ + 'newAccount': 'http://url/acme/new-account', + }) + eab = ExternalAccountBinding.from_data(key, kid, hmac_key, directory) + reg = NewRegistration.from_data(email='admin@foo.com', external_account_binding=eab) + self.assertEqual(reg.contact, ( + 'mailto:admin@foo.com', + )) + self.assertEqual(sorted(reg.external_account_binding.keys()), + sorted(['protected', 'payload', 'signature'])) + def test_phones(self): self.assertEqual(('1234',), self.reg.phones) diff --git a/certbot/cli.py b/certbot/cli.py index 882ddba86..7d2a6ccfc 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -944,6 +944,18 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "specified or you already have a certificate with the same " "name. In the case of a name collision it will append a number " "like 0001 to the file path name. (default: Ask)") + helpful.add( + [None, "run", "certonly", "register"], + "--eab-kid", dest="eab_kid", + metavar="EAB_KID", + help="Key Identifier for External Account Binding" + ) + helpful.add( + [None, "run", "certonly", "register"], + "--eab-hmac-key", dest="eab_hmac_key", + metavar="EAB_HMAC_KEY", + help="HMAC key for External Account Binding" + ) helpful.add( [None, "run", "certonly", "manage", "delete", "certificates", "renew", "enhance"], "--cert-name", dest="certname", diff --git a/certbot/client.py b/certbot/client.py index e634b6bd9..38b77a772 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -203,9 +203,27 @@ def perform_registration(acme, config, tos_cb): :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` """ + + eab_credentials_supplied = config.eab_kid and config.eab_hmac_key + if eab_credentials_supplied: + account_public_key = acme.client.net.key.public_key() + eab = messages.ExternalAccountBinding.from_data(account_public_key=account_public_key, + kid=config.eab_kid, + hmac_key=config.eab_hmac_key, + directory=acme.client.directory) + else: + eab = None + + if acme.external_account_required(): + if not eab_credentials_supplied: + msg = ("Server requires external account binding." + " Please use --eab-kid and --eab-hmac-key.") + raise errors.Error(msg) + try: - return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), - tos_cb) + newreg = messages.NewRegistration.from_data(email=config.email, + external_account_binding=eab) + return acme.new_account_and_tos(newreg, tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: diff --git a/certbot/constants.py b/certbot/constants.py index 5bf68589d..eb4105f82 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -68,6 +68,8 @@ CLI_DEFAULTS = dict( directory_hooks=True, reuse_key=False, disable_renew_updates=False, + eab_hmac_key=None, + eab_kid=None, # Subparsers num=None, diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a4e70ce35..330529fc6 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -13,6 +13,8 @@ from certbot import util import certbot.tests.util as test_util +from josepy import interfaces + KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") @@ -64,9 +66,28 @@ class RegisterTest(test_util.ConfigTestCase): tos_cb = mock.MagicMock() return register(self.config, self.account_storage, tos_cb) + @staticmethod + def _public_key_mock(): + m = mock.Mock(__class__=interfaces.JSONDeSerializable) + m.to_partial_json.return_value = '{"a": 1}' + return m + + @staticmethod + def _new_acct_dir_mock(): + return "/acme/new-account" + + @staticmethod + def _true_mock(): + return True + + @staticmethod + def _false_mock(): + return False + def test_no_tos(self): with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: mock_client.new_account_and_tos().terms_of_service = "http://tos" + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): mock_client().new_account_and_tos.side_effect = errors.Error @@ -78,7 +99,8 @@ class RegisterTest(test_util.ConfigTestCase): self.assertTrue(mock_handle.called) def test_it(self): - with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -91,6 +113,7 @@ class RegisterTest(test_util.ConfigTestCase): msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.eff.handle_subscription") as mock_handle: mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() @@ -104,6 +127,7 @@ class RegisterTest(test_util.ConfigTestCase): msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.eff.handle_subscription"): mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) @@ -115,7 +139,8 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_clnt: + mock_clnt().external_account_required.side_effect = self._false_mock with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -129,6 +154,7 @@ class RegisterTest(test_util.ConfigTestCase): def test_dry_run_no_staging_account(self, _rep, mock_get_email): """Tests dry-run for no staging account, expect account created with no email""" with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.eff.handle_subscription"): with mock.patch("certbot.account.report_new_account"): self.config.dry_run = True @@ -138,11 +164,53 @@ class RegisterTest(test_util.ConfigTestCase): # check Certbot created an account with no email. Contact should return empty self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) + def test_with_eab_arguments(self): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().client.directory.__getitem__ = mock.Mock( + side_effect=self._new_acct_dir_mock + ) + mock_client().external_account_required.side_effect = self._false_mock + with mock.patch("certbot.eff.handle_subscription"): + target = "certbot.client.messages.ExternalAccountBinding.from_data" + with mock.patch(target) as mock_eab_from_data: + self.config.eab_kid = "test-kid" + self.config.eab_hmac_key = "J2OAqW4MHXsrHVa_PVg0Y-L_R4SYw0_aL1le6mfblbE" + self._call() + + self.assertTrue(mock_eab_from_data.called) + + def test_without_eab_arguments(self): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().external_account_required.side_effect = self._false_mock + with mock.patch("certbot.eff.handle_subscription"): + target = "certbot.client.messages.ExternalAccountBinding.from_data" + with mock.patch(target) as mock_eab_from_data: + self.config.eab_kid = None + self.config.eab_hmac_key = None + self._call() + + self.assertFalse(mock_eab_from_data.called) + + def test_external_account_required_without_eab_arguments(self): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().client.net.key.public_key = mock.Mock(side_effect=self._public_key_mock) + mock_client().external_account_required.side_effect = self._true_mock + with mock.patch("certbot.eff.handle_subscription"): + with mock.patch("certbot.client.messages.ExternalAccountBinding.from_data"): + self.config.eab_kid = None + self.config.eab_hmac_key = None + + self.assertRaises(errors.Error, self._call) + def test_unsupported_error(self): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client().client.directory.__getitem__ = mock.Mock( + side_effect=self._new_acct_dir_mock + ) + mock_client().external_account_required.side_effect = self._false_mock with mock.patch("certbot.eff.handle_subscription") as mock_handle: mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 03226fc84..2346300a3 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ -acme[dev]==0.26.0 +-e acme[dev] diff --git a/setup.py b/setup.py index f8f5feadc..768501b82 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.26.0', + 'acme>=0.29.0.dev0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From d30afbea5782bdb30c36661f1a9cbaea0bfc5bc8 Mon Sep 17 00:00:00 2001 From: Oshawk Date: Mon, 3 Dec 2018 23:32:47 +0000 Subject: [PATCH 458/639] Remove duplicate route53 options (#6405) * Remove duplicate route53 options Duplicate route53 option (certbot-route53:auth) removed. * Remove duplicate route53 options from help Removed duplicate route53 options from help by adding a couple of if statements in cli.py. * Fix formatting * Use cleaner solution to remove duplicate route53 options from help Used a cleaner method to remove the duplicate option from the help text * Fix line too long * Add regression test Added a regression test and fixed a couple of minor issues. * Fix trailing whitespace * Add missing newline * Put newline in correct place --- certbot/cli.py | 4 +++- certbot/tests/cli_test.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 7d2a6ccfc..8557b12c8 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -859,7 +859,9 @@ class HelpfulArgumentParser(object): if chosen_topic == "everything": chosen_topic = "run" if chosen_topic == "all": - return dict([(t, True) for t in self.help_topics]) + # Addition of condition closes #6209 (removal of duplicate route53 option). + return dict([(t, True) if t != 'certbot-route53:auth' else (t, False) + for t in self.help_topics]) elif not chosen_topic: return dict([(t, False) for t in self.help_topics]) else: diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 69ef16597..1448b86e7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -431,6 +431,11 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertRaises(errors.Error, self.parse, "--allow-subset-of-names -d *.example.org".split()) + def test_route53_no_revert(self): + for help_flag in ['-h', '--help']: + for topic in ['all', 'plugins', 'dns-route53']: + self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" From daa77b502353424244050b0fb8c6daf6276cc1d6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 3 Dec 2018 16:42:52 -0800 Subject: [PATCH 459/639] During the release, reset local-oldest-requirements.txts that might have been changed during development. (#6547) * During the release, reset local-oldest-requirements.txts that might have been changed during development. * only modify the file if it exists * no need to specifically add certbot-compatibility-test because it doesn't have the file anyway * Use double quotes instead of single * escape dots and brackets * Escape and quote correctly --- tools/_release.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/_release.sh b/tools/_release.sh index dd08974ae..90c176b0e 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -265,6 +265,16 @@ if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 letsencrypt-auto-source/build.py git add letsencrypt-auto-source/letsencrypt-auto + for pkg_dir in $SUBPKGS_NO_CERTBOT . + do + if [ -f "$pkg_dir/local-oldest-requirements.txt" ]; then + sed -i "s/-e acme\[dev\]/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e acme/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e \.\[dev\]/certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e \./certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + git add "$pkg_dir/local-oldest-requirements.txt" + fi + done git diff git commit -m "Bump version to $nextversion" fi From f5aad1440f8143f003698670177fabfc5fa7bb9c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 4 Dec 2018 10:56:15 -0800 Subject: [PATCH 460/639] Conditionally depend on imgconverter for newer versions of Sphinx (#6536) Fixes #6343. * conditionally depend on imgconverter * Pin docutils dependency for old Sphinx bug --- CHANGELOG.md | 1 + docs/conf.py | 8 ++++++-- setup.py | 5 +++-- tools/dev_constraints.txt | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f93a881..6d9447ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed * Update code and dependencies to clean up Resource and Deprecation Warnings. +* Only depend on imgconverter extension for Sphinx >= 1.6 Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only diff --git a/docs/conf.py b/docs/conf.py index 2e6c5a9b7..c72d1c1cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,6 +17,8 @@ import os import re import sys +import sphinx + here = os.path.abspath(os.path.dirname(__file__)) @@ -33,14 +35,13 @@ sys.path.insert(0, os.path.abspath(os.path.join(here, '..'))) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.0' +needs_sphinx = '1.2' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', - 'sphinx.ext.imgconverter', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', @@ -48,6 +49,9 @@ extensions = [ 'repoze.sphinx.autointerface', ] +if sphinx.version_info >= (1, 6): + extensions.append('sphinx.ext.imgconverter') + autodoc_member_order = 'bysource' autodoc_default_flags = ['show-inheritance', 'private-members'] diff --git a/setup.py b/setup.py index 768501b82..382149054 100644 --- a/setup.py +++ b/setup.py @@ -68,9 +68,10 @@ dev3_extras = [ ] docs_extras = [ + # If you have Sphinx<1.5.1, you need docutils<0.13.1 + # https://github.com/sphinx-doc/sphinx/issues/3212 'repoze.sphinx.autointerface', - # sphinx.ext.imgconverter - 'Sphinx >=1.6', + 'Sphinx>=1.2', # Annotation support 'sphinx_rtd_theme', ] diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 65182bcde..778012d31 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -14,7 +14,7 @@ coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.7.14 dnspython==1.15.0 -docutils==0.14 +docutils==0.12 execnet==1.5.0 future==0.16.0 futures==3.1.1 From 9c0b27de6826fdcbfc26d588460ea02426a5d492 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 4 Dec 2018 10:59:23 -0800 Subject: [PATCH 461/639] Preserve other-read bit on private keys too (#6544) Not updating the guidelines in using.rst-- I don't want to encourage people to use this. now that Certbot preserves gid, the better way to set permissions is to change the group permisisons! * Preserve other-read bit on private keys too * fix integration test * fix and rename permission routines in integration tests --- certbot/storage.py | 10 ++++---- certbot/tests/storage_test.py | 6 ++--- tests/certbot-boulder-integration.sh | 35 ++++++++++++++++++---------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index c90a95a73..eb17e1d38 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -1141,11 +1141,11 @@ class RenewableCert(object): with util.safe_open(target["privkey"], "wb", chmod=BASE_PRIVKEY_MODE) as f: logger.debug("Writing new private key to %s.", target["privkey"]) f.write(new_privkey) - # If the previous privkey in this lineage has an existing gid or group mode > 0, - # let's keep those. - group_mode = stat.S_IMODE(os.stat(old_privkey).st_mode) & \ - (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) - mode = BASE_PRIVKEY_MODE | group_mode + # Preserve gid and (mode & 074) from previous privkey in this lineage. + old_mode = stat.S_IMODE(os.stat(old_privkey).st_mode) & \ + (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | \ + stat.S_IROTH) + mode = BASE_PRIVKEY_MODE | old_mode os.chown(target["privkey"], -1, os.stat(old_privkey).st_gid) os.chmod(target["privkey"], mode) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 5fd7746ed..d75f4f595 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -562,12 +562,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) self.assertTrue(compat.compare_file_modes( os.stat(self.test_rc.version("privkey", 2)).st_mode, 0o444)) - # If new key, permissions should be rest to 600 + preserved group + # If new key, permissions should be kept as 644 self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(compat.compare_file_modes( - os.stat(self.test_rc.version("privkey", 3)).st_mode, 0o640)) + os.stat(self.test_rc.version("privkey", 3)).st_mode, 0o644)) # If permissions reverted, next renewal will also revert permissions of new key - os.chmod(self.test_rc.version("privkey", 3), 0o404) + os.chmod(self.test_rc.version("privkey", 3), 0o400) self.test_rc.save_successor(3, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(compat.compare_file_modes( os.stat(self.test_rc.version("privkey", 4)).st_mode, 0o600)) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 709daa217..87e3cc319 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -280,13 +280,20 @@ CheckCertCount() { fi } -CheckGroup() { - group_mode() { echo $((0`stat -c %a $1` & 070)); } - group_owner() { echo `stat -c %G $1`; } - if [ `group_mode $1` -ne `group_mode $2` ] ; then - echo "Expected group permission `group_mode $1`, got `group_mode $2` on file $2" +CheckPermissions() { +# Args: +# Checks mode of two files match under + masked_mode() { echo $((0`stat -c %a $1` & 0$2)); } + if [ `masked_mode $1 $3` -ne `masked_mode $2 $3` ] ; then + echo "With $3 mask, expected mode `masked_mode $1 $3`, got `masked_mode $2 $3` on file $2" exit 1 fi +} + +CheckGID() { +# Args: +# Checks group owner of two files match + group_owner() { echo `stat -c %G $1`; } if [ `group_owner $1` != `group_owner $2` ] ; then echo "Expected group owner `group_owner $1`, got `group_owner $2` on file $2" exit 1 @@ -294,9 +301,11 @@ CheckGroup() { } CheckOthersPermission() { +# Args: +# Tests file's other/world permission against expected mode other_permission=$((0`stat -c %a $1` & 07)) - if [ $other_permission -ne 0 ] ; then - echo "Expected file $1 not to be accessible by others" + if [ $other_permission -ne $2 ] ; then + echo "Expected file $1 to have others mode $2, got $other_permission instead" exit 1 fi } @@ -316,9 +325,10 @@ rm -rf "$renewal_hooks_root" common renew --cert-name le.wtf --authenticator manual CheckCertCount "le.wtf" 2 -CheckOthersPermission "${root}/conf/archive/le.wtf/privkey1.pem" -CheckOthersPermission "${root}/conf/archive/le.wtf/privkey2.pem" -CheckGroup "${root}/conf/archive/le.wtf/privkey1.pem" "${root}/conf/archive/le.wtf/privkey2.pem" +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey1.pem" 0 +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey2.pem" 0 +CheckPermissions "${root}/conf/archive/le.wtf/privkey1.pem" "${root}/conf/archive/le.wtf/privkey2.pem" 074 +CheckGID "${root}/conf/archive/le.wtf/privkey1.pem" "${root}/conf/archive/le.wtf/privkey2.pem" chmod 0444 "${root}/conf/archive/le.wtf/privkey2.pem" # test renewal with no executables in hook directories @@ -337,8 +347,9 @@ CreateDirHooks sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks CheckCertCount "le.wtf" 3 -CheckGroup "${root}/conf/archive/le.wtf/privkey2.pem" "${root}/conf/archive/le.wtf/privkey3.pem" -CheckOthersPermission "${root}/conf/archive/le.wtf/privkey3.pem" +CheckGID "${root}/conf/archive/le.wtf/privkey2.pem" "${root}/conf/archive/le.wtf/privkey3.pem" +CheckPermissions "${root}/conf/archive/le.wtf/privkey2.pem" "${root}/conf/archive/le.wtf/privkey3.pem" 074 +CheckOthersPermission "${root}/conf/archive/le.wtf/privkey3.pem" 04 if [ -s "$HOOK_DIRS_TEST" ]; then echo "Directory hooks were executed with --no-directory-hooks!" >&2 From 9b9ba9b5fe4af5e0a76df001bd4fb9ca7690b4c0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 10:52:02 -0800 Subject: [PATCH 462/639] List packages that changed in CHANGELOG. --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d9447ae2..f2c89a4ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,13 @@ Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme +* certbot +* certbot-apache +* certbot-dns-cloudflare +* certbot-dns-digitalocean +* certbot-dns-google +* certbot-nginx More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/milestone/62?closed=1 From d151481dea200f99f65c67488124a5be7be53f2b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 10:52:20 -0800 Subject: [PATCH 463/639] Update changelog for 0.29.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c89a4ff..cf83679e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.29.0 - master +## 0.29.0 - 2018-12-05 ### Added From 6476663516b9c84d67964f1af58e868fa3f70431 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 10:57:43 -0800 Subject: [PATCH 464/639] Release 0.29.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 154 +++++++++++------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 21 +-- letsencrypt-auto | 154 +++++++++++------- letsencrypt-auto-source/certbot-auto.asc | 16 +- letsencrypt-auto-source/letsencrypt-auto | 26 +-- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +-- setup.py | 2 +- 27 files changed, 249 insertions(+), 186 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ad70c2947..635a98a32 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.29.0.dev0' +version = '0.29.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index e6f6f1e23..250db9a16 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index fe87317a7..8d9528559 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0" +LE_AUTO_VERSION="0.29.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -593,8 +593,7 @@ BootstrapArchCommon() { # - ArchLinux (x86_64) # # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # only "virtualenv2" binary, not "virtualenv". deps=" python2 @@ -912,6 +911,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -1017,43 +1045,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse @@ -1146,9 +1170,9 @@ pytz==2015.7 \ --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.12.1 \ - --hash=sha256:3f3f27a9d0f9092935efc78054ef324eb9f8166718270aefe036dfa1e4f68e1e \ - --hash=sha256:2109ecea94df90980be040490ff1d879971b024861539abb00054062388b612e +requests==2.20.0 \ + --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ + --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 six==1.10.0 \ --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a @@ -1185,6 +1209,15 @@ zope.interface==4.1.3 \ requests-toolbelt==0.8.0 \ --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 +chardet==3.0.2 \ + --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ + --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 +urllib3==1.21.1 \ + --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ + --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +certifi==2017.4.17 \ + --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ + --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a # Contains the requirements for the letsencrypt package. # @@ -1197,31 +1230,29 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.28.0 \ - --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ - --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a -acme==0.28.0 \ - --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ - --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 -certbot-apache==0.28.0 \ - --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ - --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb -certbot-nginx==0.28.0 \ - --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ - --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb +certbot==0.29.0 \ + --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ + --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 +acme==0.29.0 \ + --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ + --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 +certbot-apache==0.29.0 \ + --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ + --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd +certbot-nginx==0.29.0 \ + --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ + --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -1348,10 +1379,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -1641,7 +1670,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bfbbe0625..1c2a65dc0 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d615fa999..2743b9cd2 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index f9880270a..9bdedc418 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index a9d46d128..0de673a34 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index ac7bb1090..a1af50d91 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ab944d40d..61da2ca29 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 97d7ca699..42e08c062 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index aa1afdc93..93521c018 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 5cff50196..a19d4ef25 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index d0735d1ec..665108da6 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index aebff5304..b1fdd6dd5 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index a0e62caa5..3776ad518 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 914bfb6e6..2d8cb2293 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 6318cbb81..1461c6d02 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 2222999d5..59803256b 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0908c4c52..eea680f62 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0.dev0' +version = '0.29.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index f6b7defbd..464474c2a 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.29.0.dev0' +__version__ = '0.29.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index d26da361b..1dc8c5cd9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -67,6 +67,10 @@ optional arguments: with the same name. In the case of a name collision it will append a number like 0001 to the file path name. (default: Ask) + --eab-kid EAB_KID Key Identifier for External Account Binding (default: + None) + --eab-hmac-key EAB_HMAC_KEY + HMAC key for External Account Binding (default: None) --cert-name CERTNAME Certificate name to apply. This name is used by Certbot for housekeeping and in file paths; it doesn't affect the content of the certificate itself. To see @@ -108,7 +112,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.28.0 + "". (default: CertbotACMEClient/0.29.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -248,8 +252,8 @@ paths: None) --config-dir CONFIG_DIR Configuration directory. (default: /etc/letsencrypt) - --work-dir WORK_DIR Working directory. (default: /var/lib/letsencrypt) - --logs-dir LOGS_DIR Logs directory. (default: /var/log/letsencrypt) + --work-dir WORK_DIR Working directory. (default: /var/letsencrypt/lib) + --logs-dir LOGS_DIR Logs directory. (default: /var/letsencrypt/log) --server SERVER ACME Directory Resource URI. (default: https://acme-v02.api.letsencrypt.org/directory) @@ -473,7 +477,7 @@ plugins: using Sakura Cloud for DNS). (default: False) apache: - Apache Web Server plugin - Beta + Apache Web Server plugin --apache-enmod APACHE_ENMOD Path to the Apache 'a2enmod' binary (default: None) @@ -503,15 +507,6 @@ apache: Full path to Apache control script (default: apachectl) -certbot-route53:auth: - Obtain certificates using a DNS TXT record (if you are using AWS Route53 - for DNS). - - --certbot-route53:auth-propagation-seconds CERTBOT_ROUTE53:AUTH_PROPAGATION_SECONDS - The number of seconds to wait for DNS to propagate - before asking the ACME server to verify the DNS - record. (default: 10) - dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare for DNS). diff --git a/letsencrypt-auto b/letsencrypt-auto index fe87317a7..8d9528559 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0" +LE_AUTO_VERSION="0.29.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -593,8 +593,7 @@ BootstrapArchCommon() { # - ArchLinux (x86_64) # # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # only "virtualenv2" binary, not "virtualenv". deps=" python2 @@ -912,6 +911,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -1017,43 +1045,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse @@ -1146,9 +1170,9 @@ pytz==2015.7 \ --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.12.1 \ - --hash=sha256:3f3f27a9d0f9092935efc78054ef324eb9f8166718270aefe036dfa1e4f68e1e \ - --hash=sha256:2109ecea94df90980be040490ff1d879971b024861539abb00054062388b612e +requests==2.20.0 \ + --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ + --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 six==1.10.0 \ --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a @@ -1185,6 +1209,15 @@ zope.interface==4.1.3 \ requests-toolbelt==0.8.0 \ --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 +chardet==3.0.2 \ + --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ + --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 +urllib3==1.21.1 \ + --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ + --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +certifi==2017.4.17 \ + --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ + --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a # Contains the requirements for the letsencrypt package. # @@ -1197,31 +1230,29 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.28.0 \ - --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ - --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a -acme==0.28.0 \ - --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ - --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 -certbot-apache==0.28.0 \ - --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ - --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb -certbot-nginx==0.28.0 \ - --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ - --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb +certbot==0.29.0 \ + --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ + --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 +acme==0.29.0 \ + --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ + --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 +certbot-apache==0.29.0 \ + --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ + --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd +certbot-nginx==0.29.0 \ + --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ + --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -1348,10 +1379,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -1641,7 +1670,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 57745758b..a75e1a59f 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X -dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj -zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i -7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq -rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO -zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg -PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q== -=1hD2 +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwIH5YACgkQTRfJlc2X +dfLonwgAt8cgnRz0rLefTJSG+Zc4SnDK+zLqtn1RjlVTtoLC+vZYpAQPqG4SVjqS +plDkyZD/9vu4tlrH9BIFCUUb2r6l4RKbfxdJMMkUfF1yYAvQbGRRtSmPDANdbVjG +9HXFMXwc7smthzHkTR4PV1GvEpNfbXRUau+h4KrPUK5pKD3Sc8PFlOZ39v9PkIxR +yyuBNYsYGxgIkmQ2zFG/877Hj/gIm18uAZtLC9ojSzra0VfmLlmegoMrs4DnuJEx +nN0Hsh979kaHV1O0yItT+oRYqONTIgvO2UG/VOqFDvpTNM53BelwnLQHLgpQkhgH +bljL/pDdu4ga+Ydz0UR0V1ExaPn/2Q== +=gc2V -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index a804af523..8d9528559 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.0.dev0" +LE_AUTO_VERSION="0.29.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.28.0 \ - --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ - --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a -acme==0.28.0 \ - --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ - --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 -certbot-apache==0.28.0 \ - --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ - --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb -certbot-nginx==0.28.0 \ - --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ - --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb +certbot==0.29.0 \ + --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ + --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 +acme==0.29.0 \ + --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ + --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 +certbot-apache==0.29.0 \ + --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ + --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd +certbot-nginx==0.29.0 \ + --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ + --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 33f5b2c00007bde13d9f58cf7e48ad17d6998502..fc93a7804072c78caabc97212f6ba77f84fa52a3 100644 GIT binary patch literal 256 zcmV+b0ssDy@;uZPVg}NwjsPtP`?lI<{(s>aCE|BAx52f)1j#5-4sDvGoyw*QT|osS z7(iG`@e3YWpZ$J379OHLQ8iQ~K{MyXiRXaLxRU_Uu`KVrPgJU?Ev1~>h3@rj0I6X?6}bYFPUfoD%JUw!xxqUi Gz0>!iiGPRy literal 256 zcmV+b0ssC{t4ydxm0o9{N03G=j7lT5c_4fvp=Ak0G{=zFl<_N_YV;M+5gCs_o{c#_ zk}R?1daPA6KJFO+@o7i}i@E(nx&;8|8w zwwanT<%NEK7%&NBc6_9#QU}r> diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 401a8e25c..d8be0dc96 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.28.0 \ - --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ - --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a -acme==0.28.0 \ - --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ - --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 -certbot-apache==0.28.0 \ - --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ - --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb -certbot-nginx==0.28.0 \ - --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ - --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb +certbot==0.29.0 \ + --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ + --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 +acme==0.29.0 \ + --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ + --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 +certbot-apache==0.29.0 \ + --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ + --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd +certbot-nginx==0.29.0 \ + --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ + --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 diff --git a/setup.py b/setup.py index 382149054..2a9b2c203 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.29.0.dev0', + 'acme>=0.29.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From 245dbc7a95b931515b499baba81fcb4ce2d719a6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 10:57:45 -0800 Subject: [PATCH 465/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf83679e5..be1a8dba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.30.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.29.0 - 2018-12-05 ### Added From 3edfe9206938c0e30ad912b74c6d8ba47269314e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 10:57:46 -0800 Subject: [PATCH 466/639] Bump version to 0.30.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- local-oldest-requirements.txt | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 635a98a32..53a4eda27 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.29.0' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 250db9a16..2562e6f3f 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1c2a65dc0..405f8ff73 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 2743b9cd2..4f12e78f7 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9bdedc418..92ed37f35 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0de673a34..0fd52c4b0 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index a1af50d91..eec35b472 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 61da2ca29..21899f3d8 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 42e08c062..2bca63ccf 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 93521c018..ee484e402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index a19d4ef25..c91aabd4f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 665108da6..9f0fe149d 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index b1fdd6dd5..4ebfeb3e1 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 3776ad518..cbd77b901 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 2d8cb2293..2ea6699df 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 1461c6d02..84ca63f00 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 59803256b..cebd48e41 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index eea680f62..970089460 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 464474c2a..c96f7b28e 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.29.0' +__version__ = '0.30.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8d9528559..c6c39752c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.0" +LE_AUTO_VERSION="0.30.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 2346300a3..d582d5c65 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ --e acme[dev] +acme[dev]==0.29.0 From 11e0fa52d0f2e7d8780465f5b2202e0cb717e065 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 14:52:58 -0800 Subject: [PATCH 467/639] Fix default directories on Linux (#6560) * fix default directories * update changelog --- CHANGELOG.md | 5 +++-- certbot/compat.py | 4 ++-- certbot/tests/cli_test.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be1a8dba1..97ba2225b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,14 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* +* The default work and log directories have been changed back to + /var/lib/letsencrypt and /var/log/letsencrypt respectively. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* certbot More details about these changes can be found on our GitHub repo. diff --git a/certbot/compat.py b/certbot/compat.py index 7e509206f..7d936aa9d 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -180,8 +180,8 @@ WINDOWS_DEFAULT_FOLDERS = { } LINUX_DEFAULT_FOLDERS = { 'config': '/etc/letsencrypt', - 'work': '/var/letsencrypt/lib', - 'logs': '/var/letsencrypt/log', + 'work': '/var/lib/letsencrypt', + 'logs': '/var/log/letsencrypt', } def get_default_folder(folder_type): diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 1448b86e7..e16a1bdcf 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,6 +4,7 @@ import unittest import os import tempfile import copy +import sys import mock import six @@ -41,6 +42,15 @@ class TestReadFile(TempDirTestCase): self.assertEqual(contents, test_contents) +class FlagDefaultTest(unittest.TestCase): + """Tests cli.flag_default""" + + def test_linux_directories(self): + if 'fcntl' in sys.modules: + self.assertEqual(cli.flag_default('config_dir'), '/etc/letsencrypt') + self.assertEqual(cli.flag_default('work_dir'), '/var/lib/letsencrypt') + self.assertEqual(cli.flag_default('logs_dir'), '/var/log/letsencrypt') + class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods '''Test the cli args entrypoint''' From 1651bdd86b53fb4662ddfe578a513518ecdc305e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 Dec 2018 15:37:06 -0800 Subject: [PATCH 468/639] Fix default directories on Linux (#6560) (#6562) * fix default directories * update changelog (cherry picked from commit 11e0fa52d0f2e7d8780465f5b2202e0cb717e065) --- CHANGELOG.md | 23 +++++++++++++++++++++++ certbot/compat.py | 4 ++-- certbot/tests/cli_test.py | 10 ++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf83679e5..d93beeecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.29.1 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* The default work and log directories have been changed back to + /var/lib/letsencrypt and /var/log/letsencrypt respectively. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* certbot + +More details about these changes can be found on our GitHub repo. + ## 0.29.0 - 2018-12-05 ### Added diff --git a/certbot/compat.py b/certbot/compat.py index 7e509206f..7d936aa9d 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -180,8 +180,8 @@ WINDOWS_DEFAULT_FOLDERS = { } LINUX_DEFAULT_FOLDERS = { 'config': '/etc/letsencrypt', - 'work': '/var/letsencrypt/lib', - 'logs': '/var/letsencrypt/log', + 'work': '/var/lib/letsencrypt', + 'logs': '/var/log/letsencrypt', } def get_default_folder(folder_type): diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 1448b86e7..e16a1bdcf 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,6 +4,7 @@ import unittest import os import tempfile import copy +import sys import mock import six @@ -41,6 +42,15 @@ class TestReadFile(TempDirTestCase): self.assertEqual(contents, test_contents) +class FlagDefaultTest(unittest.TestCase): + """Tests cli.flag_default""" + + def test_linux_directories(self): + if 'fcntl' in sys.modules: + self.assertEqual(cli.flag_default('config_dir'), '/etc/letsencrypt') + self.assertEqual(cli.flag_default('work_dir'), '/var/lib/letsencrypt') + self.assertEqual(cli.flag_default('logs_dir'), '/var/log/letsencrypt') + class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods '''Test the cli args entrypoint''' From 87215c7fafc21febc1267540e71a2f8433d1dce3 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Dec 2018 15:47:59 -0800 Subject: [PATCH 469/639] Update changelog for 0.29.1 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d93beeecc..d75939464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.29.1 - master +## 0.29.1 - 2018-12-05 ### Added From be8638dad0e934880ded0cc4a583d42a65451546 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Dec 2018 16:31:07 -0800 Subject: [PATCH 470/639] Release 0.29.1 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 19 +++++++------ letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 88 insertions(+), 87 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 635a98a32..f76150483 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.29.0' +version = '0.29.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 250db9a16..aa91dda7b 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 8d9528559..6003afcf4 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.0" +LE_AUTO_VERSION="0.29.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.0 \ - --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ - --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 -acme==0.29.0 \ - --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ - --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 -certbot-apache==0.29.0 \ - --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ - --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd -certbot-nginx==0.29.0 \ - --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ - --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 +certbot==0.29.1 \ + --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ + --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 +acme==0.29.1 \ + --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ + --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c +certbot-apache==0.29.1 \ + --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ + --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 +certbot-nginx==0.29.1 \ + --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ + --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1c2a65dc0..fc17a4ec2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 2743b9cd2..0b3b1b0eb 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9bdedc418..6c5e460e3 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0de673a34..bc9847aae 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index a1af50d91..95c8e75b5 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 61da2ca29..62725c098 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 42e08c062..a6956b604 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 93521c018..d540b04fb 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index a19d4ef25..aa56650f4 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 665108da6..ffc499dfc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index b1fdd6dd5..6117bb361 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 3776ad518..329edee93 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 2d8cb2293..a7dd6d1ce 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 1461c6d02..907755a77 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 59803256b..6498365b6 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index eea680f62..ae0f63fe9 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.0' +version = '0.29.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 464474c2a..65144ba37 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.29.0' +__version__ = '0.29.1' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 1dc8c5cd9..37d67cfe0 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -112,7 +112,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.29.0 + "". (default: CertbotACMEClient/0.29.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -252,8 +252,8 @@ paths: None) --config-dir CONFIG_DIR Configuration directory. (default: /etc/letsencrypt) - --work-dir WORK_DIR Working directory. (default: /var/letsencrypt/lib) - --logs-dir LOGS_DIR Logs directory. (default: /var/letsencrypt/log) + --work-dir WORK_DIR Working directory. (default: /var/lib/letsencrypt) + --logs-dir LOGS_DIR Logs directory. (default: /var/log/letsencrypt) --server SERVER ACME Directory Resource URI. (default: https://acme-v02.api.letsencrypt.org/directory) @@ -480,9 +480,10 @@ apache: Apache Web Server plugin --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: None) + Path to the Apache 'a2enmod' binary (default: a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: None) + Path to the Apache 'a2dismod' binary (default: + a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -496,16 +497,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: False) + (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: False) + Ubuntu/Debian currently) (default: True) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apachectl) + apache2ctl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index 8d9528559..6003afcf4 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.0" +LE_AUTO_VERSION="0.29.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.0 \ - --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ - --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 -acme==0.29.0 \ - --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ - --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 -certbot-apache==0.29.0 \ - --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ - --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd -certbot-nginx==0.29.0 \ - --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ - --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 +certbot==0.29.1 \ + --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ + --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 +acme==0.29.1 \ + --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ + --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c +certbot-apache==0.29.1 \ + --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ + --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 +certbot-nginx==0.29.1 \ + --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ + --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index a75e1a59f..b4e520fb2 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwIH5YACgkQTRfJlc2X -dfLonwgAt8cgnRz0rLefTJSG+Zc4SnDK+zLqtn1RjlVTtoLC+vZYpAQPqG4SVjqS -plDkyZD/9vu4tlrH9BIFCUUb2r6l4RKbfxdJMMkUfF1yYAvQbGRRtSmPDANdbVjG -9HXFMXwc7smthzHkTR4PV1GvEpNfbXRUau+h4KrPUK5pKD3Sc8PFlOZ39v9PkIxR -yyuBNYsYGxgIkmQ2zFG/877Hj/gIm18uAZtLC9ojSzra0VfmLlmegoMrs4DnuJEx -nN0Hsh979kaHV1O0yItT+oRYqONTIgvO2UG/VOqFDvpTNM53BelwnLQHLgpQkhgH -bljL/pDdu4ga+Ydz0UR0V1ExaPn/2Q== -=gc2V +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwIbaAACgkQTRfJlc2X +dfLIGQf+JZr3oP89qyMREYAL3/Bx+vxsx+c01IuDaG1pBUeVwL5rdeU1kDZ1WkKb +61nCoMPbSLqLMor2IpobFj44lEFJS1WYrtfe8sgMLeSaQkWXlB3breLKI09p/IBm +X3lN7VIMPW8eLziSArGivKTpsW+cDau8Rbnn5FSMNiZojCp+bSPehOpZBmHb2OrS +2akgCBQYh2e+cadv5MmGPqta8iTDMDgMOrTzAstPKRfeHozyoOsRjeq1T9Yl65lk +XF+m0yl2r4w8VxemWzqaT53XHS+3tgCunDtB7pDXuM6zpNnWghxN1UJ8Ywle1nt4 +ecxxPwHfGAq9cAVCM7M0x4y+bxdnbw== +=8xDr -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8d9528559..6003afcf4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.0" +LE_AUTO_VERSION="0.29.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.0 \ - --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ - --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 -acme==0.29.0 \ - --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ - --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 -certbot-apache==0.29.0 \ - --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ - --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd -certbot-nginx==0.29.0 \ - --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ - --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 +certbot==0.29.1 \ + --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ + --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 +acme==0.29.1 \ + --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ + --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c +certbot-apache==0.29.1 \ + --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ + --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 +certbot-nginx==0.29.1 \ + --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ + --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index fc93a7804072c78caabc97212f6ba77f84fa52a3..7b3874da00e87d2dceb20372e6079f44fdf2c6a9 100644 GIT binary patch literal 256 zcmV+b0ssC&MV4ZBp~!>0zD9eP{pH!EBXbu^4*LFC6`15?<s2Pgj|8@y%FjNX}t56=4%&hte0AGS1xZSCP?+WUH-uVvaPBwiWlaCE|BAx52f)1j#5-4sDvGoyw*QT|osS z7(iG`@e3YWpZ$J379OHLQ8iQ~K{MyXiRXaLxRU_Uu`KVrPgJU?Ev1~>h3@rj0I6X?6}bYFPUfoD%JUw!xxqUi Gz0>!iiGPRy diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index d8be0dc96..cfe3f6fbe 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.29.0 \ - --hash=sha256:fd055f3c132f966bf72bca653a52cf5974395cea6ce4f3f727f304bd52a332e3 \ - --hash=sha256:3c9f18cc502ff3b3bc8811ee1e90b73021fcf3891e13003fbe3d499fe7384753 -acme==0.29.0 \ - --hash=sha256:d7e80231b17a8fd31d625ec4be964583b7aa08824fbfcd7e35b3185ecf342d28 \ - --hash=sha256:37a2f933ee07c0f12dcaea519e6549f6f68145788a1f5db594afaabb592a6b79 -certbot-apache==0.29.0 \ - --hash=sha256:0db527c8664a4e9cd3989e947f32dda499cc779ac0626ba6a12c1f96f9d1bbe3 \ - --hash=sha256:bd55e5d7ece13f30935d6454cc32ffcc50229a3ed39eb2a6724e5db2e53549cd -certbot-nginx==0.29.0 \ - --hash=sha256:c788ea49542959210b88c5dfb3d0e29cbf2a04bbf7f424533812ec0d47e1627b \ - --hash=sha256:675b8a9acc3945c539f6a0582d1cf9e2cd5bed183f0db262810724365812d5a5 +certbot==0.29.1 \ + --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ + --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 +acme==0.29.1 \ + --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ + --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c +certbot-apache==0.29.1 \ + --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ + --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 +certbot-nginx==0.29.1 \ + --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ + --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d From 5ff4fa91b064e9609bf2cfd5ee0d3e9dcc160980 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Dec 2018 16:31:08 -0800 Subject: [PATCH 471/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75939464..74bc0fd98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.30.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.29.1 - 2018-12-05 ### Added From 82cfff8d3519cf2ef06a091205f0c92ae783fc6e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Dec 2018 16:31:09 -0800 Subject: [PATCH 472/639] Bump version to 0.30.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- local-oldest-requirements.txt | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index f76150483..53a4eda27 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.29.1' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aa91dda7b..2562e6f3f 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index fc17a4ec2..405f8ff73 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 0b3b1b0eb..4f12e78f7 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6c5e460e3..92ed37f35 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index bc9847aae..0fd52c4b0 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 95c8e75b5..eec35b472 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 62725c098..21899f3d8 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index a6956b604..2bca63ccf 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d540b04fb..ee484e402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index aa56650f4..c91aabd4f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index ffc499dfc..9f0fe149d 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 6117bb361..4ebfeb3e1 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 329edee93..cbd77b901 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index a7dd6d1ce..2ea6699df 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 907755a77..84ca63f00 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 6498365b6..cebd48e41 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index ae0f63fe9..970089460 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.29.1' +version = '0.30.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 65144ba37..c96f7b28e 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.29.1' +__version__ = '0.30.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 6003afcf4..0ffe6c5f5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.1" +LE_AUTO_VERSION="0.30.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt index 2346300a3..302339918 100644 --- a/local-oldest-requirements.txt +++ b/local-oldest-requirements.txt @@ -1 +1 @@ --e acme[dev] +acme[dev]==0.29.1 From adedcc641644b05add3a785ddedeb771e94dd146 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 6 Dec 2018 18:33:46 +0100 Subject: [PATCH 473/639] Reduce to the minimal requirements to ensure Windows compatibility: execute tests only against lower and higher version of Python supported by Certbot on Windows. (#6565) --- appveyor.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 725ecfbff..ce2b5998c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,17 +1,9 @@ +image: Visual Studio 2015 + environment: matrix: - - FYI: Python 3.4 on Windows Server 2012 R2 - TOXENV: py34 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - - FYI: Python 3.4 on Windows Server 2016 - TOXENV: py34 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - FYI: Python 3.5 on Windows Server 2016 - TOXENV: py35 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - FYI: Python 3.7 on Windows Server 2016 + code coverage - TOXENV: py37-cover - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - TOXENV: py35 + - TOXENV: py37-cover branches: only: @@ -23,7 +15,6 @@ install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env - - "echo %APPVEYOR_BUILD_WORKER_IMAGE%" - "python --version" # Upgrade pip to avoid warnings - "python -m pip install --upgrade pip" From 353d09258575a5a8bfcfe7cb145834f787f2b7bb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 6 Dec 2018 16:02:16 -0800 Subject: [PATCH 474/639] Remove -q/--quiet from pip invocations. (#6568) While reducing noise in test output is valuable, this flag has made a couple aspects of Certbot's development difficult: 1. We test with different sets of dependencies and running pip in quiet mode removes all output about the packages being installed which has made reviewing changes to these tests more difficult. 2. When pip fails, it provides significantly less output about the failure in quiet mode than it does normally. The output is reduced so much that in the two times I've hit this issue in the last month, I was only able to see that installing package X failed rather than what the cause of that failure was which could be seen with `--quiet` removed. Also, since running pip without `--quiet` is the tox default, I expect Python developers to be familiar with what they see here. --- tools/install_and_test.py | 4 ++-- tools/pip_install.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 1eb81770c..79a7c2264 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -23,7 +23,7 @@ def call_with_print(command, cwd=None): def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-q', '-e'] + command = [sys.executable, '-m', 'pip', '-e'] else: script_dir = os.path.dirname(os.path.abspath(__file__)) command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] @@ -50,7 +50,7 @@ def main(args): shutil.copy2("pytest.ini", temp_cwd) try: call_with_print(' '.join([ - sys.executable, '-m', 'pytest', '--quiet', pkg.replace('-', '_')]), cwd=temp_cwd) + sys.executable, '-m', 'pytest', pkg.replace('-', '_')]), cwd=temp_cwd) finally: shutil.rmtree(temp_cwd) diff --git a/tools/pip_install.py b/tools/pip_install.py index 8878674c9..354dce32b 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -83,10 +83,10 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print('"{0}" -m pip install -q --constraint "{1}" --requirement "{2}"' + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' .format(sys.executable, all_constraints, requirements)) - call_with_print('"{0}" -m pip install -q --constraint "{1}" {2}' + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' .format(sys.executable, all_constraints, ' '.join(args))) finally: shutil.rmtree(working_dir) From 85f8f68263851b8f329cafdf8fee16b0f56e2d04 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 7 Dec 2018 14:18:28 -0800 Subject: [PATCH 475/639] Documentation fix-ups --- docs/challenges.rst | 18 ++++++++++++------ docs/contributing.rst | 4 ++-- docs/install.rst | 2 +- docs/using.rst | 14 +++++++------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/challenges.rst b/docs/challenges.rst index 034f79956..607d5c08b 100644 --- a/docs/challenges.rst +++ b/docs/challenges.rst @@ -3,9 +3,9 @@ Challenges To receive a certificate from Let's Encrypt certificate authority (CA), you must pass a *challenge* to prove you control each of the domain names that will be listed in the certificate. A challenge is one of -three tasks that only someone who controls the domain should be able to accomplish: +a list of specified tasks that only someone who controls the domain should be able to accomplish, such as: -* Posting a specified file in a specified location on a web site (the HTTP-01 challenge) +* Posting a specified file in a specified location on a web site (the HTTP-01 challenge) * Posting a specified DNS record in the domain name system (the DNS-01 challenge) It’s possible to complete each type of challenge *automatically* (Certbot directly makes the necessary @@ -15,15 +15,21 @@ design favors performing challenges automatically, and this is the normal case f Some plugins offer an *authenticator*, meaning that they can satisfy challenges: +* Apache plugin: (HTTP-01) Tries to edit your Apache configuration files to temporarily serve files to + satisfy challenges from the certificate authority. Use the Apache plugin when you're running Certbot on a + web server with Apache listening on port 80. +* nginx plugin: (HTTP-01) Tries to edit your nginx configuration files to temporarily serve files to + satisfy challenges from the certificate authority. Use the nginx plugin when you're running Certbot on a + web server with nginx listening on port 80. * Webroot plugin: (HTTP-01) Tries to place a file where it can be served over HTTP on port 80 by a web server running on your system. Use the Webroot plugin when you're running Certbot on a web server with any server application listening on port 80 serving files from a folder on disk in response. -* Standalone plugin: (HTTP-01) Tries to run a temporary web server listening on HTTP on - port 80 (for HTTP-01). Use the Standalone plugin if no existing program - is listening to these ports. Choose HTTP-01 using the `--preferred-challenges` option. +* Standalone plugin: (HTTP-01) Tries to run a temporary web server listening on HTTP on port 80. Use the + Standalone plugin if no existing program is listening to this port. * Manual plugin: (DNS-01 or HTTP-01) Either tells you what changes to make to your configuration or updates your DNS records using an external script (for DNS-01) or your webroot (for HTTP-01). Use the Manual - plugin if you have the technical knowledge to make configuration changes yourself when asked to do so. + plugin if you have the technical knowledge to make configuration changes yourself when asked to do so, + and are prepared to repeat these steps every time the certificate needs to be renewed. Tips for Challenges ------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index f05b0fa9a..229c63841 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -187,7 +187,7 @@ Authenticators Authenticators are plugins that prove control of a domain name by solving a challenge provided by the ACME server. ACME currently defines three types of -challenges: HTTP, TLS-SNI, and DNS, represented by classes in `acme.challenges`. +challenges: HTTP, TLS-SNI (deprecated), TLS-ALPR, and DNS, represented by classes in `acme.challenges`. An authenticator plugin should implement support for at least one challenge type. An Authenticator indicates which challenges it supports by implementing @@ -215,7 +215,7 @@ support for IIS, Icecast and Plesk. Installers and Authenticators will oftentimes be the same class/object (because for instance both tasks can be performed by a webserver like nginx) though this is not always the case (the standalone plugin is an authenticator -that listens on port 443, but it cannot install certs; a postfix plugin would +that listens on port 80, but it cannot install certs; a postfix plugin would be an installer but not an authenticator). Installers and Authenticators are kept separate because diff --git a/docs/install.rst b/docs/install.rst index fc6abad7a..35b262482 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -29,7 +29,7 @@ System Requirements Certbot currently requires Python 2.7 or 3.4+ running on a UNIX-like operating system. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to -bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and +bind to port 80 (if you use the ``standalone`` plugin) and to read and modify webserver configurations (if you use the ``apache`` or ``nginx`` plugins). If none of these apply to you, it is theoretically possible to run without root privileges, but for most users who want to avoid running an ACME diff --git a/docs/using.rst b/docs/using.rst index fd6e8e509..65c54cc8d 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -44,10 +44,13 @@ a combination_ of distinct authenticator and installer plugins. =========== ==== ==== =============================================================== ============================= Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= +apache_ Y Y | Automates obtaining and installing a certificate with Apache http-01_ (80) + | 2.4 on OSes with ``libaugeas0`` 1.0+. +nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. http-01_ (80) webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. -standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. http-01_ (80) - | Requires port 80 to be available. This is useful on +standalone_ Y N | Uses a "standalone" webserver to obtain a certificate. http-01_ (80) + | Requires port 80 to be available. This is useful on | systems with no webserver, or when direct integration with | the local webserver is not supported or not desired. |dns_plugs| Y N | This category of plugins automates obtaining a certificate by dns-01_ (53) @@ -156,10 +159,7 @@ To obtain a certificate using a "standalone" webserver, you can use the standalone plugin by including ``certonly`` and ``--standalone`` on the command line. This plugin needs to bind to port 80 in order to perform domain validation, so you may need to stop your -existing webserver. To control which port the plugin uses, include -one of the options shown below on the command line. - - * ``--preferred-challenges http`` to use port 80 +existing webserver. It must still be possible for your machine to accept inbound connections from the Internet on the specified port using each requested domain name. @@ -252,7 +252,7 @@ installer plugins. To do so, specify the authenticator plugin with For instance, you may want to create a certificate using the webroot_ plugin for authentication and the apache_ plugin for installation, perhaps because you -use a proxy or CDN for SSL and only want to secure the connection between them +use a proxy or CDN for HTTPS and only want to secure the connection between them and your origin server. :: From 9e1ee01547ff8d98382db7e8e5bd2b94f703caa8 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 10 Dec 2018 16:17:30 -0800 Subject: [PATCH 476/639] "Four shalt thou not count, neither count thou two" --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 229c63841..d95cbf4c0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -186,7 +186,7 @@ Authenticators -------------- Authenticators are plugins that prove control of a domain name by solving a -challenge provided by the ACME server. ACME currently defines three types of +challenge provided by the ACME server. ACME currently defines four types of challenges: HTTP, TLS-SNI (deprecated), TLS-ALPR, and DNS, represented by classes in `acme.challenges`. An authenticator plugin should implement support for at least one challenge type. From 5a8bea458037c59c7f09f343889a722198703225 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 10 Dec 2018 16:18:57 -0800 Subject: [PATCH 477/639] Consistent capitalization for list --- docs/challenges.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/challenges.rst b/docs/challenges.rst index 607d5c08b..ee8bb8e61 100644 --- a/docs/challenges.rst +++ b/docs/challenges.rst @@ -18,7 +18,7 @@ Some plugins offer an *authenticator*, meaning that they can satisfy challenges: * Apache plugin: (HTTP-01) Tries to edit your Apache configuration files to temporarily serve files to satisfy challenges from the certificate authority. Use the Apache plugin when you're running Certbot on a web server with Apache listening on port 80. -* nginx plugin: (HTTP-01) Tries to edit your nginx configuration files to temporarily serve files to +* Nginx plugin: (HTTP-01) Tries to edit your nginx configuration files to temporarily serve files to satisfy challenges from the certificate authority. Use the nginx plugin when you're running Certbot on a web server with nginx listening on port 80. * Webroot plugin: (HTTP-01) Tries to place a file where it can be served over HTTP on port 80 by a From 6c06a10d0abb152934621ce68e82d19ba5f48515 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 10 Dec 2018 16:28:28 -0800 Subject: [PATCH 478/639] Remove motivation for combining plugins, add 2nd example --- docs/using.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 65c54cc8d..090ba49cb 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -250,15 +250,20 @@ installer plugins. To do so, specify the authenticator plugin with ``--authenticator`` or ``-a`` and the installer plugin with ``--installer`` or ``-i``. -For instance, you may want to create a certificate using the webroot_ plugin -for authentication and the apache_ plugin for installation, perhaps because you -use a proxy or CDN for HTTPS and only want to secure the connection between them -and your origin server. +For instance, you could create a certificate using the webroot_ plugin +for authentication and the apache_ plugin for installation. :: certbot run -a webroot -i apache -w /var/www/html -d example.com +Or you could create a certificate using the manual_ plugin for authentication +and the nginx_ plugin for installation. (Note that this certificate cannot +be renewed automatically.) + +:: + certbot run -a manual -i nginx -d example.com + .. _third-party-plugins: Third-party plugins From 38ae7c8f9950740cadf605186700877be0dbeb1f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 10 Dec 2018 17:48:59 -0800 Subject: [PATCH 479/639] An unspecified number of challenges exist --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index d95cbf4c0..264db630f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -186,7 +186,7 @@ Authenticators -------------- Authenticators are plugins that prove control of a domain name by solving a -challenge provided by the ACME server. ACME currently defines four types of +challenge provided by the ACME server. ACME currently defines several types of challenges: HTTP, TLS-SNI (deprecated), TLS-ALPR, and DNS, represented by classes in `acme.challenges`. An authenticator plugin should implement support for at least one challenge type. From 64e570d63c89f36eba8419558bf7910bdfef4762 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 10 Dec 2018 17:49:24 -0800 Subject: [PATCH 480/639] Remove spurious comma --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 090ba49cb..5e8675418 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,7 +67,7 @@ manual_ Y N | Helps you obtain a certificate by giving you instruction .. |dns_plugs| replace:: :ref:`DNS plugins ` Under the hood, plugins use one of several ACME protocol challenges_ to -prove you control a domain. The options are http-01_ (which uses port 80), +prove you control a domain. The options are http-01_ (which uses port 80) and dns-01_ (requiring configuration of a DNS server on port 53, though that's often not the same machine as your webserver). A few plugins support more than one challenge type, in which case you can choose one From f137d55b31c022e9e574b1d48edc55938048ae21 Mon Sep 17 00:00:00 2001 From: dschlessman <36820111+dschlessman@users.noreply.github.com> Date: Tue, 11 Dec 2018 21:57:33 -0500 Subject: [PATCH 481/639] Issue 3816/revamp register subcommand (#6006) * address issue #3816 * formatting update * remove unused variable * address pylint trailing whitespace error * revert whitespace add * update boulder ci test for new update_registration verb * address code review comments * Issue 3816: Revert renaming '...update_regristration...' tests to '...update_account...'. Fix removing update_registration default argument value. * Issue 3816: Fix '--update-registration' not referring to 'update_registration' default as opposed to 'update_account'. * Issue 3816: delint tox output. * Issue 3816: Change @example.org domain to @domain.org in boulder test script * Issue 3816: Update CHANGELOG.md for Issue 3816 and remove extraneous space in main.py * Issue 3816: Remove extraneous default variable. --- CHANGELOG.md | 7 ++-- certbot/cli.py | 25 ++++++++----- certbot/main.py | 54 ++++++++++++++++++++-------- certbot/tests/main_test.py | 53 +++++++++++++++++++++++++-- tests/certbot-boulder-integration.sh | 6 ++++ 5 files changed, 116 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74bc0fd98..a0c6f0eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,14 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added -* +* Added the `update_account` subcommand for account management commands. ### Changed -* +* Copied account management functionality from the `register` subcommand + to the `update_account` subcommand. +* Marked usage `register --update-registration` for deprecation and + removal in a future release. ### Fixed diff --git a/certbot/cli.py b/certbot/cli.py index 8557b12c8..8e599cfda 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -101,6 +101,7 @@ manage certificates: manage your account with Let's Encrypt: register Create a Let's Encrypt ACME account + update_account Update a Let's Encrypt ACME account --agree-tos Agree to the ACME server's Subscriber Agreement -m EMAIL Email address for important account notifications """ @@ -397,9 +398,14 @@ VERB_HELP = [ }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", - "opts": "Options for account registration & modification", + "opts": "Options for account registration", "usage": "\n\n certbot register --email user@example.com [options]\n\n" }), + ("update_account", { + "short": "Update existing account with Let's Encrypt / other ACME server", + "opts": "Options for account modification", + "usage": "\n\n certbot update_account --email updated_email@example.com [options]\n\n" + }), ("unregister", { "short": "Irrevocably deactivate your account", "opts": "Options for account deactivation.", @@ -465,6 +471,7 @@ class HelpfulArgumentParser(object): "install": main.install, "plugins": main.plugins_cmd, "register": main.register, + "update_account": main.update_account, "unregister": main.unregister, "renew": main.renew, "revoke": main.revoke, @@ -993,21 +1000,21 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "certificates. Updates to the Subscriber Agreement will still " "affect you, and will be effective 14 days after posting an " "update to the web site.") + # TODO: When `certbot register --update-registration` is fully deprecated, + # delete following helpful.add helpful.add( "register", "--update-registration", action="store_true", - default=flag_default("update_registration"), - help="With the register verb, indicates that details associated " - "with an existing registration, such as the e-mail address, " - "should be updated, rather than registering a new account.") + default=flag_default("update_registration"), dest="update_registration", + help=argparse.SUPPRESS) helpful.add( - ["register", "unregister", "automation"], "-m", "--email", + ["register", "update_account", "unregister", "automation"], "-m", "--email", default=flag_default("email"), help=config_help("email")) - helpful.add(["register", "automation"], "--eff-email", action="store_true", + helpful.add(["register", "update_account", "automation"], "--eff-email", action="store_true", default=flag_default("eff_email"), dest="eff_email", help="Share your e-mail address with EFF") - helpful.add(["register", "automation"], "--no-eff-email", action="store_false", - default=flag_default("eff_email"), dest="eff_email", + helpful.add(["register", "update_account", "automation"], "--no-eff-email", + action="store_false", default=flag_default("eff_email"), dest="eff_email", help="Don't share your e-mail address with EFF") helpful.add( ["automation", "certonly", "run"], diff --git a/certbot/main.py b/certbot/main.py index 692dafeed..b24c5c622 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -654,7 +654,45 @@ def unregister(config, unused_plugins): def register(config, unused_plugins): - """Create or modify accounts on the server. + """Create accounts on the server. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` or a string indicating and error + :rtype: None or str + + """ + # TODO: When `certbot register --update-registration` is fully deprecated, + # delete the true case of if block + if config.update_registration: + msg = ("Usage 'certbot register --update-registration' is deprecated.\n" + "Please use 'cerbot update_account [options]' instead.\n") + logger.warning(msg) + return update_account(config, unused_plugins) + + # Portion of _determine_account logic to see whether accounts already + # exist or not. + account_storage = account.AccountFileStorage(config) + accounts = account_storage.find_all() + + if len(accounts) > 0: + # TODO: add a flag to register a duplicate account (this will + # also require extending _determine_account's behavior + # or else extracting the registration code from there) + return ("There is an existing account; registration of a " + "duplicate account with this command is currently " + "unsupported.") + # _determine_account will register an account + _determine_account(config) + return + + +def update_account(config, unused_plugins): + """Modify accounts on the server. :param config: Configuration object :type config: interfaces.IConfig @@ -673,20 +711,6 @@ def register(config, unused_plugins): reporter_util = zope.component.getUtility(interfaces.IReporter) add_msg = lambda m: reporter_util.add_message(m, reporter_util.MEDIUM_PRIORITY) - # registering a new account - if not config.update_registration: - if len(accounts) > 0: - # TODO: add a flag to register a duplicate account (this will - # also require extending _determine_account's behavior - # or else extracting the registration code from there) - return ("There is an existing account; registration of a " - "duplicate account with this command is currently " - "unsupported.") - # _determine_account will register an account - _determine_account(config) - return - - # --update-registration if len(accounts) == 0: return "Could not find an existing account to update." if config.email is None: diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index adfeff872..786b91a94 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1398,7 +1398,20 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met x = self._call_no_clientmock(["register", "--email", "user@example.org"]) self.assertTrue("There is an existing account" in x[0]) - def test_update_registration_no_existing_accounts(self): + def test_update_account_no_existing_accounts(self): + # with mock.patch('certbot.main.client') as mocked_client: + with mock.patch('certbot.main.account') as mocked_account: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = [] + x = self._call_no_clientmock( + ["update_account", "--email", + "user@example.org"]) + self.assertTrue("Could not find an existing account" in x[0]) + + # TODO: When `certbot register --update-registration` is fully deprecated, + # delete the following test + def test_update_registration_no_existing_accounts_deprecated(self): # with mock.patch('certbot.main.client') as mocked_client: with mock.patch('certbot.main.account') as mocked_account: mocked_storage = mock.MagicMock() @@ -1409,7 +1422,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met "user@example.org"]) self.assertTrue("Could not find an existing account" in x[0]) - def test_update_registration_unsafely(self): + # TODO: When `certbot register --update-registration` is fully deprecated, + # delete the following test + def test_update_registration_unsafely_deprecated(self): # This test will become obsolete when register --update-registration # supports removing an e-mail address from the account with mock.patch('certbot.main.account') as mocked_account: @@ -1423,7 +1438,39 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.display_ops.get_email') @test_util.patch_get_utility() - def test_update_registration_with_email(self, mock_utility, mock_email): + def test_update_account_with_email(self, mock_utility, mock_email): + email = "user@example.com" + mock_email.return_value = email + with mock.patch('certbot.eff.handle_subscription') as mock_handle: + with mock.patch('certbot.main._determine_account') as mocked_det: + with mock.patch('certbot.main.account') as mocked_account: + with mock.patch('certbot.main.client') as mocked_client: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = ["an account"] + mocked_det.return_value = (mock.MagicMock(), "foo") + cb_client = mock.MagicMock() + mocked_client.Client.return_value = cb_client + x = self._call_no_clientmock( + ["update_account"]) + # When registration change succeeds, the return value + # of register() is None + self.assertTrue(x[0] is None) + # and we got supposedly did update the registration from + # the server + self.assertTrue( + cb_client.acme.update_registration.called) + # and we saved the updated registration on disk + self.assertTrue(mocked_storage.save_regr.called) + self.assertTrue( + email in mock_utility().add_message.call_args[0][0]) + self.assertTrue(mock_handle.called) + + # TODO: When `certbot register --update-registration` is fully deprecated, + # delete the following test + @mock.patch('certbot.main.display_ops.get_email') + @test_util.patch_get_utility() + def test_update_registration_with_email_deprecated(self, mock_utility, mock_email): email = "user@example.com" mock_email.return_value = email with mock.patch('certbot.eff.handle_subscription') as mock_handle: diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 87e3cc319..9011d8ba3 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -207,10 +207,16 @@ common unregister common register --email ex1@domain.org,ex2@domain.org +# TODO: When `certbot register --update-registration` is fully deprecated, delete the two following deprecated uses + common register --update-registration --email ex1@domain.org common register --update-registration --email ex1@domain.org,ex2@domain.org +common update_account --email example@domain.org + +common update_account --email ex1@domain.org,ex2@domain.org + common plugins --init --prepare | grep webroot # We start a server listening on the port for the From 6b145a480ef19309c84bfecc61465571172a4f3f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 13 Dec 2018 20:13:39 +0100 Subject: [PATCH 482/639] Correct boulder integration tests using the latest challtestsrv version (#6600) --- tests/boulder-fetch.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 31e0f6b30..f34deb74e 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -11,7 +11,6 @@ if [ ! -d ${BOULDERPATH} ]; then fi cd ${BOULDERPATH} -sed -i "s/FAKE_DNS: .*/FAKE_DNS: 10.77.77.1/" docker-compose.yml docker-compose up -d boulder @@ -28,3 +27,6 @@ if ! curl http://localhost:4000/directory 2>/dev/null; then echo "timed out waiting for boulder to start" exit 1 fi + +# Setup the DNS resolution used by boulder instance to docker host +curl -X POST -d '{"ip":"10.77.77.1"}' http://localhost:8055/set-default-ipv4 From 346a4246398b72d27b74b537142ee05073e71cd0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 13 Dec 2018 15:54:38 -0800 Subject: [PATCH 483/639] Update pinned urllib3 (#6601) GitHub notified us about a security vulnerability in our pinned version of `urllib3` earlier this week. It doesn't affect us, but we might as well upgrade anyway. I checked: * There are no backwards incompatible features we care about listed at https://github.com/urllib3/urllib3/blob/master/CHANGES.rst. * urllib3's dependencies don't also need to be updated according to https://github.com/urllib3/urllib3/blob/1.24.1/setup.py. * The hashes match when obtained from different network vantage points. --- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/dependency-requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ffe6c5f5..9703b3506 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1212,9 +1212,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 983d2bb95..eb297bc6e 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -188,9 +188,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a From e3cb782e5992ba306de59ba96dfb6f125720cd06 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 17 Dec 2018 13:23:57 -0800 Subject: [PATCH 484/639] Allow josepy to be accessed through acme.jose. (#6592) When working on an update to our packages in Ubuntu Xenial, @NCommander noticed that importing code through acme.jose no longer works since josepy became a separate package and remembers having to fix up some code that was using acme.jose himself. This PR should fix that problem by making all of josepy accessible through acme.jose. This is primarily beneficial to our OS package maintainers who want to avoid subtle API changes when updating packages in stable repositories. They will likely backport this change, but I figure we might as well add it ourselves to minimize divergences in our OS packages in the future and avoid problems on the off chance someone hasn't upgraded acme and was relying on this feature. --- CHANGELOG.md | 8 ++++++-- acme/acme/__init__.py | 15 +++++++++++++++ acme/acme/jose_test.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 acme/acme/jose_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c6f0eb2..c6fac76fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,13 +17,17 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* +* Older modules in the josepy library can now be accessed through acme.jose + like it could in previous versions of acme. This is only done to preserve + backwards compatibility and support for doing this with new modules in josepy + will not be added. Users of the acme library should switch to using josepy + directly if they haven't done so already. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme More details about these changes can be found on our GitHub repo. diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index e8a0b16a8..d2cb1ee06 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,3 +10,18 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ +import sys + +import josepy + +# This code exists to keep backwards compatibility with people using acme.jose +# before it became the standalone josepy package. +# +# It is based on +# https://github.com/requests/requests/blob/1278ecdf71a312dc2268f3bfc0aabfab3c006dcf/requests/packages.py + +for mod in list(sys.modules): + # This traversal is apparently necessary such that the identities are + # preserved (acme.jose.* is josepy.*) + if mod == 'josepy' or mod.startswith('josepy.'): + sys.modules['acme.' + mod.replace('josepy', 'jose', 1)] = sys.modules[mod] diff --git a/acme/acme/jose_test.py b/acme/acme/jose_test.py new file mode 100644 index 000000000..ecd483662 --- /dev/null +++ b/acme/acme/jose_test.py @@ -0,0 +1,43 @@ +"""Tests for acme.jose shim.""" +import importlib +import unittest + +class JoseTest(unittest.TestCase): + """Tests for acme.jose shim.""" + + def _test_it(self, submodule, attribute): + if submodule: + acme_jose_path = 'acme.jose.' + submodule + josepy_path = 'josepy.' + submodule + else: + acme_jose_path = 'acme.jose' + josepy_path = 'josepy' + acme_jose = importlib.import_module(acme_jose_path) + josepy = importlib.import_module(josepy_path) + + self.assertIs(acme_jose, josepy) + self.assertIs(getattr(acme_jose, attribute), getattr(josepy, attribute)) + + def test_top_level(self): + self._test_it('', 'RS512') + + def test_submodules(self): + # This test ensures that the modules in josepy that were + # available at the time it was moved into its own package are + # available under acme.jose. Backwards compatibility with new + # modules or testing code is not maintained. + mods_and_attrs = [('b64', 'b64decode',), + ('errors', 'Error',), + ('interfaces', 'JSONDeSerializable',), + ('json_util', 'Field',), + ('jwa', 'HS256',), + ('jwk', 'JWK',), + ('jws', 'JWS',), + ('util', 'ImmutableMap',),] + + for mod, attr in mods_and_attrs: + self._test_it(mod, attr) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover From 62cb67ec670f70dc22a7c7ad2e738c8235164b33 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 17 Dec 2018 15:38:28 -0800 Subject: [PATCH 485/639] default to any py3 test (#6609) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 61cee8eff..b672286ac 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},py27-cover,lint +envlist = modification,py3,py27-cover,lint [base] # pip installs the requested packages in editable mode From f90561012241171ed8e0dd9996c703c384357eba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 17 Dec 2018 16:22:03 -0800 Subject: [PATCH 486/639] add mypy to envlist (#6610) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b672286ac..021c23949 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3,py27-cover,lint +envlist = modification,py3,py27-cover,lint,mypy [base] # pip installs the requested packages in editable mode From 856bfe354427203136161e037fad92cdb1fa03b3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 19 Dec 2018 01:17:54 +0100 Subject: [PATCH 487/639] Tag to disable random sleep upon a renew task (#6599) * Extraction from #6541 to add flag to disable shuffle sleep on renew action * Move the logic of random sleep to execute it only if there is effectively a certificate to renew. * Add comments * Correct lint * Suspend lint rule * Revert code cleaning * Hide the flag * Ignore lint * Update cli.py --- certbot/cli.py | 4 ++++ certbot/constants.py | 1 + certbot/main.py | 12 ------------ certbot/renewal.py | 22 +++++++++++++++++++++- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 8e599cfda..ff1827cb1 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1215,6 +1215,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis " one will be run.") helpful.add("renew", "--renew-hook", action=_RenewHookAction, help=argparse.SUPPRESS) + helpful.add( + "renew", "--no-random-sleep-on-renew", action="store_false", + default=flag_default("random_sleep_on_renew"), dest="random_sleep_on_renew", + help=argparse.SUPPRESS) helpful.add( "renew", "--deploy-hook", action=_DeployHookAction, help='Command to be run in a shell once for each successfully' diff --git a/certbot/constants.py b/certbot/constants.py index eb4105f82..7d0e0c879 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -68,6 +68,7 @@ CLI_DEFAULTS = dict( directory_hooks=True, reuse_key=False, disable_renew_updates=False, + random_sleep_on_renew=True, eab_hmac_key=None, eab_kid=None, diff --git a/certbot/main.py b/certbot/main.py index b24c5c622..ac639bc80 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,9 +4,7 @@ from __future__ import print_function import functools import logging.handlers import os -import random import sys -import time import configobj import josepy as jose @@ -1269,16 +1267,6 @@ def renew(config, unused_plugins): :rtype: None """ - if not sys.stdin.isatty(): - # Noninteractive renewals include a random delay in order to spread - # out the load on the certificate authority servers, even if many - # users all pick the same time for renewals. This delay precedes - # running any hooks, so that side effects of the hooks (such as - # shutting down a web service) aren't prolonged unnecessarily. - sleep_time = random.randint(1, 60*8) - logger.info("Non-interactive renewal: random delay of %s seconds", sleep_time) - time.sleep(sleep_time) - try: renewal.handle_renewal_request(config) finally: diff --git a/certbot/renewal.py b/certbot/renewal.py index 5b2e00740..4c592b27f 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -5,6 +5,9 @@ import itertools import logging import os import traceback +import sys +import time +import random import six import zope.component @@ -372,7 +375,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, disp.notification("\n".join(out), wrap=False) -def handle_renewal_request(config): +def handle_renewal_request(config): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Examine each lineage; renew if due and report results""" # This is trivially False if config.domains is empty @@ -396,6 +399,14 @@ def handle_renewal_request(config): renew_failures = [] renew_skipped = [] parse_failures = [] + + # Noninteractive renewals include a random delay in order to spread + # out the load on the certificate authority servers, even if many + # users all pick the same time for renewals. This delay precedes + # running any hooks, so that side effects of the hooks (such as + # shutting down a web service) aren't prolonged unnecessarily. + apply_random_sleep = not sys.stdin.isatty() and config.random_sleep_on_renew + for renewal_file in conf_files: disp = zope.component.getUtility(interfaces.IDisplay) disp.notification("Processing " + renewal_file, pause=False) @@ -424,6 +435,15 @@ def handle_renewal_request(config): from certbot import main plugins = plugins_disco.PluginsRegistry.find_all() if should_renew(lineage_config, renewal_candidate): + # Apply random sleep upon first renewal if needed + if apply_random_sleep: + sleep_time = random.randint(1, 60 * 8) + logger.info("Non-interactive renewal: random delay of %s seconds", + sleep_time) + time.sleep(sleep_time) + # We will sleep only once this day, folks. + apply_random_sleep = False + # domains have been restored into lineage_config by reconstitute # but they're unnecessary anyway because renew_cert here # will just grab them from the certificate From 1cdcc15e64ba8b4f01ca96cc170e26b7d1cac183 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 2 Jan 2019 10:08:08 -0800 Subject: [PATCH 488/639] Pin all dependency installation in the release script (#6584) (#6602) Fixes #6584. * Pin all dependency installation in the release script * also use pip_install.py to install pytest --- tools/_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index 90c176b0e..ec2deda22 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -143,7 +143,7 @@ pip install -U pip # (or our dependencies) have conditional dependencies implemented with if # statements in setup.py and we have cached wheels lying around that would # cause those ifs to not be evaluated. -pip install \ +python ../tools/pip_install.py \ --no-cache-dir \ --extra-index-url http://localhost:$PORT \ $SUBPKGS @@ -166,7 +166,7 @@ fi mkdir kgs kgs="kgs/$version" pip freeze | tee $kgs -pip install pytest +python ../tools/pip_install.py pytest for module in $subpkgs_modules ; do echo testing $module # use an empty configuration file rather than the one in the repo root From ba358f9d07f76ae5f5540e1c8f3d6e622731a6ca Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 2 Jan 2019 12:01:39 -0800 Subject: [PATCH 489/639] Update changelog for 0.30.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6fac76fe..182f6db32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.30.0 - master +## 0.30.0 - 2019-01-02 ### Added From 3971573d7a9426fdd7394e11c5dc41b3b8624134 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 2 Jan 2019 12:33:19 -0800 Subject: [PATCH 490/639] Release 0.30.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 32 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 13 ++++--- letsencrypt-auto | 32 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++----- letsencrypt-auto-source/letsencrypt-auto | 26 +++++++------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++------- 26 files changed, 90 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 53a4eda27..7ddcf0f06 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.0.dev0' +version = '0.30.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 2562e6f3f..42a88b7b4 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 6003afcf4..be2c3679b 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.1" +LE_AUTO_VERSION="0.30.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1212,9 +1212,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 405f8ff73..e64c7bba1 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4f12e78f7..47888bec1 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 92ed37f35..66420e354 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0fd52c4b0..0260d5011 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index eec35b472..5f5e939c2 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 21899f3d8..5d5874459 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2bca63ccf..5ba4ece27 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ee484e402..bc04d3ff1 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index c91aabd4f..4db08c575 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 9f0fe149d..f07b08024 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4ebfeb3e1..0467da1f0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index cbd77b901..3959e0b84 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 2ea6699df..811d8aec1 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 84ca63f00..5a99c4ffa 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index cebd48e41..8225e47de 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 970089460..c19fd862d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.30.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index c96f7b28e..768efc2b5 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.0.dev0' +__version__ = '0.30.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 37d67cfe0..bad6275e7 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -29,6 +29,7 @@ manage certificates: manage your account with Let's Encrypt: register Create a Let's Encrypt ACME account + update_account Update a Let's Encrypt ACME account --agree-tos Agree to the ACME server's Subscriber Agreement -m EMAIL Email address for important account notifications @@ -112,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.29.1 + "". (default: CertbotACMEClient/0.30.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -359,7 +360,7 @@ revoke: certificates. (default: None) register: - Options for account registration & modification + Options for account registration --register-unsafely-without-email Specifying this flag enables registering an account @@ -371,11 +372,6 @@ register: to the Subscriber Agreement will still affect you, and will be effective 14 days after posting an update to the web site. (default: False) - --update-registration - With the register verb, indicates that details - associated with an existing registration, such as the - e-mail address, should be updated, rather than - registering a new account. (default: False) -m EMAIL, --email EMAIL Email used for registration and recovery contact. Use comma to register multiple emails, ex: @@ -384,6 +380,9 @@ register: --no-eff-email Don't share your e-mail address with EFF (default: None) +update_account: + Options for account modification + unregister: Options for account deactivation. diff --git a/letsencrypt-auto b/letsencrypt-auto index 6003afcf4..be2c3679b 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.1" +LE_AUTO_VERSION="0.30.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1212,9 +1212,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index b4e520fb2..4e2a700e5 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwIbaAACgkQTRfJlc2X -dfLIGQf+JZr3oP89qyMREYAL3/Bx+vxsx+c01IuDaG1pBUeVwL5rdeU1kDZ1WkKb -61nCoMPbSLqLMor2IpobFj44lEFJS1WYrtfe8sgMLeSaQkWXlB3breLKI09p/IBm -X3lN7VIMPW8eLziSArGivKTpsW+cDau8Rbnn5FSMNiZojCp+bSPehOpZBmHb2OrS -2akgCBQYh2e+cadv5MmGPqta8iTDMDgMOrTzAstPKRfeHozyoOsRjeq1T9Yl65lk -XF+m0yl2r4w8VxemWzqaT53XHS+3tgCunDtB7pDXuM6zpNnWghxN1UJ8Ywle1nt4 -ecxxPwHfGAq9cAVCM7M0x4y+bxdnbw== -=8xDr +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwtH9cACgkQTRfJlc2X +dfIqUwf/RXLZAeFF/59PjTAzcV+eEISlvEmFcV0zL3vv23PsY3S5Iuuwcd6rTm5M +UWNtmUTmFVo0xmxAj6Eqfpnt0P+JPpPcnbLNIGKFekBWIshgH84RRFWPJjNh/hu1 +pyzkkcWaOB86egdVfjvuRJ0j7AGd0ih6ur2rlgfHVjTYR+0EdWszFDEFBlq8cpct +9d1gCgH7VWKSIQMhzGLMsmdMxNoDl4hiqVPU0FP5/mn2xGF7FgeKNW3+NiTouKuB +mZOeEl3f3uOze/suHPyfOu+49jk+TWWE05Xfqfowjf486nKPg6/uSA2izW/MwIKN +HuIuY3bBf+lx5yUVIraoZhH2MxODDQ== +=BZqz -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9703b3506..be2c3679b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0.dev0" +LE_AUTO_VERSION="0.30.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7b3874da00e87d2dceb20372e6079f44fdf2c6a9..e4902c0ee7241e37491b8a941e25744b837cb7f8 100644 GIT binary patch literal 256 zcmV+b0ssDv4s4Tx!Gpv36M!K{^ zade@s@$}@HWNXZloBGk`1V%UIbVc*QUSk|U~K*`EpZhpmOER`)Vh~vtkxoSl**~O#gMxciG8{xI~ zBjQkiW9*%R^8`&dakxvsB>SH8_YUxTWg^fH-(kERXdaGN%kJjgKStmdpmEs%4pGXQ zEz^kZX>73#HrgfHH^KzS6(C0T%A&Lv6bMbMN>!BFV}3U0NOGuT;#))Lwb_f~H7)4W GuwXBg?}bPJ literal 256 zcmV+b0ssC&MV4ZBp~!>0zD9eP{pH!EBXbu^4*LFC6`15?<s2Pgj|8@y%FjNX}t56=4%&hte0AGS1xZSCP?+WUH-uVvaPBwiWl Date: Wed, 2 Jan 2019 12:33:31 -0800 Subject: [PATCH 491/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 182f6db32..73e6d13b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.31.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.30.0 - 2019-01-02 ### Added From c25e6a8adf41d8a8551fb153aca16143c5dbc495 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 2 Jan 2019 12:33:31 -0800 Subject: [PATCH 492/639] Bump version to 0.31.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7ddcf0f06..77ff7bae8 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 42a88b7b4..14d6cacb6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e64c7bba1..f519ed422 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 47888bec1..ff33293fe 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 66420e354..5c8709445 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0260d5011..2f7fa37d6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 5f5e939c2..3bb43cf5d 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 5d5874459..599cec486 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 5ba4ece27..894d809ac 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index bc04d3ff1..c99ad38aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 4db08c575..588b6a40a 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index f07b08024..b09a09762 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 0467da1f0..e48428191 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 3959e0b84..8b6d73d33 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 811d8aec1..edf7b6ba6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 5a99c4ffa..69c2c7ed3 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8225e47de..05843d2ed 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index c19fd862d..70e11f62b 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 768efc2b5..bf68034c8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.0' +__version__ = '0.31.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index be2c3679b..c08a08160 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0" +LE_AUTO_VERSION="0.31.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From bb0f3566109e6d1265454619f70e22a500b03f16 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 2 Jan 2019 16:15:25 -0800 Subject: [PATCH 493/639] Tell people to update changed package list. (#6633) --- pull_request_template.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pull_request_template.md b/pull_request_template.md index c071d4135..60fd6da7e 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1 +1,3 @@ -Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. +Be sure to edit the `master` section of `CHANGELOG.md`. This includes a +description of the change and ensuring the modified package(s) are listed as +having been changed. From 3cb6d6c25bb57410e6a66b9b92a7f0848a826cd1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 3 Jan 2019 11:26:15 -0800 Subject: [PATCH 494/639] Don't sleep in integration tests (#6636) Fixes #6635. * Don't sleep in integration tests. * add backslash --- tests/integration/_common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 1e444fa26..8a1c07e3c 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -43,5 +43,6 @@ certbot_test_no_force_renew () { --register-unsafely-without-email \ --debug \ -vv \ + --no-random-sleep-on-renew \ "$@" } From ec297ccf72e95961586ec2382c3e3225ce578aa4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 3 Jan 2019 12:46:25 -0800 Subject: [PATCH 495/639] Add missing acme.jose attribute. (#6637) Fixes the problem at https://github.com/certbot/certbot/pull/6592#discussion_r245106383. The tests use `eval` which neither myself or `pylint` like very much. I started to change this by splitting the path we wanted to test and repeatedly calling `getattr`, but it didn't seem worth the effort to me. * Add missing acme.jose attribute. * update changelog --- CHANGELOG.md | 5 +++-- acme/acme/__init__.py | 4 ++-- acme/acme/jose_test.py | 18 ++++++++++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73e6d13b5..501070baf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,14 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed -* +* Fixed accessing josepy contents through acme.jose when the full acme.jose + path is used. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme More details about these changes can be found on our GitHub repo. diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index d2cb1ee06..bab5b40ce 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -12,14 +12,14 @@ supported version: `draft-ietf-acme-01`_. """ import sys -import josepy - # This code exists to keep backwards compatibility with people using acme.jose # before it became the standalone josepy package. # # It is based on # https://github.com/requests/requests/blob/1278ecdf71a312dc2268f3bfc0aabfab3c006dcf/requests/packages.py +import josepy as jose + for mod in list(sys.modules): # This traversal is apparently necessary such that the identities are # preserved (acme.jose.* is josepy.*) diff --git a/acme/acme/jose_test.py b/acme/acme/jose_test.py index ecd483662..340624a4f 100644 --- a/acme/acme/jose_test.py +++ b/acme/acme/jose_test.py @@ -12,11 +12,21 @@ class JoseTest(unittest.TestCase): else: acme_jose_path = 'acme.jose' josepy_path = 'josepy' - acme_jose = importlib.import_module(acme_jose_path) - josepy = importlib.import_module(josepy_path) + acme_jose_mod = importlib.import_module(acme_jose_path) + josepy_mod = importlib.import_module(josepy_path) - self.assertIs(acme_jose, josepy) - self.assertIs(getattr(acme_jose, attribute), getattr(josepy, attribute)) + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) + + # We use the imports below with eval, but pylint doesn't + # understand that. + # pylint: disable=eval-used,unused-variable + import acme + import josepy + acme_jose_mod = eval(acme_jose_path) + josepy_mod = eval(josepy_path) + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) def test_top_level(self): self._test_it('', 'RS512') From 59bbda51abf641483b4f760bb732f6e0646d0d39 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 3 Jan 2019 17:48:09 -0800 Subject: [PATCH 496/639] Compatibility with more traditional versions of awk --- tests/certbot-boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 9011d8ba3..630571148 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -174,7 +174,7 @@ CheckRenewHook() { TotalAndDistinctLines() { total=$1 distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' + awk '{a[$1] = 1}; END {n = 0; for (i in a) { n++ }; exit(NR !='$total' || n !='$distinct')}' } # Cleanup coverage data From fd023cc3ea13c41e4f0612029ae2d657891d7675 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Jan 2019 09:27:45 -0800 Subject: [PATCH 497/639] Use HTTPS link to semver. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 501070baf..724820356 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Certbot change log -Certbot adheres to [Semantic Versioning](http://semver.org/). +Certbot adheres to [Semantic Versioning](https://semver.org/). ## 0.31.0 - master From 233cfe52075d96fa928099c402c5d1e8e3c28ddd Mon Sep 17 00:00:00 2001 From: Trinopoty Biswas Date: Sat, 5 Jan 2019 00:44:12 +0530 Subject: [PATCH 498/639] Added sorting to renewal_conf_files() --- certbot/storage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/storage.py b/certbot/storage.py index eb17e1d38..7472df975 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -41,7 +41,9 @@ def renewal_conf_files(config): :rtype: `list` of `str` """ - return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result = glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result.sort() + return result def renewal_file_for_certname(config, certname): """Return /path/to/certname.conf in the renewal conf directory""" From 33090ab77a51a15e2aff46ee925ab7cb16a2b20b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 4 Jan 2019 12:44:31 -0800 Subject: [PATCH 499/639] Fix oldest nginx integration tests (#6642) #6636 broke [test-everything tests](https://travis-ci.org/certbot/certbot/builds/475173804) because `_common.sh` is a common file shared between Certbot and Nginx integration tests and `--no-random-sleep-on-renew` isn't defined for the version of Certbot used in the "oldest" integration tests. This PR adds code to `_common.sh` to check the Certbot version and if it's new enough, add `--no-random-sleep-on-renew` to the command line. I repurposed `$store_flags` and stopped exporting it because it's not used anywhere outside of this file. Other approaches I considered and decided against were: 1. Adding this flag in `certbot-boulder-integration.sh`. I decided against it because it's setting us up for the same problem in the future if the oldest version of Certbot is upgraded in the Nginx tests and we call `certbot renew`. 2. Just upgrading the oldest version of Certbot required by Nginx to avoid these issues. While this would work (with perhaps some unnecessary burden for our packagers), I think it's avoiding the real problem here which should now be able to addressed easily with the addition of `$other_flags` and `version_at_least`. * Add version_at_least(). * Conditionally disable sleep. * Consolidate store_flags and other_flags. * update comments --- tests/integration/_common.sh | 38 ++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 8a1c07e3c..83aa91a9e 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,12 +3,15 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -store_flags="--config-dir $config_dir --work-dir $root/work" -store_flags="$store_flags --logs-dir $root/logs" tls_sni_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir store_flags tls_sni_01_port http_01_port sources +export root config_dir tls_sni_01_port http_01_port sources +certbot_path="$(command -v certbot)" +# Flags that are added here will be added to Certbot calls within +# certbot_test_no_force_renew. +other_flags="--config-dir $config_dir --work-dir $root/work" +other_flags="$other_flags --logs-dir $root/logs" certbot_test () { certbot_test_no_force_renew \ @@ -16,11 +19,35 @@ certbot_test () { "$@" } +# Succeeds if Certbot version is at least the given version number and fails +# otherwise. This is useful for making sure Certbot has certain features +# available. The patch version is currently ignored. +# +# Arguments: +# First argument is the minimum major version +# Second argument is the minimum minor version +version_at_least () { + # Certbot major and minor version (e.g. 0.30) + major_minor=$("$certbot_path" --version 2>&1 | cut -d' ' -f2 | cut -d. -f1,2) + major=$(echo "$major_minor" | cut -d. -f1) + minor=$(echo "$major_minor" | cut -d. -f2) + # Test that either the major version is greater or major version is equal + # and minor version is greater than or equal to. + [ \( "$major" -gt "$1" \) -o \( "$major" -eq "$1" -a "$minor" -ge "$2" \) ] +} + # Use local ACMEv2 endpoint if requested and SERVER isn't already set. if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then SERVER="http://localhost:4001/directory" fi +# --no-random-sleep-on-renew was added in +# https://github.com/certbot/certbot/pull/6599 and first released in Certbot +# 0.30.0. +if version_at_least 0 30; then + other_flags="$other_flags --no-random-sleep-on-renew" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" @@ -30,19 +57,18 @@ certbot_test_no_force_renew () { --append \ --source $sources \ --omit $omit_patterns \ - $(command -v certbot) \ + "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --tls-sni-01-port $tls_sni_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ - $store_flags \ + $other_flags \ --non-interactive \ --no-redirect \ --agree-tos \ --register-unsafely-without-email \ --debug \ -vv \ - --no-random-sleep-on-renew \ "$@" } From efaaf48f909923562c127398d317ca1065327bbe Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 7 Jan 2019 21:00:01 +0100 Subject: [PATCH 500/639] [Windows|Unix] Use double quote compatible both for windows and linux (#6553) File _venv_common.py uses single quotes to ask pip to install setuptools>=30.3. Using single quotes to enclose a string is not supported on Windows shell (Batch). This PR replaces theses single quotes by double quotes supported both on Windows and Linux. --- tools/_venv_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index ecd438f94..540842773 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -156,7 +156,7 @@ def main(venv_name, venv_args, args): new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), env=new_environ, shell=True) - subprocess_with_print("python -m pip install --upgrade 'setuptools>=30.3'", + subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', env=new_environ, shell=True) subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), env=new_environ, shell=True) From 5130e9ba1e958225d951a5ce0e0eb714be317cd1 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 9 Jan 2019 04:44:25 +0100 Subject: [PATCH 501/639] Correct the oldest requirements environment (#6569) I observed that the current set of oldest requirements do not correspond to any environment, except the specific Xenial image in Travis CI (and standard Xenial containers will also fail). It is because the requirements make cryptography and requests fail against standard libraries available in the typical Linux distributions that are targeted by the oldest requirements approach (Centos 6, Centos 7, Xenial, Jessie). This PR fixes that, by aligning the minimal version requirements of cryptography and requests to the maximal versions that are available on Centos 6. Centos 7, Jessie and Xenial stay unusable with oldest requirements for other reasons, but at least one old and supported Linux distribution is able to run the tests with oldest requirements out of the box. A test is also corrected to match the expected error message that old versions of urllib3 will raise. --- acme/acme/client_test.py | 6 +++--- acme/setup.py | 6 +++--- setup.py | 2 +- tools/oldest_constraints.txt | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 33ae3886b..b3d0f1921 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -707,6 +707,7 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) + class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" @@ -950,7 +951,6 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual(jws.signature.combined.kid, u'acct-uri') self.assertEqual(jws.signature.combined.url, u'url') - def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -1113,8 +1113,8 @@ class ClientNetworkTest(unittest.TestCase): # Requests Library Exceptions except requests.exceptions.ConnectionError as z: #pragma: no cover - self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))" - == str(z) or "[WinError 10061]" in str(z)) + self.assertTrue("'Connection aborted.'" in str(z) or "[WinError 10061]" in str(z)) + class ClientNetworkWithMockedResponseTest(unittest.TestCase): """Tests for acme.client.ClientNetwork which mock out response.""" diff --git a/acme/setup.py b/acme/setup.py index 77ff7bae8..eac3974fa 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,15 +9,15 @@ version = '0.31.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8', + 'cryptography>=1.2.3', # formerly known as acme.jose: 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', - 'PyOpenSSL>=0.13', + 'PyOpenSSL>=0.13.1', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests[security]>=2.6.0', # security extras added in 2.4.1 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible diff --git a/setup.py b/setup.py index 2a9b2c203..9e6af2d4f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', 'configobj', - 'cryptography>=1.2', # load_pem_x509_certificate + 'cryptography>=1.2.3', # load_pem_x509_certificate 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 80fce8b33..20fa0672a 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -37,13 +37,13 @@ pytz==2012rc0 # Our setup.py constraints cloudflare==1.5.1 -cryptography==1.2.0 +cryptography==1.2.3 google-api-python-client==1.5 oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.4.1 +requests[security]==2.6.0 # Ubuntu Xenial constraints ConfigArgParse==0.10.0 From b52cbc0fb706d2837b01b90c4bfb38ad160da2ad Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 9 Jan 2019 05:45:16 +0100 Subject: [PATCH 502/639] Reduce build log verbosity on Travis (#6597) PR #6568 removed the --quiet option in pip invocations, because this option deletes a lot of extremely useful logs when something goes wrong. However, when everything goes right, or at least when pip install is correctly executed, theses logs add hundreds of lines that are only noise, making hard to debug errors that can be in only one or two lines. We can have best of both worlds. Travis allows to fold large blocks of logs, that can be expanded directly from the UI if needed. It only requires to print in the console some specific code, that this PR implements in the pip_install.py script when the build is run in Travis (known by the existence of TRAVIS environment variable). I also take the occasion to clean up a little tox.ini. Note that AppVeyor does not have this fold capability, but it can be emulated using a proper capture of stdout/stderr delivered only when an error is detected. * Fold pip install log on travis * Global test env * Export env variable --- .travis.yml | 1 + appveyor.yml | 1 + tools/pip_install.py | 7 +++++++ tox.ini | 7 ------- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b8eafc13..89885d08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ before_install: before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' + - export TOX_TESTENV_PASSENV=TRAVIS matrix: include: diff --git a/appveyor.yml b/appveyor.yml index ce2b5998c..2b6b82747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,7 @@ install: build: off test_script: + - set TOX_TESTENV_PASSENV=APPVEYOR # Test env is set by TOXENV env variable - tox diff --git a/tools/pip_install.py b/tools/pip_install.py index 354dce32b..4466729e0 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -71,6 +71,11 @@ def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + if os.environ.get('TRAVIS'): + # When this script is executed on Travis, the following print will make the log + # be folded until the end command is printed (see finally section). + print('travis_fold:start:install_certbot_deps') + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -89,6 +94,8 @@ def main(args): call_with_print('"{0}" -m pip install --constraint "{1}" {2}' .format(sys.executable, all_constraints, ' '.join(args))) finally: + if os.environ.get('TRAVIS'): + print('travis_fold:end:install_certbot_deps') shutil.rmtree(working_dir) diff --git a/tox.ini b/tox.ini index 021c23949..71491c34a 100644 --- a/tox.ini +++ b/tox.ini @@ -64,9 +64,6 @@ source_paths = tests/lock_test.py [testenv] -passenv = - TRAVIS - APPVEYOR commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -176,7 +173,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS [testenv:nginx_compat] commands = @@ -187,7 +183,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS [testenv:le_auto_precise] # At the moment, this tests under Python 2.7 only, as only that version is @@ -199,7 +194,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -212,7 +206,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS TRAVIS_BRANCH [testenv:le_auto_wheezy] From 95557fa9b49fa2615774ed6ef734817dcd11b819 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Jan 2019 12:37:45 -0800 Subject: [PATCH 503/639] Stop using staging in apacheconftests (#6647) Fixes #6585. I wrote up three suggestions for fixing this at https://github.com/certbot/certbot/issues/6585#issuecomment-448054502. I took the middle approach of requiring the user to provide an ACME server to use. I like this better than the other approaches which were: > Resolve #5938 instead of this issue. There is value in these tests as is over the compatibility tests in that they don't use Docker and run on different OSes. > Spin up a local Python server to return the directory object. Trying to set up a dummy ACME server seemed hacky and error prone. Other notes about this PR are: * I put the Pebble setup in `tox.ini` rather than `.travis.yml` as this seems much cleaner and more natural. * I created a new `tox` environment called `apacheconftest-with-pebble` that reuses the code from `testenv:apacheconftest` so `apacheconftest` can continue to be used with servers other than Pebble like is done in our test farm tests. * I chose the environment variable `SERVER` for consistency with our integration tests. I chose to not give this environment variable a default but to fail fast when it is not set. * I ran test farm tests on this PR and they passed. --- .travis.yml | 3 ++- .../tests/apache-conf-files/apache-conf-test | 10 +++++++++- tests/letstest/scripts/test_apache2.sh | 1 + tox.ini | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89885d08e..a35d43b63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,8 +58,9 @@ matrix: before_install: addons: - python: "2.7" - env: TOXENV=apacheconftest + env: TOXENV=apacheconftest-with-pebble sudo: required + services: docker - python: "2.7" env: TOXENV=nginxroundtrip diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test index dcbba9d3e..4838a6eee 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test @@ -3,6 +3,11 @@ # A hackish script to see if the client is behaving as expected # with each of the "passing" conf files. +if [ -z "$SERVER" ]; then + echo "Please set SERVER to the ACME server's directory URL." + exit 1 +fi + export EA=/etc/apache2/ TESTDIR="`dirname $0`" cd $TESTDIR/passing @@ -56,13 +61,16 @@ if [ "$1" = --debian-modules ] ; then done fi +CERTBOT_CMD="sudo $(command -v certbot) --server $SERVER -vvvv" +CERTBOT_CMD="$CERTBOT_CMD --debug --apache --register-unsafely-without-email" +CERTBOT_CMD="$CERTBOT_CMD --agree-tos certonly -t --no-verify-ssl" FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup - RESULT=`echo c | sudo $(command -v certbot) -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | $CERTBOT_CMD 2>&1` if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then echo passed else diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4036e6efa..d24de2458 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -54,6 +54,7 @@ if [ $? -ne 0 ] ; then fi if [ "$OS_TYPE" = "ubuntu" ] ; then + export SERVER="$BOULDER_URL" venv/bin/tox -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE diff --git a/tox.ini b/tox.ini index 71491c34a..95b2f2c64 100644 --- a/tox.ini +++ b/tox.ini @@ -152,6 +152,20 @@ commands = commands = {[base]pip_install} acme . certbot-apache certbot-compatibility-test {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules +passenv = + SERVER + +[testenv:apacheconftest-with-pebble] +commands = + {toxinidir}/tests/pebble-fetch.sh + {[testenv:apacheconftest]commands} +passenv = + HOME + GOPATH + PEBBLEPATH + PEBBLE_STRICT +setenv = + SERVER=https://localhost:14000/dir [testenv:nginxroundtrip] commands = From 130c29e333eb8b660c02f0b4cd85ba236311ef01 Mon Sep 17 00:00:00 2001 From: Sebastiaan Lokhorst Date: Wed, 9 Jan 2019 21:47:44 +0100 Subject: [PATCH 504/639] Remove some irrelevant history in using.rst (#6639) The documentation only applies to the current version of Certbot. Move Apache plugin details out of the table. --- docs/using.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 5e8675418..d77badd65 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -44,8 +44,7 @@ a combination_ of distinct authenticator and installer plugins. =========== ==== ==== =============================================================== ============================= Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= -apache_ Y Y | Automates obtaining and installing a certificate with Apache http-01_ (80) - | 2.4 on OSes with ``libaugeas0`` 1.0+. +apache_ Y Y | Automates obtaining and installing a certificate with Apache. http-01_ (80) nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. http-01_ (80) webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. @@ -83,8 +82,7 @@ the circumstances in which each plugin can be used, and how to use it. Apache ------ -The Apache plugin currently requires an OS with augeas version 1.0; currently `it -supports +The Apache plugin currently `supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. This automates both obtaining *and* installing certificates on an Apache @@ -136,9 +134,8 @@ the webserver. Nginx ----- -The Nginx plugin has been distributed with Certbot since version 0.9.0 and should -work for most configurations. We recommend backing up Nginx -configurations before using it (though you can also revert changes to +The Nginx plugin should work for most configurations. We recommend backing up +Nginx configurations before using it (though you can also revert changes to configurations with ``certbot --nginx rollback``). You can use it by providing the ``--nginx`` flag on the commandline. From 8cf3bcd3f3120041d070b0fc8acb4029e1dd1b4b Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 9 Jan 2019 21:52:53 +0100 Subject: [PATCH 505/639] [Windows|Unix] Avoid to re-execute challenges already validated (#6551) In response to #5342. Currently, certbot will execute the operations necessary to validate a challenge even if the challenge has already been validated before against the acme ca server. This can occur for instance if a certificate is asked and issue correctly, then deleted locally, then asked again. It is a corner case, but it will lead to some heavy operations (like updating a DNS zone, or creating an HTTP server) that are not needed. This PR corrects this behavior by not executing challenges already validated, and use them directly instead to issue the certificate. Fixes #5342 * Avoid to execute a given challenge that have been already validated by acme ca server. * Execute tls challenge on a separate dns name, to avoid reusing the existing valid http challenge. * Align with master * Improve log * Simplify the implementation * Update changelog * Add a unit test to ensure that validated challenges are not rerun --- CHANGELOG.md | 4 ++- .../tests/boulder-integration.conf.sh | 3 +- certbot-nginx/tests/boulder-integration.sh | 4 +-- certbot/auth_handler.py | 31 ++++++++++++------- certbot/tests/auth_handler_test.py | 20 +++++++++++- 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 724820356..07d00e3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Avoid to process again challenges that are already validated + when a certificate is issued. ### Changed @@ -22,6 +23,7 @@ all Certbot components during releases for the time being, however, the only package with changes other than its version number was: * acme +* certbot More details about these changes can be found on our GitHub repo. diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 4374f9094..470eab28e 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Based on # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf @@ -52,7 +53,7 @@ http { listen 5002 $default_server; # IPv6. listen [::]:5002 $default_server; - server_name nginx.wtf nginx2.wtf; + server_name nginx.wtf nginx-tls.wtf nginx2.wtf; root $root/webroot; diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 2a24e645f..03425734d 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,8 +39,8 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni -test_deployment_and_rollback nginx.wtf +certbot_test_nginx --domains nginx-tls.wtf run --preferred-challenges tls-sni +test_deployment_and_rollback nginx-tls.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index efee49143..3dfaaf26f 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -31,7 +31,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme_client: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -40,9 +40,9 @@ class AuthHandler(object): type strings with the most preferred challenge listed first """ - def __init__(self, auth, acme, account, pref_challs): + def __init__(self, auth, acme_client, account, pref_challs): self.auth = auth - self.acme = acme + self.acme = acme_client self.account = account self.pref_challs = pref_challs @@ -85,19 +85,26 @@ class AuthHandler(object): self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + ret_val = [aauthzr.authzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] - if not retVal: + if not ret_val: raise errors.AuthorizationError( "Challenges failed for all domains") - return retVal + return ret_val def _choose_challenges(self, aauthzrs): - """Retrieve necessary challenges to satisfy server.""" - logger.info("Performing the following challenges:") - for aauthzr in aauthzrs: + """ + Retrieve necessary and pending challenges to satisfy server. + NB: Necessary and already validated challenges are not retrieved, + as they can be reused for a certificate issuance. + """ + pending_authzrs = [aauthzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status != messages.STATUS_VALID] + if pending_authzrs: + logger.info("Performing the following challenges:") + for aauthzr in pending_authzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -125,7 +132,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[acme.challenges.ChallengeResponse] + resp = [] # type: Collection[challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -531,7 +538,7 @@ def _report_failed_challs(failed_achalls): """ problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: problems[achall.error.typ].append(achall) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index e1319b614..fe0ece12e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,7 +57,7 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, authzr, [0]) -class HandleAuthorizationsTest(unittest.TestCase): +class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-public-methods """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -316,6 +316,24 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_validated_challenge_not_rerun(self): + # With pending challenge, we expect the challenge to be tried, and fail. + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "0", + [acme_util.HTTP01], + [messages.STATUS_PENDING], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + # With validated challenge; we expect the challenge not be tried again, and succeed. + authzr = acme_util.gen_authzr( + messages.STATUS_VALID, "0", + [acme_util.HTTP01], + [messages.STATUS_VALID], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.handler.handle_authorizations(mock_order) + def _validate_all(self, aauthzrs, unused_1, unused_2): for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr From 651de2dd2fea9c79caae96d039f2570518735bdc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 10 Jan 2019 20:36:15 +0100 Subject: [PATCH 506/639] Update Certbot dependency to Lexicon to 3.x (#6593) This PR updates Lexicon dependency to the latest version available, 3.0.6, for every lexicon-based DNS plugins. It updates also the provider construction to use the new ConfigResolverobject, and to remove the legacy configuration process. --- CHANGELOG.md | 12 +++++- .../certbot_dns_cloudxns/dns_cloudxns.py | 8 ++-- .../local-oldest-requirements.txt | 4 +- certbot-dns-cloudxns/setup.py | 6 +-- .../certbot_dns_dnsimple/dns_dnsimple.py | 8 ++-- .../local-oldest-requirements.txt | 5 ++- certbot-dns-dnsimple/setup.py | 6 +-- .../dns_dnsmadeeasy.py | 8 ++-- .../local-oldest-requirements.txt | 4 +- certbot-dns-dnsmadeeasy/setup.py | 6 +-- .../certbot_dns_gehirn/dns_gehirn.py | 8 ++-- .../local-oldest-requirements.txt | 2 + certbot-dns-gehirn/setup.py | 4 +- .../certbot_dns_linode/dns_linode.py | 9 +++-- .../local-oldest-requirements.txt | 4 +- certbot-dns-linode/setup.py | 4 +- .../certbot_dns_luadns/dns_luadns.py | 8 ++-- .../local-oldest-requirements.txt | 4 +- certbot-dns-luadns/setup.py | 6 +-- .../certbot_dns_nsone/dns_nsone.py | 8 ++-- .../local-oldest-requirements.txt | 4 +- certbot-dns-nsone/setup.py | 6 +-- certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py | 8 ++-- certbot-dns-ovh/local-oldest-requirements.txt | 4 +- certbot-dns-ovh/setup.py | 6 +-- .../dns_sakuracloud.py | 8 ++-- .../local-oldest-requirements.txt | 2 + certbot-dns-sakuracloud/setup.py | 4 +- certbot/plugins/dns_common_lexicon.py | 37 ++++++++++++++++++- tools/dev_constraints.txt | 2 +- tools/oldest_constraints.txt | 6 +++ 31 files changed, 143 insertions(+), 68 deletions(-) create mode 100644 certbot-dns-gehirn/local-oldest-requirements.txt create mode 100644 certbot-dns-sakuracloud/local-oldest-requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d00e3fa..1ca22f830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support + on 2.x branch is maintained). ### Fixed @@ -24,6 +25,15 @@ package with changes other than its version number was: * acme * certbot +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-sakuracloud More details about these changes can be found on our GitHub repo. diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py index 658db6072..5132137f8 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -69,13 +69,15 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_CloudXNSLexiconClient, self).__init__() - self.provider = cloudxns.Provider({ - 'provider_name': 'cloudxns', + config = dns_common_lexicon.build_lexicon_config('cloudxns', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = cloudxns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-cloudxns/local-oldest-requirements.txt +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5c8709445..1a6f900d8 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py index 3eb56e37c..ad2a3fa30 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -65,12 +65,14 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, token, ttl): super(_DNSimpleLexiconClient, self).__init__() - self.provider = dnsimple.Provider({ - 'provider_name': 'dnssimple', - 'auth_token': token, + config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, + }, { + 'auth_token': token, }) + self.provider = dnsimple.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 8368d266e..5ecd3cbc7 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] +dns-lexicon==2.2.1 \ No newline at end of file diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 3bb43cf5d..1a2ce5d92 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py index 4236ce37a..4b63cb4b5 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py @@ -71,13 +71,15 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_DNSMadeEasyLexiconClient, self).__init__() - self.provider = dnsmadeeasy.Provider({ - 'provider_name': 'dnsmadeeasy', + config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = dnsmadeeasy.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and str(e).startswith('404 Client Error: Not Found for url:'): return diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 599cec486..0a99f452d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py index 9c35e72ab..edf530072 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -72,13 +72,15 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_GehirnLexiconClient, self).__init__() - self.provider = gehirn.Provider({ - 'provider_name': 'gehirn', + config = dns_common_lexicon.build_lexicon_config('gehirn', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = gehirn.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-gehirn/local-oldest-requirements.txt b/certbot-dns-gehirn/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-gehirn/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 894d809ac..f4a75379c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -6,8 +6,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.22', 'mock', 'setuptools', diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 01da2cf60..4e0500fa0 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -54,6 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): def _get_linode_client(self): return _LinodeLexiconClient(self.credentials.conf('key')) + class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ Encapsulates all communication with the Linode API. @@ -61,11 +62,13 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key): super(_LinodeLexiconClient, self).__init__() - self.provider = linode.Provider({ - 'provider_name': 'linode', - 'auth_token': api_key + + config = dns_common_lexicon.build_lexicon_config('linode', {}, { + 'auth_token': api_key, }) + self.provider = linode.Provider(config) + def _handle_general_error(self, e, domain_name): if not str(e).startswith('Domain not found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 588b6a40a..31c2c20bc 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -5,8 +5,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.2.1', 'mock', 'setuptools', diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py index bd6a16f69..7cdd4c8e1 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py @@ -68,13 +68,15 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, email, token, ttl): super(_LuaDNSLexiconClient, self).__init__() - self.provider = luadns.Provider({ - 'provider_name': 'luadns', + config = dns_common_lexicon.build_lexicon_config('luadns', { + 'ttl': ttl, + }, { 'auth_username': email, 'auth_token': token, - 'ttl': ttl, }) + self.provider = luadns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-luadns/local-oldest-requirements.txt +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b09a09762..31472e8cf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py index 5f33efbba..3e23df11c 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py @@ -65,12 +65,14 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, ttl): super(_NS1LexiconClient, self).__init__() - self.provider = nsone.Provider({ - 'provider_name': 'nsone', - 'auth_token': api_key, + config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, + }, { + 'auth_token': api_key, }) + self.provider = nsone.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:') or \ str(e).startswith("400 Client Error: Bad Request for url:")): diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-nsone/local-oldest-requirements.txt +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e48428191..41b99cc73 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py index 578ee8e89..84771b0a8 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -77,15 +77,17 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): super(_OVHLexiconClient, self).__init__() - self.provider = ovh.Provider({ - 'provider_name': 'ovh', + config = dns_common_lexicon.build_lexicon_config('ovh', { + 'ttl': ttl, + }, { 'auth_entrypoint': endpoint, 'auth_application_key': application_key, 'auth_application_secret': application_secret, 'auth_consumer_key': consumer_key, - 'ttl': ttl, }) + self.provider = ovh.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8b6d73d33..5b3329568 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py index b892330f5..7fd6d3ef5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -75,13 +75,15 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_SakuraCloudLexiconClient, self).__init__() - self.provider = sakuracloud.Provider({ - 'provider_name': 'sakuracloud', + config = dns_common_lexicon.build_lexicon_config('sakuracloud', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = sakuracloud.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-sakuracloud/local-oldest-requirements.txt b/certbot-dns-sakuracloud/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-sakuracloud/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 05843d2ed..4ebfc6e1d 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -6,8 +6,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.23', 'mock', 'setuptools', diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py index f9610b816..5b50cc285 100644 --- a/certbot/plugins/dns_common_lexicon.py +++ b/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,22 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" - import logging from requests.exceptions import HTTPError, RequestException +from acme.magic_typing import Union, Dict, Any # pylint: disable=unused-import,no-name-in-module from certbot import errors from certbot.plugins import dns_common +# Lexicon is not declared as a dependency in Certbot itself, +# but in the Certbot plugins backed by Lexicon. +# So we catch import error here to allow this module to be +# always importable, even if it does not make sense to use it +# if Lexicon is not available, obviously. +try: + from lexicon.config import ConfigResolver +except ImportError: + ConfigResolver = None # type: ignore + logger = logging.getLogger(__name__) @@ -100,3 +110,28 @@ class LexiconClient(object): if not str(e).startswith('No domain found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' .format(domain_name, e)) + + +def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): + # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] + """ + Convenient function to build a Lexicon 2.x/3.x config object. + :param str lexicon_provider_name: the name of the lexicon provider to use + :param dict lexicon_options: options specific to lexicon + :param dict provider_options: options specific to provider + :return: configuration to apply to the provider + :rtype: ConfigurationResolver or dict + """ + config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config.update(lexicon_options) + if not ConfigResolver: + # Lexicon 2.x + config.update(provider_options) + else: + # Lexicon 3.x + provider_config = {} + provider_config.update(provider_options) + config[lexicon_provider_name] = provider_config + config = ConfigResolver().with_dict(config).with_env() + + return config diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 778012d31..111dc5495 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.12.36 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.14 +dns-lexicon==3.0.8 dnspython==1.15.0 docutils==0.12 execnet==1.5.0 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 20fa0672a..642af660e 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -50,6 +50,12 @@ ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 +# Ubuntu Bionic constraints +# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some +# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure +# that Lexicon 2.x works with Certbot. +dns-lexicon==2.7.14 + # Plugin constraints # These aren't necessarily the oldest versions we need to support # Tracking at https://github.com/certbot/certbot/issues/6473 From e34e0d9f1aa7d472916d3700367b016a1fc02a2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 14 Jan 2019 14:06:52 -0800 Subject: [PATCH 507/639] Change build status links to travis-ci.com. (#6656) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 62681c7eb..f55581268 100644 --- a/README.rst +++ b/README.rst @@ -99,8 +99,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme |build-status| |coverage| |docs| |container| -.. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master - :target: https://travis-ci.org/certbot/certbot +.. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master + :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status .. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg From f7b8e1f99e2b5b5a05b7a37105b4d109bd150410 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 15 Jan 2019 12:03:49 -0800 Subject: [PATCH 508/639] Modify delete after revoke question to clarify that the flag deletes the entire lineage --- certbot/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index ac639bc80..a0c0ab64d 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -548,7 +548,8 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b attempt_deletion = config.delete_after_revoke if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked?") + msg = ("Would you like to delete the cert(s) you just revoked, along with all earlier and " + "later versions of the cert?") attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", force_interactive=True, default=True) From 6f754f63b187f0a3b9bbc5271675a7a32f42c7e0 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 15 Jan 2019 12:05:00 -0800 Subject: [PATCH 509/639] Update --help for --delete-after-revoke to clarify that the flag deletes the entire lineage --- certbot/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index ff1827cb1..31f55711f 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1311,7 +1311,8 @@ def _create_subparsers(helpful): helpful.add("revoke", "--delete-after-revoke", action="store_true", default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them.") + help="Delete certificates after revoking them, along with all previous and later " + "versions of those certificates.") helpful.add("revoke", "--no-delete-after-revoke", action="store_false", dest="delete_after_revoke", From 340d5468453dd9c88bb4537a451ff26555151abe Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 15 Jan 2019 12:11:22 -0800 Subject: [PATCH 510/639] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca22f830..1861e38bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fixed accessing josepy contents through acme.jose when the full acme.jose path is used. +* Clarify behavior for deleting certs as part of revocation. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only @@ -47,7 +48,7 @@ More details about these changes can be found on our GitHub repo. * Copied account management functionality from the `register` subcommand to the `update_account` subcommand. -* Marked usage `register --update-registration` for deprecation and +* Marked usage `register --update-registration` for deprecation and removal in a future release. ### Fixed From d3a8a54c9520fa91b9d394c2e990d1890376bad3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 16 Jan 2019 21:16:54 +0100 Subject: [PATCH 511/639] Setup cron build with specific jobs (#6661) Fixes #6659 It appears that the conditional statement if: cron = type works as expected: on a push event, jobs marked with this statement are completely ignored on a cron event, these jobs are executed This PR takes advantages of it, by integrating the specific environments from test-everything branch to be merged into master, and add the conditional statement to execute them only during cron builds. Once this PR is merged, a cron job can be set to build master at the appropriate time of the day, and test-everything will not be needed anymore. For execution examples from this branch, see: during a push event: https://travis-ci.org/adferrand/certbot/builds/480127828 during a cron event: https://travis-ci.org/adferrand/certbot/builds/480130236 * Use the conditional expression in travis.yml to execute jobs from test-everything only during cron executions. * Update .travis.yml * Update .travis.yml * Remove before_install --- .travis.yml | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a35d43b63..e733c5fc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,15 +4,13 @@ cache: directories: - $HOME/.cache/pip -before_install: - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' - before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' - export TOX_TESTENV_PASSENV=TRAVIS matrix: include: + # These environments are always executed - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required @@ -64,6 +62,109 @@ matrix: - python: "2.7" env: TOXENV=nginxroundtrip + # These environments are executed on cron events only + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_precise + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_wheezy + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_centos6 + services: docker + if: type = cron + - sudo: required + env: TOXENV=docker_dev + services: docker + addons: + apt: + packages: # don't install nginx and apache + - libaugeas0 + if: type = cron + - language: generic + env: TOXENV=py27 + os: osx + addons: + homebrew: + packages: + - augeas + - python2 + if: type = cron + - language: generic + env: TOXENV=py3 + os: osx + addons: + homebrew: + packages: + - augeas + - python3 + if: type = cron + + # Only build pushes to the master branch, PRs, and branches beginning with # `test-` or of the form `digit(s).digit(s).x`. This reduces the number of From 693bac4b4943898ced3397fa0d6e436ef2343159 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 Jan 2019 13:17:37 -0800 Subject: [PATCH 512/639] Update outdated tests (#6515) These tests were running on Ubuntu Precise and Debian Wheezy which have reached their end of life and are no longer maintained by the respective distros. This updates the tests to a newer version of Debian and Ubuntu. * Remove tests on the deprecated precise. * Add tests for Xenial. * update Jessie tests to use Wheezy * update .travis.yml --- .travis.yml | 4 ++-- .../{Dockerfile.wheezy => Dockerfile.jessie} | 2 +- .../{Dockerfile.precise => Dockerfile.xenial} | 2 +- tox.ini | 24 +++++++++---------- 4 files changed, 15 insertions(+), 17 deletions(-) rename letsencrypt-auto-source/{Dockerfile.wheezy => Dockerfile.jessie} (98%) rename letsencrypt-auto-source/{Dockerfile.precise => Dockerfile.xenial} (98%) diff --git a/.travis.yml b/.travis.yml index e733c5fc7..38543845a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,11 +126,11 @@ matrix: services: docker if: type = cron - sudo: required - env: TOXENV=le_auto_precise + env: TOXENV=le_auto_xenial services: docker if: type = cron - sudo: required - env: TOXENV=le_auto_wheezy + env: TOXENV=le_auto_jessie services: docker if: type = cron - sudo: required diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.jessie similarity index 98% rename from letsencrypt-auto-source/Dockerfile.wheezy rename to letsencrypt-auto-source/Dockerfile.jessie index f4f3fea15..9ee37b763 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.jessie @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM debian:wheezy +FROM debian:jessie # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.xenial similarity index 98% rename from letsencrypt-auto-source/Dockerfile.precise rename to letsencrypt-auto-source/Dockerfile.xenial index 39a167c14..931f1c6d3 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.xenial @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM ubuntu:precise +FROM ubuntu:xenial # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/tox.ini b/tox.ini index 95b2f2c64..36057b5df 100644 --- a/tox.ini +++ b/tox.ini @@ -198,17 +198,6 @@ whitelist_externals = passenv = DOCKER_* -[testenv:le_auto_precise] -# At the moment, this tests under Python 2.7 only, as only that version is -# readily available on the Precise Docker image. -commands = - docker build -f letsencrypt-auto-source/Dockerfile.precise -t lea letsencrypt-auto-source - docker run --rm -t -i lea -whitelist_externals = - docker -passenv = - DOCKER_* - [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Trusty Docker image. @@ -222,11 +211,20 @@ passenv = DOCKER_* TRAVIS_BRANCH -[testenv:le_auto_wheezy] +[testenv:le_auto_xenial] +# At the moment, this tests under Python 2.7 only. +commands = + docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source + docker run --rm -t -i lea +whitelist_externals = + docker +passenv = DOCKER_* + +[testenv:le_auto_jessie] # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Wheezy Docker image. commands = - docker build -f letsencrypt-auto-source/Dockerfile.wheezy -t lea letsencrypt-auto-source + docker build -f letsencrypt-auto-source/Dockerfile.jessie -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = docker From 6f388945cde7e6ec68a83378dea9c6122b86d2c9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 Jan 2019 09:41:46 -0800 Subject: [PATCH 513/639] Remove reference to latest draft implemented. (#6669) --- acme/acme/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index bab5b40ce..d91072a3b 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,14 +1,9 @@ """ACME protocol implementation. -This module is an implementation of the `ACME protocol`_. Latest -supported version: `draft-ietf-acme-01`_. - +This module is an implementation of the `ACME protocol`_. .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme -.. _`draft-ietf-acme-01`: - https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ import sys From 835f4b9271b7b5ee3cb3f54254a015e28f7bffdc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 17 Jan 2019 22:02:35 +0100 Subject: [PATCH 514/639] Allow to execute a tox target without pinned dependencies (#6590) This PR passes the CERTBOT_NO_PIN environment variable to the unit tests tox envs. By setting CERTBOT_NO_PIN to 1 before executing a given tox env, certbot dependencies will be installed at their latest version instead of the usual pinned version. I also moved the unpin logic one layer below to allow it to be used potentially more widely, and avoid unnecessary merging constraints operation in this case. As warnings are errors now, latest versions of Python will break now the tests, because collections launch a warning when some classes are imported from collections instead of collections.abc. Certbot code is patched, and warning is ignored for now, because a lot of third party libraries still depend on this behavior. * Allow to execute a tox target without pinned dependencies * Correct lint * Retrigger build. * Remove debug code * Added test against unpinned dependencies from test-everything-unpinned-dependencies branch * Remove duplicated assertion to pass TRAVIS and APPVEYOR in default tox environment. --- .travis.yml | 4 ++++ acme/acme/messages.py | 7 +++++-- pytest.ini | 10 +++++++--- tools/install_and_test.py | 9 ++++----- tools/pip_install.py | 27 +++++++++++++++++---------- tox.ini | 2 ++ 6 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38543845a..16cb6f23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,10 @@ matrix: env: TOXENV=nginxroundtrip # These environments are executed on cron events only + - python: "3.7" + dist: xenial + env: TOXENV=py37 CERTBOT_NO_PIN=1 + if: type = cron - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4400a6c31..7c82c8507 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,7 +1,10 @@ """ACME protocol messages.""" -import collections import six import json +try: + from collections.abc import Hashable # pylint: disable=no-name-in-module +except ImportError: + from collections import Hashable import josepy as jose @@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None).decode() -class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore +class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented diff --git a/pytest.ini b/pytest.ini index 9a5807f34..8f009045c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,10 +3,14 @@ # directly. [pytest] addopts = --numprocesses auto --pyargs -# ResourceWarnings are ignored as errors, since they're raised at close -# decodestring: https://github.com/rthalley/dnspython/issues/338 -# ignore our own TLS-SNI-01 warning +# In general, all warnings are treated as errors. Here are the exceptions: +# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- ignore our own TLS-SNI-01 warning +# 3- ignore warn for importing abstract classes from collections instead of collections.abc, +# too much third party dependencies are still relying on this behavior, +# but it should be corrected to allow Certbot compatiblity with Python >= 3.8 filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning + ignore:.*collections\.abc:DeprecationWarning diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 79a7c2264..0142eeea4 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -17,16 +17,15 @@ import re SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): - if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-e'] - else: - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] new_args = [] for arg in args: diff --git a/tools/pip_install.py b/tools/pip_install.py index 4466729e0..332422019 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -80,19 +80,26 @@ def main(args): test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') - requirements = None - if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + if os.environ.get('CERTBOT_NO_PIN') == '1': + # With unpinned dependencies, there is no constraint + call_with_print('"{0}" -m pip install --ignore-installed {1}' + .format(sys.executable, ' '.join(args))) else: - certbot_normal_processing(tools_path, test_constraints) + # Otherwise, we merge requirements to build the constraints and pin dependencies + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) + merge_requirements(tools_path, test_constraints, all_constraints) - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + if requirements: + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) + + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: if os.environ.get('TRAVIS'): print('travis_fold:end:install_certbot_deps') diff --git a/tox.ini b/tox.ini index 36057b5df..363f06bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,8 @@ source_paths = tests/lock_test.py [testenv] +passenv = + CERTBOT_NO_PIN commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py From bb1242c04767c8686de39af78f4f562ae33ded1d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 18 Jan 2019 00:21:13 +0100 Subject: [PATCH 515/639] Process isolated local oldest requirements for DNS plugins (#6653) * Include local-oldest-requirements in the constraints when oldest is invoked * Add oldest constraints for lexicon * Skip editable requirements during merge * Pin to lexicon 2.2.1 for oldest requirements. Override locally oldest if needed for specific providers. --- .../local-oldest-requirements.txt | 1 - certbot-dns-ovh/local-oldest-requirements.txt | 1 + tools/dev_constraints.txt | 6 ++-- tools/merge_requirements.py | 34 +++++++++---------- tools/oldest_constraints.txt | 9 +++-- tools/pip_install.py | 25 +++++++++----- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 5ecd3cbc7..65f5a758e 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,3 +1,2 @@ -e acme[dev] -e .[dev] -dns-lexicon==2.2.1 \ No newline at end of file diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 65f5a758e..01cbcb317 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -e acme[dev] -e .[dev] +dns-lexicon==2.7.14 diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 111dc5495..88340cb00 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,7 @@ -# Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# Specifies Python package versions for development. +# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# Some dev package versions specified here may be overridden by higher level constraints +# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 asn1crypto==0.22.0 diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index ad44a55d0..4205e6bcf 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only simple SomeProject==1.2.3 format is currently supported. """ - from __future__ import print_function import sys + def read_file(file_path): """Reads in a Python requirements file. + Ignore empty lines, comments and editable requirements :param str file_path: path to requirements file @@ -19,16 +20,16 @@ def read_file(file_path): :rtype: dict """ - d = {} - with open(file_path) as f: - for line in f: + data = {} + with open(file_path) as file_h: + for line in file_h: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith('#') and not line.startswith('-e'): project, version = line.split('==') if not version: raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d + data[project] = version + return data def output_requirements(requirements): @@ -37,25 +38,24 @@ def output_requirements(requirements): :param dict requirements: mapping from a project to its pinned version """ - return '\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items())) + return '\n'.join('{0}=={1}'.format(key, value) + for key, value in sorted(requirements.items())) -def main(*files): +def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier files. - :param tuple files: paths to requirements files + :param tuple paths: paths to requirements files """ - d = {} - for f in files: - d.update(read_file(f)) - return output_requirements(d) + data = {} + for path in paths: + data.update(read_file(path)) + return output_requirements(data) if __name__ == '__main__': - merged_requirements = main(*sys.argv[1:]) - print(merged_requirements) + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 642af660e..e48d6b13c 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -50,11 +50,10 @@ ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 -# Ubuntu Bionic constraints -# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some -# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure -# that Lexicon 2.x works with Certbot. -dns-lexicon==2.7.14 +# Ubuntu Bionic constraints. +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==2.2.1 # Plugin constraints # These aren't necessarily the oldest versions we need to support diff --git a/tools/pip_install.py b/tools/pip_install.py index 332422019..a03ed70d7 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements if not os.path.isfile(requirements): - requirements = None - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + return None return requirements @@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints): fd.write('{0}{1}'.format(search.group(1), os.linesep)) -def merge_requirements(tools_path, test_constraints, all_constraints): - merged_requirements = merge_module.main( - os.path.join(tools_path, 'dev_constraints.txt'), - test_constraints - ) +def merge_requirements(tools_path, requirements, test_constraints, all_constraints): + # Order of the files in the merge function matters. + # Indeed version retained for a given package will be the last version + # found when following all requirements in the given order. + # Here is the order by increasing priority: + # 1) The general development constraints (tools/dev_constraints.txt) + # 2) The general tests constraints (oldest_requirements.txt or + # certbot-auto's dependency-requirements.txt for the normal processing) + # 3) The local requirement file, typically local-oldest-requirement in oldest tests + files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + if requirements: + files.append(requirements) + merged_requirements = merge_module.main(*files) with open(all_constraints, 'w') as fd: fd.write(merged_requirements) @@ -92,8 +100,7 @@ def main(args): else: certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - + merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' .format(sys.executable, all_constraints, requirements)) From 39eb66a8ad6e326118c36319818ad144fab6e85d Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 17 Jan 2019 16:00:39 -0800 Subject: [PATCH 516/639] pin dependency-requirements.txt in Dockerfile (#6670) Fixes #6580 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d1296b30f..f3626dc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . COPY acme src/acme COPY certbot src/certbot @@ -21,6 +22,7 @@ RUN apk add --no-cache --virtual .build-deps \ openssl-dev \ musl-dev \ libffi-dev \ + && pip install -r /opt/certbot/dependency-requirements.txt \ && pip install --no-cache-dir \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ From b58d0e722d4d3b54ede48a67aeb25d4fa6d6fc45 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 Jan 2019 05:27:31 -0800 Subject: [PATCH 517/639] Remove certbot-auto code for Wheezy and Precise (#6672) * Remove lsb_release dependency. * Remove more code. --- letsencrypt-auto-source/letsencrypt-auto | 52 ------------------- .../pieces/bootstrappers/deb_common.sh | 52 ------------------- 2 files changed, 104 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c08a08160..72d32af8d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index eb22225e4..93bdc63b4 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -43,63 +43,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ From 0f1d78e8974319dde66bf8f8b12acf8419f3e6b7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 18 Jan 2019 18:47:43 +0100 Subject: [PATCH 518/639] Do not ignore editable distributions already installed. (#6675) This PR fix tests when run with unpinned dependencies. Option --ignore-installed is removed from pip command, allowing to use local dev modules (like acme) that are not still released. Be warned that these tests will mostly work correctly if the relevant virtualenv in .tox does not exist yet. Otherwise, already installed distribution, potentially not up-to-date, will be reused without update from pip if they match the minimal requirements in relevant projects. This is unlikely to be an issue on CI systems, that will start from a fresh system. We may build a dedicated tox environment for unpinned dependencies tests purpose, and ensure consistency of these tests on all situations. Fixes #6674 --- tools/pip_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pip_install.py b/tools/pip_install.py index a03ed70d7..dd6302b48 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -90,7 +90,7 @@ def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint - call_with_print('"{0}" -m pip install --ignore-installed {1}' + call_with_print('"{0}" -m pip install {1}' .format(sys.executable, ' '.join(args))) else: # Otherwise, we merge requirements to build the constraints and pin dependencies From dde27e5aefdfb093636f83801d6a489b8f89fe0d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sun, 20 Jan 2019 16:53:18 +0100 Subject: [PATCH 519/639] Remove tls-sni-01 challenges in integration tests (#6679) * Remove tls-sni-01 challenges in integration tests * Remove the tls-sni test in the less invasive way * Correct code coverage from tls-sni logic not been tested anymore. * Update certbot-boulder-integration.sh --- certbot-nginx/tests/boulder-integration.sh | 4 +--- tests/certbot-boulder-integration.sh | 16 ++++++++-------- tests/integration/_common.sh | 6 +++--- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 03425734d..547502ccf 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,8 +39,6 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -certbot_test_nginx --domains nginx-tls.wtf run --preferred-challenges tls-sni -test_deployment_and_rollback nginx-tls.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 @@ -66,4 +64,4 @@ test_deployment_and_rollback nginx6.wtf # top nginx -c $nginx_root/nginx.conf -s stop -coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing +coverage report --fail-under 72 --include 'certbot-nginx/*' --show-missing diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 630571148..c03b3def8 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -221,20 +221,20 @@ common plugins --init --prepare | grep webroot # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & +python ./tests/run_http_server.py $tls_alpn_01_port & python_server_pid=$! - certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ +common --domains le1.wtf --preferred-challenges http-01 auth \ --cert-name $certname \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid CheckDeployHook $certname -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! +# Previous test used to be a tls-sni-01 challenge that is not supported anymore. +# Now it is a http-01 challenge and this makes it a duplicate of the following test. +# But removing it would break many tests here, as they are strongly coupled. +# See https://github.com/certbot/certbot/pull/6679 certname="le2.wtf" common --domains le2.wtf --preferred-challenges http-01 run \ --cert-name $certname \ @@ -254,7 +254,7 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ CheckRenewHook $certname certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ +common -a manual -d dns.le.wtf --preferred-challenges dns run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ @@ -396,7 +396,7 @@ CheckDirHooks 1 # with fail. common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ + --preferred-challenges dns \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 83aa91a9e..76b990ac4 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,10 +3,10 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -tls_sni_01_port=5001 +tls_alpn_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir tls_sni_01_port http_01_port sources +export root config_dir tls_alpn_01_port http_01_port sources certbot_path="$(command -v certbot)" # Flags that are added here will be added to Certbot calls within # certbot_test_no_force_renew. @@ -60,7 +60,7 @@ certbot_test_no_force_renew () { "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ - --tls-sni-01-port $tls_sni_01_port \ + --tls-sni-01-port $tls_alpn_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ $other_flags \ From a1313267f033f27133d18e3e7315b5e3947a79bf Mon Sep 17 00:00:00 2001 From: Rishabh Kumar Date: Sun, 20 Jan 2019 23:47:03 +0530 Subject: [PATCH 520/639] Remove dead autodeployment code (#6678) Fixes #4315 --- certbot/storage.py | 39 ----------------------------------- certbot/tests/storage_test.py | 26 +---------------------- 2 files changed, 1 insertion(+), 64 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 7472df975..d17a0f29d 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -879,45 +879,6 @@ class RenewableCert(object): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def autodeployment_is_enabled(self): - """Is automatic deployment enabled for this cert? - - If autodeploy is not specified, defaults to True. - - :returns: True if automatic deployment is enabled - :rtype: bool - - """ - return ("autodeploy" not in self.configuration or - self.configuration.as_bool("autodeploy")) - - def should_autodeploy(self, interactive=False): - """Should this lineage now automatically deploy a newer version? - - This is a policy question and does not only depend on whether - there is a newer version of the cert. (This considers whether - autodeployment is enabled, whether a relevant newer version - exists, and whether the time interval for autodeployment has - been reached.) - - :param bool interactive: set to True to examine the question - regardless of whether the renewal configuration allows - automated deployment (for interactive use). Default False. - - :returns: whether the lineage now ought to autodeploy an - existing newer cert version - :rtype: bool - - """ - if interactive or self.autodeployment_is_enabled(): - if self.has_pending_deployment(): - interval = self.configuration.get("deploy_before_expiry", - "5 days") - now = pytz.UTC.fromutc(datetime.datetime.utcnow()) - if self.target_expiry < add_time_interval(now, interval): - return True - return False - def ocsp_revoked(self, version=None): # pylint: disable=no-self-use,unused-argument """Is the specified cert version revoked according to OCSP? diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index d75f4f595..e61ed2aca 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -388,8 +388,7 @@ class RenewableCertTests(BaseRenewableCertTest): @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.datetime") def test_time_interval_judgments(self, mock_datetime, mock_cli): - """Test should_autodeploy() and should_autorenew() on the basis - of expiry time windows.""" + """Test should_autorenew() on the basis of expiry time windows.""" test_cert = test_util.load_vector("cert_512.pem") self._write_out_ex_kinds() @@ -430,31 +429,8 @@ class RenewableCertTests(BaseRenewableCertTest): mock_datetime.datetime.utcnow.return_value = sometime self.test_rc.configuration["deploy_before_expiry"] = interval self.test_rc.configuration["renew_before_expiry"] = interval - self.assertEqual(self.test_rc.should_autodeploy(), result) self.assertEqual(self.test_rc.should_autorenew(), result) - def test_autodeployment_is_enabled(self): - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - self.test_rc.configuration["autodeploy"] = "1" - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.autodeployment_is_enabled()) - - def test_should_autodeploy(self): - """Test should_autodeploy() on the basis of reasons other than - expiry time window.""" - # pylint: disable=too-many-statements - # Autodeployment turned off - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.should_autodeploy()) - self.test_rc.configuration["autodeploy"] = "1" - # No pending deployment - for ver in six.moves.range(1, 6): - for kind in ALL_FOUR: - self._write_out_kind(kind, ver) - self.assertFalse(self.test_rc.should_autodeploy()) - def test_autorenewal_is_enabled(self): self.test_rc.configuration["renewalparams"] = {} self.assertTrue(self.test_rc.autorenewal_is_enabled()) From 04ac84c4c1d535b497cb70865d175be931e02147 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 20 Jan 2019 10:24:43 -0800 Subject: [PATCH 521/639] Update pinned version of pyOpenSSL (#6571) * Update pinned version of pyOpenSSL. * Run build to update letsencrypt-auto. --- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/dependency-requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 72d32af8d..d86401904 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1086,9 +1086,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index eb297bc6e..1fac78836 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -114,9 +114,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ From dc8217939516409b04439926b4263491a5ce7d72 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 22 Jan 2019 01:39:31 -0800 Subject: [PATCH 522/639] Remove ACMEv1 example. (#6668) --- acme/docs/index.rst | 7 ----- acme/examples/example_client.py | 47 --------------------------------- 2 files changed, 54 deletions(-) delete mode 100644 acme/examples/example_client.py diff --git a/acme/docs/index.rst b/acme/docs/index.rst index a200808da..a94c02e5c 100644 --- a/acme/docs/index.rst +++ b/acme/docs/index.rst @@ -16,13 +16,6 @@ Contents: .. automodule:: acme :members: - -Example client: - -.. include:: ../examples/example_client.py - :code: python - - Indices and tables ================== diff --git a/acme/examples/example_client.py b/acme/examples/example_client.py deleted file mode 100644 index abeedc082..000000000 --- a/acme/examples/example_client.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Example script showing how to use acme client API.""" -import logging -import os -import pkg_resources - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa -import josepy as jose -import OpenSSL - -from acme import client -from acme import messages - - -logging.basicConfig(level=logging.DEBUG) - - -DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' -BITS = 2048 # minimum for Boulder -DOMAIN = 'example1.com' # example.com is ignored by Boulder - -# generate_private_key requires cryptography>=0.5 -key = jose.JWKRSA(key=rsa.generate_private_key( - public_exponent=65537, - key_size=BITS, - backend=default_backend())) -acme = client.Client(DIRECTORY_URL, key) - -regr = acme.register() -logging.info('Auto-accepting TOS: %s', regr.terms_of_service) -acme.agree_to_tos(regr) -logging.debug(regr) - -authzr = acme.request_challenges( - identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN)) -logging.debug(authzr) - -authzr, authzr_response = acme.poll(authzr) - -csr = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, pkg_resources.resource_string( - 'acme', os.path.join('testdata', 'csr.der'))) -try: - acme.request_issuance(jose.util.ComparableX509(csr), (authzr,)) -except messages.Error as error: - print ("This script is doomed to fail as no authorization " - "challenges are ever solved. Error from server: {0}".format(error)) From 658b7b9d47cb05fa43383b5bdf3beea0dc99b886 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 10:03:52 -0800 Subject: [PATCH 523/639] Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv (#6690) This will immediately address the breakage reported in #6682 and tracked at #6685. Virtualenv downloads the latest pip, which causes issues, so tell virtualenv to not download the latest pip. I added the flag preemptively to other files as well, they're in separate commits so it will be easy to revert any spots we don't want. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. * Update changelog * Use an environment variable instead of a flag for compatibility with old versions * Run build.py --- CHANGELOG.md | 2 ++ Dockerfile-old | 2 +- certbot-compatibility-test/Dockerfile | 2 +- letsencrypt-auto-source/letsencrypt-auto | 6 ++++-- letsencrypt-auto-source/letsencrypt-auto.template | 6 ++++-- tools/_release.sh | 4 ++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1861e38bf..9640d662f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fixed accessing josepy contents through acme.jose when the full acme.jose path is used. * Clarify behavior for deleting certs as part of revocation. +* Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages + from venv downloading the latest pip Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only diff --git a/Dockerfile-old b/Dockerfile-old index da48c7fc8..c52a9937b 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -51,7 +51,7 @@ COPY certbot-apache /opt/certbot/src/certbot-apache/ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv # PATH is set now so pipstrap upgrades the correct (v)env ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index cbbdf7193..1e9e0d727 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tools /opt/certbot/src/tools -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d86401904..02f6b810b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -945,10 +945,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f431e32e4..2cd3f4336 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -537,10 +537,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then diff --git a/tools/_release.sh b/tools/_release.sh index ec2deda22..d75a0f487 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -42,7 +42,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -virtualenv --no-site-packages -p python2 $tmpvenv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 $tmpvenv . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -135,7 +135,7 @@ cd "dist.$version" python -m SimpleHTTPServer $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) -virtualenv --no-site-packages ../venv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip From 34d655151d158b0227e210682ac7bb753afea0d4 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 11:37:36 -0800 Subject: [PATCH 524/639] Remove commas in filename (#6692) This will immediately address the breakage reported in #6682 and tracked at #6685. Pip 19.0.0 and 19.0.1 don't allow commas in filenames, so don't use commas in filenames in certbot-apache test code. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. Steps to test: push master to test box run tools/venv.py activate venv pip --version: 19.0.1 pip install ./certbot-apache/: fails push branch code to test box confirm pip --version still 19.0.1 pip install ./certbot-apache/: success * Rename old,default.conf to old-and-default.conf * Update changelog * sites-enabled should contain a symlink to sites-available --- CHANGELOG.md | 3 +++ certbot-apache/certbot_apache/tests/configurator_test.py | 4 ++-- .../{old,default.conf => old-and-default.conf} | 0 .../augeas_vhosts/apache2/sites-enabled/old,default.conf | 1 - .../augeas_vhosts/apache2/sites-enabled/old-and-default.conf | 1 + 5 files changed, 6 insertions(+), 3 deletions(-) rename certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/{old,default.conf => old-and-default.conf} (100%) delete mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 9640d662f..7d00da4ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fixed accessing josepy contents through acme.jose when the full acme.jose path is used. * Clarify behavior for deleting certs as part of revocation. +* Rename old,default.conf to old-and-default.conf to address commas in filenames + breaking recent versions of pip. * Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages from venv downloading the latest pip @@ -28,6 +30,7 @@ package with changes other than its version number was: * acme * certbot +* certbot-apache * certbot-dns-cloudxns * certbot-dns-dnsimple * certbot-dns-dnsmadeeasy diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index a77dbf637..4aaa23ea4 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1523,12 +1523,12 @@ class AugeasVhostsTest(util.ApacheTest): def test_choosevhost_with_illegal_name(self): self.config.aug = mock.MagicMock() self.config.aug.match.side_effect = RuntimeError - path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf" + path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf" chosen_vhost = self.config._create_vhost(path) self.assertEqual(None, chosen_vhost) def test_choosevhost_works(self): - path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf" + path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf" chosen_vhost = self.config._create_vhost(path) self.assertTrue(chosen_vhost == None or chosen_vhost.path == path) diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf similarity index 100% rename from certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf rename to certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf deleted file mode 120000 index 6782fe9e5..000000000 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf +++ /dev/null @@ -1 +0,0 @@ -../sites-available/old,default.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf new file mode 120000 index 000000000..f7fdf1bbe --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf @@ -0,0 +1 @@ +../sites-available/old-and-default.conf \ No newline at end of file From aee847a6fb13163b720d9f8e02b252ba95fc7051 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 10:03:52 -0800 Subject: [PATCH 525/639] Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv (#6690) This will immediately address the breakage reported in #6682 and tracked at #6685. Virtualenv downloads the latest pip, which causes issues, so tell virtualenv to not download the latest pip. I added the flag preemptively to other files as well, they're in separate commits so it will be easy to revert any spots we don't want. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. * Update changelog * Use an environment variable instead of a flag for compatibility with old versions * Run build.py (cherry picked from commit 658b7b9d47cb05fa43383b5bdf3beea0dc99b886) --- CHANGELOG.md | 15 ++++++++++++++- Dockerfile-old | 2 +- certbot-compatibility-test/Dockerfile | 2 +- letsencrypt-auto-source/letsencrypt-auto | 6 ++++-- letsencrypt-auto-source/letsencrypt-auto.template | 6 ++++-- tools/_release.sh | 4 ++-- 6 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 182f6db32..3b05f1127 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Certbot change log -Certbot adheres to [Semantic Versioning](http://semver.org/). +Certbot adheres to [Semantic Versioning](https://semver.org/). + +## 0.31.1 - master + +### Fixed + +* Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages + from venv downloading the latest pip + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +More details about these changes can be found on our GitHub repo. ## 0.30.0 - 2019-01-02 diff --git a/Dockerfile-old b/Dockerfile-old index da48c7fc8..c52a9937b 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -51,7 +51,7 @@ COPY certbot-apache /opt/certbot/src/certbot-apache/ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv # PATH is set now so pipstrap upgrades the correct (v)env ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index cbbdf7193..1e9e0d727 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tools /opt/certbot/src/tools -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index be2c3679b..d387f1934 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -997,10 +997,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f431e32e4..2cd3f4336 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -537,10 +537,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then diff --git a/tools/_release.sh b/tools/_release.sh index ec2deda22..d75a0f487 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -42,7 +42,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -virtualenv --no-site-packages -p python2 $tmpvenv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 $tmpvenv . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -135,7 +135,7 @@ cd "dist.$version" python -m SimpleHTTPServer $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) -virtualenv --no-site-packages ../venv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip From 8b3cea67148ec76e51247037fdffcf8c6fb0aed9 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 11:37:36 -0800 Subject: [PATCH 526/639] Remove commas in filename (#6692) This will immediately address the breakage reported in #6682 and tracked at #6685. Pip 19.0.0 and 19.0.1 don't allow commas in filenames, so don't use commas in filenames in certbot-apache test code. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. Steps to test: push master to test box run tools/venv.py activate venv pip --version: 19.0.1 pip install ./certbot-apache/: fails push branch code to test box confirm pip --version still 19.0.1 pip install ./certbot-apache/: success * Rename old,default.conf to old-and-default.conf * Update changelog * sites-enabled should contain a symlink to sites-available (cherry picked from commit 34d655151d158b0227e210682ac7bb753afea0d4) --- CHANGELOG.md | 4 ++++ certbot-apache/certbot_apache/tests/configurator_test.py | 4 ++-- .../{old,default.conf => old-and-default.conf} | 0 .../augeas_vhosts/apache2/sites-enabled/old,default.conf | 1 - .../augeas_vhosts/apache2/sites-enabled/old-and-default.conf | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) rename certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/{old,default.conf => old-and-default.conf} (100%) delete mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b05f1127..5d6ea81e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed +* Rename old,default.conf to old-and-default.conf to address commas in filenames + breaking recent versions of pip. * Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages from venv downloading the latest pip @@ -13,6 +15,8 @@ Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: +* certbot-apache + More details about these changes can be found on our GitHub repo. ## 0.30.0 - 2019-01-02 diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index a77dbf637..4aaa23ea4 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1523,12 +1523,12 @@ class AugeasVhostsTest(util.ApacheTest): def test_choosevhost_with_illegal_name(self): self.config.aug = mock.MagicMock() self.config.aug.match.side_effect = RuntimeError - path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf" + path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf" chosen_vhost = self.config._create_vhost(path) self.assertEqual(None, chosen_vhost) def test_choosevhost_works(self): - path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf" + path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf" chosen_vhost = self.config._create_vhost(path) self.assertTrue(chosen_vhost == None or chosen_vhost.path == path) diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf similarity index 100% rename from certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf rename to certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf deleted file mode 120000 index 6782fe9e5..000000000 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf +++ /dev/null @@ -1 +0,0 @@ -../sites-available/old,default.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf new file mode 120000 index 000000000..f7fdf1bbe --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf @@ -0,0 +1 @@ +../sites-available/old-and-default.conf \ No newline at end of file From 9746c310d84daf68038d2a1cf8fb7429abb2966e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 12:03:21 -0800 Subject: [PATCH 527/639] Always download the pinned version of pip in pipstrap (#6691) This will immediately address the breakage reported in #6682 and tracked at #6685. Virtualenv downloads the latest pip, which causes issues, so after virtualenv upgrades pip, downgrade to the pinned version. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. * Always download the pinned version of pip in pipstrap * Run build.py * Update changelog * Remove unused variable * Run build.py --- CHANGELOG.md | 1 + letsencrypt-auto-source/letsencrypt-auto | 3 --- letsencrypt-auto-source/pieces/pipstrap.py | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d00da4ae..9d6c0318f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fixed accessing josepy contents through acme.jose when the full acme.jose path is used. * Clarify behavior for deleting certs as part of revocation. +* Always download the pinned version of pip in pipstrap to address breakages * Rename old,default.conf to old-and-default.conf to address commas in filenames breaking recent versions of pip. * Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 02f6b810b..c41dc47bc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1346,9 +1346,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 727040c3c..2b3025776 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -148,9 +148,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') From 4c4dcf49871ad61af3f7ea53a01816b4adb63709 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 24 Jan 2019 12:03:21 -0800 Subject: [PATCH 528/639] Always download the pinned version of pip in pipstrap (#6691) This will immediately address the breakage reported in #6682 and tracked at #6685. Virtualenv downloads the latest pip, which causes issues, so after virtualenv upgrades pip, downgrade to the pinned version. I've confirmed that this fixes the issue on a machine that fails with the version of certbot-auto currently in master: recent version of virtualenv, python 2.7. * Always download the pinned version of pip in pipstrap * Run build.py * Update changelog * Remove unused variable * Run build.py (cherry picked from commit 9746c310d84daf68038d2a1cf8fb7429abb2966e) --- CHANGELOG.md | 1 + letsencrypt-auto-source/letsencrypt-auto | 3 --- letsencrypt-auto-source/pieces/pipstrap.py | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d6ea81e6..2ecbf33d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed +* Always download the pinned version of pip in pipstrap to address breakages * Rename old,default.conf to old-and-default.conf to address commas in filenames breaking recent versions of pip. * Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d387f1934..ffa93262d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1398,9 +1398,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 727040c3c..2b3025776 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -148,9 +148,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') From 5de41572e4f3563127d2c0caeb11d79b966c45ab Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 12:40:05 -0800 Subject: [PATCH 529/639] pin back boulder --- tests/boulder-fetch.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index f34deb74e..74ba50e76 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -7,7 +7,10 @@ set -xe export GOPATH=${GOPATH:-$HOME/gopath} BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder} if [ ! -d ${BOULDERPATH} ]; then - git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH} + # Use the latest version of boulder that has TLS-SNI-01 enabled by default. + git clone https://github.com/letsencrypt/boulder ${BOULDERPATH} + cd "$BOULDERPATH" + git checkout 93ac7fbe9e77b190c5c6496b4e9d765775588963 fi cd ${BOULDERPATH} From 73713d7871f9e6787cca5eb5f7d505184b7abcc2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 14:07:15 -0800 Subject: [PATCH 530/639] Update changelog for 0.30.1 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecbf33d2..acf1fb0eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.31.1 - master +## 0.31.1 - 2019-01-24 ### Fixed From fc8f70097b42ce505c7cc267db1d4118b57620db Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 14:13:06 -0800 Subject: [PATCH 531/639] Release 0.30.1 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 35 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 15 ++++---- letsencrypt-auto | 35 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++---- letsencrypt-auto-source/letsencrypt-auto | 26 ++++++------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++------ 26 files changed, 93 insertions(+), 96 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7ddcf0f06..3682b0174 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.0' +version = '0.30.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 42a88b7b4..83c0cd2fd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index be2c3679b..93bb05419 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0" +LE_AUTO_VERSION="0.30.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -997,10 +997,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then @@ -1230,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.0 \ - --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ - --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 -acme==0.30.0 \ - --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ - --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 -certbot-apache==0.30.0 \ - --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ - --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf -certbot-nginx==0.30.0 \ - --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ - --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 +certbot==0.30.1 \ + --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ + --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 +acme==0.30.1 \ + --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ + --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df +certbot-apache==0.30.1 \ + --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ + --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 +certbot-nginx==0.30.1 \ + --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ + --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1396,9 +1398,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e64c7bba1..4f20d5994 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 47888bec1..4bc30c815 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 66420e354..ad9a70d66 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0260d5011..503f71d7f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 5f5e939c2..21097ff24 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 5d5874459..366ffdbf1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 5ba4ece27..815a6d543 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index bc04d3ff1..78fe30dc4 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 4db08c575..b26a79c29 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index f07b08024..0c8080c51 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 0467da1f0..ddae991f4 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 3959e0b84..07cf7392c 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 811d8aec1..22f5c0014 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 5a99c4ffa..3cdf28f8f 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8225e47de..49749c883 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index c19fd862d..6d8a28d3c 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0' +version = '0.30.1' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 768efc2b5..6ff2db69a 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.0' +__version__ = '0.30.1' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index bad6275e7..77c6dd230 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.30.0 + "". (default: CertbotACMEClient/0.30.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -479,10 +479,9 @@ apache: Apache Web Server plugin --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -496,16 +495,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index be2c3679b..93bb05419 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0" +LE_AUTO_VERSION="0.30.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -997,10 +997,12 @@ if [ "$1" = "--le-auto-phase2" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$PYVER" -le 27 ]; then + # Use an environment variable instead of a flag for compatibility with old versions if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ + > /dev/null fi else if [ "$VERBOSE" = 1 ]; then @@ -1230,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.0 \ - --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ - --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 -acme==0.30.0 \ - --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ - --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 -certbot-apache==0.30.0 \ - --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ - --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf -certbot-nginx==0.30.0 \ - --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ - --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 +certbot==0.30.1 \ + --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ + --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 +acme==0.30.1 \ + --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ + --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df +certbot-apache==0.30.1 \ + --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ + --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 +certbot-nginx==0.30.1 \ + --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ + --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1396,9 +1398,6 @@ def get_index_base(): def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) - min_pip_version = StrictVersion(PIP_VERSION) - if pip_version >= min_pip_version: - return 0 has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 4e2a700e5..a5920c7e5 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwtH9cACgkQTRfJlc2X -dfIqUwf/RXLZAeFF/59PjTAzcV+eEISlvEmFcV0zL3vv23PsY3S5Iuuwcd6rTm5M -UWNtmUTmFVo0xmxAj6Eqfpnt0P+JPpPcnbLNIGKFekBWIshgH84RRFWPJjNh/hu1 -pyzkkcWaOB86egdVfjvuRJ0j7AGd0ih6ur2rlgfHVjTYR+0EdWszFDEFBlq8cpct -9d1gCgH7VWKSIQMhzGLMsmdMxNoDl4hiqVPU0FP5/mn2xGF7FgeKNW3+NiTouKuB -mZOeEl3f3uOze/suHPyfOu+49jk+TWWE05Xfqfowjf486nKPg6/uSA2izW/MwIKN -HuIuY3bBf+lx5yUVIraoZhH2MxODDQ== -=BZqz +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxKOFoACgkQTRfJlc2X +dfI/ZAf+NpqBT6ieOTMt+Us/mBou82EEVZ4tEn6W+v5+nRWFv6pDZQof/A2EU+Ti +QwUGc2CGPFhXwMxM5ASMnwJjYZ4fHZLUFTNrVXXV+lKXRXyT1jWH4D1iStCuvH/1 +l/fstEeH1csLH5VxBiMIfuiepowaqLmE6DDCjYdERfPoxS09WJftLzOJolDCpXmj +XX0/EvS9MWfFznBqE23qRjaRNH0A+SVPlLjVffmIwQb2NIonSYinTt/7DnF9kO3J +bSwVdEiLlTQyYgvblriWlsC/E2Tb0oY4exD3qE1WT6IgYaaMj75DMzqJHW1pw0RP +mhC5E2wFsfutkeBgNjFrXiB83X9rcw== +=W4KG -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ffa93262d..93bb05419 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0" +LE_AUTO_VERSION="0.30.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1232,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.0 \ - --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ - --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 -acme==0.30.0 \ - --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ - --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 -certbot-apache==0.30.0 \ - --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ - --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf -certbot-nginx==0.30.0 \ - --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ - --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 +certbot==0.30.1 \ + --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ + --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 +acme==0.30.1 \ + --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ + --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df +certbot-apache==0.30.1 \ + --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ + --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 +certbot-nginx==0.30.1 \ + --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ + --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index e4902c0ee7241e37491b8a941e25744b837cb7f8..4cd20c9c49f1f8a1af2f3d6989f45e30e2eb7e7d 100644 GIT binary patch literal 256 zcmV+b0ssCgN8mSVGVJnrTtAFfZ5;3U#TDzRV-D(ns8F8!*_~&O)rN#485^vFt-gP# zNhWC41jFGJ4H$K2>pcO}Emn;_!HqwTSx@4t0Sf}o3Xl{x7Q5-DyW@a_Sk~w}(raZr z@L67Bv_^Xiwr$n zMaABM0SHm(b<1CM+teljAD-FZpQisOilH%M{lTdDM?)#F;6dL9;8yuTS1HovImqWd GiZav&C4}Gr literal 256 zcmV+b0ssDv4s4Tx!Gpv36M!K{^ zade@s@$}@HWNXZloBGk`1V%UIbVc*QUSk|U~K*`EpZhpmOER`)Vh~vtkxoSl**~O#gMxciG8{xI~ zBjQkiW9*%R^8`&dakxvsB>SH8_YUxTWg^fH-(kERXdaGN%kJjgKStmdpmEs%4pGXQ zEz^kZX>73#HrgfHH^KzS6(C0T%A&Lv6bMbMN>!BFV}3U0NOGuT;#))Lwb_f~H7)4W GuwXBg?}bPJ diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 08d8d553f..58a539b89 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.30.0 \ - --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ - --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 -acme==0.30.0 \ - --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ - --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 -certbot-apache==0.30.0 \ - --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ - --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf -certbot-nginx==0.30.0 \ - --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ - --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 +certbot==0.30.1 \ + --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ + --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 +acme==0.30.1 \ + --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ + --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df +certbot-apache==0.30.1 \ + --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ + --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 +certbot-nginx==0.30.1 \ + --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ + --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 From f0f1a4838ed72c819f7c4ee17e347d7f5d5b528a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 14:13:07 -0800 Subject: [PATCH 532/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index acf1fb0eb..4de61860a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.31.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.31.1 - 2019-01-24 ### Fixed From 18281766dfc867e12e7a7d885b024cfa1933a7c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 14:13:08 -0800 Subject: [PATCH 533/639] Bump version to 0.31.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 3682b0174..77ff7bae8 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.1' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 83c0cd2fd..14d6cacb6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 4f20d5994..f519ed422 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4bc30c815..ff33293fe 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index ad9a70d66..5c8709445 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 503f71d7f..2f7fa37d6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 21097ff24..3bb43cf5d 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 366ffdbf1..599cec486 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 815a6d543..894d809ac 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 78fe30dc4..c99ad38aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index b26a79c29..588b6a40a 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0c8080c51..b09a09762 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index ddae991f4..e48428191 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 07cf7392c..8b6d73d33 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 22f5c0014..edf7b6ba6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 3cdf28f8f..69c2c7ed3 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 49749c883..05843d2ed 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6d8a28d3c..70e11f62b 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 6ff2db69a..bf68034c8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.1' +__version__ = '0.31.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 93bb05419..392fa738c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.1" +LE_AUTO_VERSION="0.31.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 1c2fc9af4531a0551b36957b2bb3e1fa9768ab5f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 15:19:36 -0800 Subject: [PATCH 534/639] Revert "pin back boulder" This reverts commit 5de41572e4f3563127d2c0caeb11d79b966c45ab. --- tests/boulder-fetch.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 74ba50e76..f34deb74e 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -7,10 +7,7 @@ set -xe export GOPATH=${GOPATH:-$HOME/gopath} BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder} if [ ! -d ${BOULDERPATH} ]; then - # Use the latest version of boulder that has TLS-SNI-01 enabled by default. - git clone https://github.com/letsencrypt/boulder ${BOULDERPATH} - cd "$BOULDERPATH" - git checkout 93ac7fbe9e77b190c5c6496b4e9d765775588963 + git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH} fi cd ${BOULDERPATH} From cc581387a9819deb4937977b36f1b61a93f5690c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Jan 2019 15:24:11 -0800 Subject: [PATCH 535/639] s/0.31.1/0.30.1 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acf1fb0eb..65227482b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.31.1 - 2019-01-24 +## 0.30.1 - 2019-01-24 ### Fixed From 01ed2409b9ef2499332a3d18dd00bb0555b060b0 Mon Sep 17 00:00:00 2001 From: Petr Messner Date: Fri, 25 Jan 2019 16:01:02 +0100 Subject: [PATCH 536/639] Fix code formatting in docs/using.txt --- docs/using.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/using.rst b/docs/using.rst index d77badd65..6fbfac04c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -259,6 +259,7 @@ and the nginx_ plugin for installation. (Note that this certificate cannot be renewed automatically.) :: + certbot run -a manual -i nginx -d example.com .. _third-party-plugins: From b7211c3f39a6e5486d83154978551e696f607bab Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Jan 2019 11:21:34 -0800 Subject: [PATCH 537/639] Update setuptools pinned in pipstrap (#6699) Fixes #6697. This PR updates the version of setuptools pinned in pipstrap which works around the problems we have seen on recent OSes. Why did I pick this version of setuptools? Because it's the latest and greatest, [supports all versions of Python that we do](https://github.com/pypa/setuptools/blob/v40.6.3/setup.py#L173), [has been out for a month and a half without the need for a point release](https://setuptools.readthedocs.io/en/latest/history.html), and has no non-optional dependencies. For the last point about dependencies, I don't think we have too much to worry about. setuptools did have a period between versions 34.0.0 and 36.0.0 where they tried to have normal dependencies on other packages, but reverted these changes. See their [changelog for 36.0.0](https://setuptools.readthedocs.io/en/latest/history.html#v36-0-0). You can also compare their [current setup.py file](https://github.com/pypa/setuptools/blob/v40.6.3/setup.py) to the [setup.py file for the currently pinned version](https://github.com/pypa/setuptools/blob/v29.0.1/setup.py) and you'll see [not much has changed](https://pastebin.com/nQj6d7D8). Not only that, but I have successfully tested these changes on: * ubuntu18.10 * ubuntu18.04LTS * ubuntu16.04LTS * ubuntu14.04LTS * ubuntu14.04LTS_32bit * debian9 * debian8.1 * amazonlinux-2015.09.1 * amazonlinux-2015.03.1 * RHEL7 * fedora23 * fedora29 * centos7 * centos6 * freebsd11 * macOS * Update setuptools to 40.6.3. * Build letsencrypt-auto. * update changelog * Don't use pipstrap in Dockerfile.centos6. --- CHANGELOG.md | 2 ++ letsencrypt-auto-source/Dockerfile.centos6 | 6 +++--- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/pipstrap.py | 6 +++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6085765c8..4f9c0782b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fixed accessing josepy contents through acme.jose when the full acme.jose path is used. * Clarify behavior for deleting certs as part of revocation. +* Update the version of setuptools pinned in certbot-auto to 40.6.3 to + solve installation problems on newer OSes. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 92fec168b..09aa52dcd 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -7,9 +7,9 @@ RUN yum install -y epel-release # Install pip and sudo: RUN yum install -y python-pip sudo -# Use pipstrap to update to a stable and tested version of pip -COPY ./pieces/pipstrap.py /opt -RUN /opt/pipstrap.py +# Update to a stable and tested version of pip. +# We do not use pipstrap here because it no longer supports Python 2.6. +RUN pip install pip==9.0.1 setuptools==29.0.1 wheel==0.29.0 # Pin pytest version for increased stability RUN pip install pytest==3.2.5 six==1.10.0 diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index cb748ccd3..d07af3d31 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1272,9 +1272,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 2b3025776..6a00dd9cb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -74,9 +74,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') From 53d13ff3a32aa448a8320ae464bedf4fb7b12c52 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Jan 2019 11:53:29 -0800 Subject: [PATCH 538/639] Update setuptools pinned in pipstrap (#6699) (#6704) Fixes #6697. This PR updates the version of setuptools pinned in pipstrap which works around the problems we have seen on recent OSes. Why did I pick this version of setuptools? Because it's the latest and greatest, [supports all versions of Python that we do](https://github.com/pypa/setuptools/blob/v40.6.3/setup.py#L173), [has been out for a month and a half without the need for a point release](https://setuptools.readthedocs.io/en/latest/history.html), and has no non-optional dependencies. For the last point about dependencies, I don't think we have too much to worry about. setuptools did have a period between versions 34.0.0 and 36.0.0 where they tried to have normal dependencies on other packages, but reverted these changes. See their [changelog for 36.0.0](https://setuptools.readthedocs.io/en/latest/history.html#v36-0-0). You can also compare their [current setup.py file](https://github.com/pypa/setuptools/blob/v40.6.3/setup.py) to the [setup.py file for the currently pinned version](https://github.com/pypa/setuptools/blob/v29.0.1/setup.py) and you'll see [not much has changed](https://pastebin.com/nQj6d7D8). Not only that, but I have successfully tested these changes on: * ubuntu18.10 * ubuntu18.04LTS * ubuntu16.04LTS * ubuntu14.04LTS * ubuntu14.04LTS_32bit * debian9 * debian8.1 * amazonlinux-2015.09.1 * amazonlinux-2015.03.1 * RHEL7 * fedora23 * fedora29 * centos7 * centos6 * freebsd11 * macOS * Update setuptools to 40.6.3. * Build letsencrypt-auto. * update changelog * Don't use pipstrap in Dockerfile.centos6. (cherry picked from commit b7211c3f39a6e5486d83154978551e696f607bab) --- CHANGELOG.md | 13 +++++++++++++ letsencrypt-auto-source/Dockerfile.centos6 | 6 +++--- letsencrypt-auto-source/letsencrypt-auto | 6 +++--- letsencrypt-auto-source/pieces/pipstrap.py | 6 +++--- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65227482b..d9fdc8e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.30.2 - master + +### Fixed + +* Update the version of setuptools pinned in certbot-auto to 40.6.3 to + solve installation problems on newer OSes. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, this +release only affects certbot-auto. + +More details about these changes can be found on our GitHub repo. + ## 0.30.1 - 2019-01-24 ### Fixed diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 92fec168b..09aa52dcd 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -7,9 +7,9 @@ RUN yum install -y epel-release # Install pip and sudo: RUN yum install -y python-pip sudo -# Use pipstrap to update to a stable and tested version of pip -COPY ./pieces/pipstrap.py /opt -RUN /opt/pipstrap.py +# Update to a stable and tested version of pip. +# We do not use pipstrap here because it no longer supports Python 2.6. +RUN pip install pip==9.0.1 setuptools==29.0.1 wheel==0.29.0 # Pin pytest version for increased stability RUN pip install pytest==3.2.5 six==1.10.0 diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 93bb05419..ca4ce53c6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1324,9 +1324,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 2b3025776..6a00dd9cb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -74,9 +74,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') From 33ea6c5d988a43b1c4b37a4192011fb672503365 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 25 Jan 2019 12:15:41 -0800 Subject: [PATCH 539/639] Update changelog for 0.30.2 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9fdc8e2e..2887dd7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.30.2 - master +## 0.30.2 - 2019-01-25 ### Fixed From 6cba691c1954137e1baaef3c901a6ce5d9b6b2ee Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 25 Jan 2019 12:36:19 -0800 Subject: [PATCH 540/639] Release 0.30.2 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 32 +++++++++---------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 15 +++++---- letsencrypt-auto | 32 +++++++++---------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 +++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | 5 +-- .../pieces/certbot-requirements.txt | 24 +++++++------- 26 files changed, 93 insertions(+), 95 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 3682b0174..301f76f02 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.1' +version = '0.30.2' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 83c0cd2fd..b3ad21aae 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 93bb05419..a79a9c5ae 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.1" +LE_AUTO_VERSION="0.30.2" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1232,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.1 \ - --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ - --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 -acme==0.30.1 \ - --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ - --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df -certbot-apache==0.30.1 \ - --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ - --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 -certbot-nginx==0.30.1 \ - --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ - --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 +certbot==0.30.2 \ + --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ + --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e +acme==0.30.2 \ + --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ + --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 +certbot-apache==0.30.2 \ + --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ + --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 +certbot-nginx==0.30.2 \ + --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ + --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1324,9 +1324,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 4f20d5994..88d82dbe3 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4bc30c815..4219e163f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index ad9a70d66..904501978 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 503f71d7f..e88f0554f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 21097ff24..bb6d568b8 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 366ffdbf1..aba5ccdbf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 815a6d543..a6c7c8d6c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 78fe30dc4..f24d44e93 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index b26a79c29..5882b5458 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0c8080c51..acea3f9d2 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index ddae991f4..6829f5504 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 07cf7392c..7a9836d01 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 22f5c0014..51e9194f0 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 3cdf28f8f..976ce47fa 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 49749c883..9d21ac2d4 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6d8a28d3c..eb8a65973 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.1' +version = '0.30.2' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 6ff2db69a..75fd4a244 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.1' +__version__ = '0.30.2' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 77c6dd230..cd6d431b3 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.30.1 + "". (default: CertbotACMEClient/0.30.2 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -479,9 +479,10 @@ apache: Apache Web Server plugin --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: None) + Path to the Apache 'a2enmod' binary (default: a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: None) + Path to the Apache 'a2dismod' binary (default: + a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -495,16 +496,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: False) + (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: False) + Ubuntu/Debian currently) (default: True) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apachectl) + apache2ctl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index 93bb05419..a79a9c5ae 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.1" +LE_AUTO_VERSION="0.30.2" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1232,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.1 \ - --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ - --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 -acme==0.30.1 \ - --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ - --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df -certbot-apache==0.30.1 \ - --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ - --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 -certbot-nginx==0.30.1 \ - --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ - --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 +certbot==0.30.2 \ + --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ + --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e +acme==0.30.2 \ + --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ + --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 +certbot-apache==0.30.2 \ + --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ + --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 +certbot-nginx==0.30.2 \ + --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ + --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1324,9 +1324,9 @@ PACKAGES = maybe_argparse + [ 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' - 'setuptools-29.0.1.tar.gz', - 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' + 'setuptools-40.6.3.zip', + '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index a5920c7e5..a60ccd8bb 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxKOFoACgkQTRfJlc2X -dfI/ZAf+NpqBT6ieOTMt+Us/mBou82EEVZ4tEn6W+v5+nRWFv6pDZQof/A2EU+Ti -QwUGc2CGPFhXwMxM5ASMnwJjYZ4fHZLUFTNrVXXV+lKXRXyT1jWH4D1iStCuvH/1 -l/fstEeH1csLH5VxBiMIfuiepowaqLmE6DDCjYdERfPoxS09WJftLzOJolDCpXmj -XX0/EvS9MWfFznBqE23qRjaRNH0A+SVPlLjVffmIwQb2NIonSYinTt/7DnF9kO3J -bSwVdEiLlTQyYgvblriWlsC/E2Tb0oY4exD3qE1WT6IgYaaMj75DMzqJHW1pw0RP -mhC5E2wFsfutkeBgNjFrXiB83X9rcw== -=W4KG +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxLcw8ACgkQTRfJlc2X +dfK35gf+PoxtrJJIjvybNqd3lb8HOg2ntIVmXcYJGuuUo6m09fzai+XI6cOm5Dpu +l2D5OrbLqmez8tYkCkEWHV0OfwyVWw+m8T3sXlcrv14eA1RfgMnZ+cmmlpDskzHU +EOtaXo1/IkLDwBRrsl8IUbwD2XxbjuLsA2Sevoa59NlfTXJUApfAzohl3epRiJjB +gugdqcsfjRRAqQqOz+iJCKBCWSTIrr/g6Y9aZu9V93t/WDSLRFjehxO1GQrLnCnX +17JGlr0/AXd67jOKS1OWmORPPAFfLIXezUMtgrz5hE7T5UviaUu9ySV8UCxq1N79 +cfSBb/HIUxZ0wf1CkTUMRFQpA7cGtw== +=cNcT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ca4ce53c6..a79a9c5ae 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.1" +LE_AUTO_VERSION="0.30.2" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1232,18 +1232,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.1 \ - --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ - --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 -acme==0.30.1 \ - --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ - --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df -certbot-apache==0.30.1 \ - --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ - --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 -certbot-nginx==0.30.1 \ - --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ - --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 +certbot==0.30.2 \ + --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ + --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e +acme==0.30.2 \ + --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ + --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 +certbot-apache==0.30.2 \ + --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ + --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 +certbot-nginx==0.30.2 \ + --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ + --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4cd20c9c4..f5175187a 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,4 +1 @@ -*Gà7j2ìòx\?ŒVmïùÅë©cê¨PžûÙgŽÕ†„#¬ƒ­¾¨I&hÖÃá ugë=Ó-V>Á?ŽYOâ« Î -7»é¦»ã€„XÖè;Òke<åkYùÚ¬£ðªëÂ= -žkñ©ŸŠ&eetþh¸nÃ:ö¾éß”BG™Ó& -`Í`ÃîôÜÔ”…£÷wášhèašóÚÙópk›§Ítg¦þ̨.ù´mì.¿Ö$X¥þ‹ =\EÅÞQèuË_uÛÔ&žÙàŸ¦ÿ'Š¡1cýÁ¨ùGC)°àAßàVùAW)Òå9Èç=Š2Ô \ No newline at end of file +^)ø‘Øìëk–ŸAŠT˜¦{àÂç`$œ9º1`tºp ž^TŽÌ%nl/cãònããjdr=ŸVê+æ~\¯ú¬ñ ²Þ¡¿äåQÉ[‚ø t”ÂY¯‘ni5Á[âéÊN ZAª&K(¬Ú´žd¤]LÌè£|?·»¦;U‡féR¸Ä¨Îx–áK+ntó;ôekãni¯‹„Œ£’,ÜÖ6½RëOÂáŤP ìV|Û;òl,±ðWî‚]zãùªÄÕºi°LgC·ZëQDæ3 ÔŸ*ULþf_O?¯_¿!œÂȵê6ó»y1o×+éÎp¿ \ No newline at end of file diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 58a539b89..80249cd9a 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.30.1 \ - --hash=sha256:debdf5cfe17d55c124508a5f720e80094b82e87f167fb45cb6fda41482b6bd76 \ - --hash=sha256:39d1a58d3b59c68e57e26bcb1b73078fc227c8944388e61646dd6bbb203618e7 -acme==0.30.1 \ - --hash=sha256:b75705eba723a6082c7bc8e41853407cb1520ce64d11359ba09b269db9356aed \ - --hash=sha256:dc9b99bdbe1582993c96a8145354a61e79470be382d4d875990dd7d41727d3df -certbot-apache==0.30.1 \ - --hash=sha256:18dae95e4785e2acdbe0b8d00db9e83b890ca8145f39092911f8ea9e1b0169b3 \ - --hash=sha256:224cf83aeb85f7ca12ad08ddce020ff04576a692305f12ab1d1e7f21ee527180 -certbot-nginx==0.30.1 \ - --hash=sha256:2360e1e0e5b61c4078eced7889b90b85dc00dbc280461bf4e8fc4c3280bb3e52 \ - --hash=sha256:4e15deba3542fcf5b98d021acfdfdea2ed93db0159824271a49694cc357de7f4 +certbot==0.30.2 \ + --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ + --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e +acme==0.30.2 \ + --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ + --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 +certbot-apache==0.30.2 \ + --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ + --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 +certbot-nginx==0.30.2 \ + --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ + --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 From 17f322d51f7c86acd1f5b64054b0c176d572a80d Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 25 Jan 2019 12:36:28 -0800 Subject: [PATCH 541/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2887dd7b1..f574cc45f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.31.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.30.2 - 2019-01-25 ### Fixed From b326adc2be4d58841ab4d13536e5f0f29dfe154a Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 25 Jan 2019 12:36:28 -0800 Subject: [PATCH 542/639] Bump version to 0.31.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 301f76f02..77ff7bae8 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.2' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b3ad21aae..14d6cacb6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 88d82dbe3..f519ed422 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4219e163f..ff33293fe 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 904501978..5c8709445 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index e88f0554f..2f7fa37d6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index bb6d568b8..3bb43cf5d 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index aba5ccdbf..599cec486 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index a6c7c8d6c..894d809ac 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index f24d44e93..c99ad38aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 5882b5458..588b6a40a 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index acea3f9d2..b09a09762 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 6829f5504..e48428191 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 7a9836d01..8b6d73d33 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 51e9194f0..edf7b6ba6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 976ce47fa..69c2c7ed3 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 9d21ac2d4..05843d2ed 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index eb8a65973..70e11f62b 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.2' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 75fd4a244..bf68034c8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.2' +__version__ = '0.31.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index a79a9c5ae..01536dea3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.2" +LE_AUTO_VERSION="0.31.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 8e5b2ac5b5e5f674fb64b1bc9ba52dc0d2f95e1c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 25 Jan 2019 14:24:11 -0800 Subject: [PATCH 543/639] Stop multitester.py from eating errors. (#6705) --- tests/letstest/multitester.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 320328d9e..8babc67b3 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -32,7 +32,7 @@ see: from __future__ import print_function from __future__ import with_statement -import sys, os, time, argparse, socket +import sys, os, time, argparse, socket, traceback import multiprocessing as mp from multiprocessing import Manager import urllib2 @@ -363,6 +363,7 @@ def test_client_process(inqueue, outqueue): except: outqueue.put((ii, target, 'fail')) print("%s - %s FAIL"%(target['ami'], target['name'])) + traceback.print_exc(file=sys.stdout) pass # append server certbot.log to each per-machine output log @@ -371,6 +372,7 @@ def test_client_process(inqueue, outqueue): execute(grab_certbot_log) except: print("log fail\n") + traceback.print_exc(file=sys.stdout) pass @@ -426,6 +428,7 @@ try: execute(local_git_clone, cl_args.repo) except FabricException: print("FAIL: trouble with git repo") + traceback.print_exc() exit() From 6ddb4e2999eca7ad6182f78e2159dad317a7cd89 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Jan 2019 01:08:29 -0800 Subject: [PATCH 544/639] No numprocesses in pytest.ini part 2 (#6715) * Remove --numprocesses from pytest.ini. * Add --numprocesses to PYTEST_ADDOPTS in tox.ini. * complexity-- --- pytest.ini | 1 - tox.ini | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 8f009045c..654b368f2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,6 @@ # settings we want to also change there must be added to the release script # directly. [pytest] -addopts = --numprocesses auto --pyargs # In general, all warnings are treated as errors. Here are the exceptions: # 1- decodestring: https://github.com/rthalley/dnspython/issues/338 # 2- ignore our own TLS-SNI-01 warning diff --git a/tox.ini b/tox.ini index 363f06bf2..76de2ea36 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,7 @@ commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py setenv = + PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto --pyargs} PYTHONHASHSEED = 0 [testenv:py27-oldest] From 3bb7dd8faf6858c29056e88e8e4cd858909859b7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 29 Jan 2019 16:12:32 -0800 Subject: [PATCH 545/639] Update test farm targets (#6700) Fixes #6106. AMIs were taken from https://wiki.debian.org/Cloud/AmazonEC2Image/Stretch and https://cloud-images.ubuntu.com/locator/ec2/. I didn't update the AMI for Fedora due to #6698. These new AMIs pass on all test farm tests we run during the release process except Ubuntu 18.04 and 18.10 fail on test_apache2.sh. This is tracked at #6706. If this PR lands before this issue is resolved, we should list these systems as expected failures in the release notes. Adding these AMIs slows down our tests significantly. I didn't measure it, but it feels 50-100% slower at least on my setup. I think it's worth it though. * Update test farm targets. * use different ubuntu ami * Fix test_leauto_upgrades.sh on newer OSes. --- .../letstest/scripts/test_leauto_upgrades.sh | 18 ++++++++++------- tests/letstest/targets.yaml | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 355fead2e..0c2b374f2 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -15,11 +15,15 @@ if ! command -v git ; then exit 1 fi fi -# 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's -# the first version that pins package versions, properly supports -# --no-self-upgrade, and works with newer versions of pip. -git checkout -f v0.5.0 letsencrypt-auto -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then +# 0.17.0 is the oldest version of letsencrypt-auto that has precompiled +# cryptography and the tagged commit is in master. 0.16.0 was the first version +# to use precompiled cryptography, but the release PR was squashed losing the +# commit. We want to use a precompiled version of cryptography for stability. +# Previous versions that have to compile against OpenSSL on installation +# started failing on newer distros with newer versions of OpenSSL. +INITIAL_VERSION="0.17.0" +git checkout -f "v$INITIAL_VERSION" letsencrypt-auto +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep "$INITIAL_VERSION" ; then echo initial installation appeared to fail exit 1 fi @@ -71,7 +75,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; exit 1 fi cp letsencrypt-auto cb-auto - if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then + if ! ./cb-auto -v --debug --version 2>&1 | grep "$INITIAL_VERSION" ; then echo "Certbot shouldn't have updated to a new version!" exit 1 fi @@ -81,7 +85,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; fi # Create a 2nd venv at the new path to ensure we properly handle this case export VENV_PATH="/opt/eff.org/certbot/venv" - if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then + if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep "$INITIAL_VERSION" ; then echo second installation appeared to fail exit 1 fi diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 57ce4811a..c1a28af98 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,6 +1,21 @@ targets: #----------------------------------------------------------------------------- #Ubuntu + - ami: ami-064bd2d44a1d6c097 + name: ubuntu18.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-012fd5eb46f56731f + name: ubuntu18.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-09677e0a6b14905b0 + name: ubuntu16.04LTS + type: ubuntu + virt: hvm + user: ubuntu - ami: ami-7b89cc11 name: ubuntu14.04LTS type: ubuntu @@ -13,6 +28,11 @@ targets: user: ubuntu #----------------------------------------------------------------------------- # Debian + - ami: ami-003f19e0e687de1cd + name: debian9 + type: ubuntu + virt: hvm + user: admin - ami: ami-116d857a name: debian8.1 type: ubuntu From d4362594378ac4e519bb81c5a9a0fe803098fd7f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 30 Jan 2019 04:23:08 +0100 Subject: [PATCH 546/639] Forcibly reactivate tls-sni-01 challenges until complete removal. (#6683) This PR reactivates tls-sni-01 challenges on recent Boulder versions checkout for integration tests. This allows to continue testing this challenge until it is officially dropped from server (Boulder) and client (Certbot). Reverts #6679. --- certbot-nginx/tests/boulder-integration.sh | 4 +++- tests/boulder-fetch.sh | 6 ++++++ tests/certbot-boulder-integration.sh | 16 ++++++++-------- tests/integration/_common.sh | 6 +++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 547502ccf..03425734d 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,6 +39,8 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf +certbot_test_nginx --domains nginx-tls.wtf run --preferred-challenges tls-sni +test_deployment_and_rollback nginx-tls.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 @@ -64,4 +66,4 @@ test_deployment_and_rollback nginx6.wtf # top nginx -c $nginx_root/nginx.conf -s stop -coverage report --fail-under 72 --include 'certbot-nginx/*' --show-missing +coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index f34deb74e..a06d37325 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -12,6 +12,12 @@ fi cd ${BOULDERPATH} +# Since https://github.com/letsencrypt/boulder/commit/92e8e1708a725e9d08a5da2f4a7132320ed2158b, +# Boulder support for tls-sni-01 challenges is disabled. We still need to support it until this +# challenge is officially removed from ACME CA server on production, and also removed from Certbot. +# This sed command reactivate tls-sni-01 challenges inplace temporarily. +sed -i 's/tls-alpn-01/tls-sni-01/g' test/config/ra.json + docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index c03b3def8..630571148 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -221,20 +221,20 @@ common plugins --init --prepare | grep webroot # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $tls_alpn_01_port & +python ./tests/run_http_server.py $http_01_port & python_server_pid=$! + certname="le1.wtf" -common --domains le1.wtf --preferred-challenges http-01 auth \ +common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ --cert-name $certname \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid CheckDeployHook $certname -# Previous test used to be a tls-sni-01 challenge that is not supported anymore. -# Now it is a http-01 challenge and this makes it a duplicate of the following test. -# But removing it would break many tests here, as they are strongly coupled. -# See https://github.com/certbot/certbot/pull/6679 +python ./tests/run_http_server.py $tls_sni_01_port & +python_server_pid=$! certname="le2.wtf" common --domains le2.wtf --preferred-challenges http-01 run \ --cert-name $certname \ @@ -254,7 +254,7 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ CheckRenewHook $certname certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns run \ +common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ @@ -396,7 +396,7 @@ CheckDirHooks 1 # with fail. common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ --allow-subset-of-names \ - --preferred-challenges dns \ + --preferred-challenges dns,tls-sni \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 76b990ac4..83aa91a9e 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,10 +3,10 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -tls_alpn_01_port=5001 +tls_sni_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir tls_alpn_01_port http_01_port sources +export root config_dir tls_sni_01_port http_01_port sources certbot_path="$(command -v certbot)" # Flags that are added here will be added to Certbot calls within # certbot_test_no_force_renew. @@ -60,7 +60,7 @@ certbot_test_no_force_renew () { "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ - --tls-sni-01-port $tls_alpn_01_port \ + --tls-sni-01-port $tls_sni_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ $other_flags \ From ca25d1b66a24e351739a6b4dbfadd8f471857721 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 30 Jan 2019 04:25:05 +0100 Subject: [PATCH 547/639] Call atexit handlers before test tearDown to remove errors on Windows (#6667) When certbot is executing, several resources are opened. It is typically file handles and locks on them. Of course, theses resources need to be cleanup. It is done in Certbot by registering cleanup functions through atexit module, that ensures theses functions will be called when Certbot exit. This allow to not care about resource cleanup everywhere in the code, as it is processed globally. The problem with atexit is it cleanup functions are called when the Python program exit. If the program is Certbot itself when used, this is Pytest in unit test execution. So during a unit test execution, cleanup is not called after a test and before its tearDown, but when Pytest exit, so way after tests and their respective tearDown. But many tearDown implies to delete folders where this kind of resources are hold. This is never a problem on Linux, thanks to its non-blocking file handling. It is usually not a problem on Windows, despite its blocking approach. But if the tearDown requires folder cleanup, exceptions are raised, and currently hidden as warnings. There is currently 504 exceptions of this type in Certbot core tests on Windows. This PR starts to correct this situation. To do so, some of the functions cleanup normally called through atexit, are explicitly called as part of the tearDown process of relevant test classes, before directory removal is done. Theses situations come all from the certbot.tests.util.TempDirTestCase, so the code is in this specific tearDown process. As a consequence, exceptions drop from 504 to 64. Then there are still a significant part of them, that will be handled in later mitigation. * Call atexit handlers before test tearDown to reduce errors on Windows * Clear locks dict after global releasing execution * Remove last tearDown errors. * Clean out mock on open. * Remove a test * Reenable some tests --- certbot/tests/lock_test.py | 7 +++++-- certbot/tests/util.py | 31 ++++++++++++++++--------------- certbot/tests/util_test.py | 24 +++++++++--------------- certbot/util.py | 6 +++--- 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa82701f3..2ade47827 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -10,7 +10,6 @@ from certbot import errors from certbot.tests import util as test_util -@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -25,7 +24,6 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) -@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod @@ -37,6 +35,7 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') + @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -54,6 +53,7 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) + @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) locked_repr = repr(lock_file) @@ -71,6 +71,7 @@ class LockFileTest(test_util.TempDirTestCase): self.assertTrue(lock_file.__class__.__name__ in lock_repr) self.assertTrue(self.lock_path in lock_repr) + @test_util.broken_on_windows def test_race(self): should_delete = [True, False] stat = os.stat @@ -86,11 +87,13 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) + @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) + @test_util.broken_on_windows @mock.patch('certbot.compat.fcntl.lockf') def test_unexpected_lockf_err(self, mock_lockf): msg = 'hi there' diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8c5db2c2f..38a9075c1 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,14 +3,15 @@ .. warning:: This module is not part of the public API. """ +import logging import multiprocessing import os import pkg_resources import shutil +import stat import tempfile import unittest import sys -import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -328,22 +329,22 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # On Windows we have various files which are not correctly closed at the time of tearDown. - # For know, we log them until a proper file close handling is written. - # Useful for development only, so no warning when we are on a CI process. - def onerror_handler(_, path, excinfo): - """On error handler""" - if not os.environ.get('APPVEYOR'): # pragma: no cover - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) - shutil.rmtree(self.tempdir, onerror=onerror_handler) + # Cleanup opened resources after a test. This is usually done through atexit handlers in + # Certbot, but during tests, atexit will not run registered functions before tearDown is + # called and instead will run them right before the entire test process exits. + # It is a problem on Windows, that does not accept to clean resources before closing them. + logging.shutdown() + util._release_locks() # pylint: disable=protected-access + + def handle_rw_files(_, path, __): + """Handle read-only files, that will fail to be removed on Windows.""" + os.chmod(path, stat.S_IWRITE) + os.remove(path) + shutil.rmtree(self.tempdir, onerror=handle_rw_files) + class ConfigTestCase(TempDirTestCase): - """Test class which sets up a NamespaceConfig object. - - """ + """Test class which sets up a NamespaceConfig object.""" def setUp(self): super(ConfigTestCase, self).setUp() self.config = configuration.NamespaceConfig( diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6685b88c6..81d0629c8 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -191,7 +191,12 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_wrong_mode(self): os.chmod(self.tempdir, 0o400) - self.assertFalse(self._call(0o600)) + try: + self.assertFalse(self._call(0o600)) + finally: + # Without proper write permissions, Windows is unable to delete a folder, + # even with admin permissions. Write access must be explicitly set first. + os.chmod(self.tempdir, 0o700) class UniqueFileTest(test_util.TempDirTestCase): @@ -277,20 +282,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for f, _ in items: f.close() - @mock.patch("certbot.util.os.fdopen") - def test_failure(self, mock_fdopen): - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") - - @mock.patch("certbot.util.os.fdopen") - def test_subsequent_failure(self, mock_fdopen): - self._call("wow") - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") + def test_failure(self): + with mock.patch("certbot.util.os.open", side_effect=OSError(errno.EIO)): + self.assertRaises(OSError, self._call, "wow") class SafelyRemoveTest(test_util.TempDirTestCase): diff --git a/certbot/util.py b/certbot/util.py index d7c542465..416075ce8 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -142,6 +142,7 @@ def _release_locks(): except: # pylint: disable=bare-except msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) logger.debug(msg, exc_info=True) + _LOCKS.clear() def set_up_core_dir(directory, mode, uid, strict): @@ -225,9 +226,8 @@ def safe_open(path, mode="w", chmod=None, buffering=None): fdopen_args = () # type: Union[Tuple[()], Tuple[int]] if buffering is not None: fdopen_args = (buffering,) - return os.fdopen( - os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), - mode, *fdopen_args) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) + return os.fdopen(fd, mode, *fdopen_args) def _unique_file(path, filename_pat, count, chmod, mode): From 4237d4a3adbc1dd9e7dfca8f669ad73dee62e273 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 30 Jan 2019 13:59:07 -0800 Subject: [PATCH 548/639] Ignore color_scheme warning from IPython. (#6714) This PR in combination with #6713 resolves issues with using ipdb with pytest. --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index 654b368f2..49db7da09 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,8 +8,11 @@ # 3- ignore warn for importing abstract classes from collections instead of collections.abc, # too much third party dependencies are still relying on this behavior, # but it should be corrected to allow Certbot compatiblity with Python >= 3.8 +# 4- ipdb uses deprecated functionality of IPython. See +# https://github.com/gotcha/ipdb/issues/144. filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning ignore:.*collections\.abc:DeprecationWarning + ignore:The `color_scheme` argument is deprecated:DeprecationWarning:IPython.* From 0484b1554d31db9fd284dcf0234d761bb5ebe4fd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 31 Jan 2019 12:57:49 -0800 Subject: [PATCH 549/639] Set --pyargs directly in the files where it is needed. (#6727) It was pointed out to me that you can no longer run tox.cover.py directly to run coverage tests on a subset of the packages in this repo. This happened after we did both of: 1. Factored out --pyargs from the different test files and put it in pytest.ini. 2. Moved the options we added to pytest.ini to tox.ini meaning that --pyargs is not set unless you run the file through tox. I think the fact that we factored out --pyargs from the files that needed it was a mistake. --pytest is needed by tox.cover.py and install_and_test.py in order to work correctly. I think CLI options like this which are needed for the file to function should be left in the file directly. Doing anything else in my opinion unnecessarily couples these scripts to other files making them more brittle and harder to maintain. With that said, I also think CLI options which are not needed (such as --numprocesses) can be left to be optionally added through PYTEST_ADDOPTS. * Add --pyargs to tox.cover.py. * Add --pyargs to install_and_test.py. * Remove --pyargs from tox.ini. --- tools/install_and_test.py | 2 +- tox.cover.py | 4 ++-- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0142eeea4..b15c8eca5 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -49,7 +49,7 @@ def main(args): shutil.copy2("pytest.ini", temp_cwd) try: call_with_print(' '.join([ - sys.executable, '-m', 'pytest', pkg.replace('-', '_')]), cwd=temp_cwd) + sys.executable, '-m', 'pytest', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) finally: shutil.rmtree(temp_cwd) diff --git a/tox.cover.py b/tox.cover.py index c8a45d82d..008424641 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -48,8 +48,8 @@ def cover(package): .format(pkg_dir))) return - subprocess.check_call([ - sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', package]) + subprocess.check_call([sys.executable, '-m', 'pytest', '--pyargs', + '--cov', pkg_dir, '--cov-append', '--cov-report=', package]) subprocess.check_call([ sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) diff --git a/tox.ini b/tox.ini index 76de2ea36..b66e330da 100644 --- a/tox.ini +++ b/tox.ini @@ -70,7 +70,7 @@ commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py setenv = - PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto --pyargs} + PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto} PYTHONHASHSEED = 0 [testenv:py27-oldest] From 8f7b2801061bb36a9b16b4a82193af700574182d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 31 Jan 2019 23:53:32 +0100 Subject: [PATCH 550/639] [Windows] Fix account paths on Windows when colons are involved (#6711) The account path used to store user credentials is calculated from the domain used to contact the relevant ACME CA server. For instance, if the directory URL is https://my.domain.net/directory, then the account path will be $CONFIG_DIR/accounts/my.domain.net. However, if non standard HTTP/HTTPS port need to be used, colons will be involved. For instance, https://my.domain.net:14000/directory will give $CONFIG_DIR/accounts/my.domain.net:14000. Colons in paths are supported on POSIX systems, but not on Windows (it is reserved for the root drive letter). This PR replaces colons by underscores for account paths on Windows, and leaves them untouched on Linux. * Fix account path on Windows when colons are involved * Protect colon in drive letter * Refactor compat platform specific logic --- certbot/account.py | 2 +- certbot/compat.py | 38 +++++++++++++++++++++++------ certbot/configuration.py | 2 ++ certbot/tests/account_test.py | 4 ++- certbot/tests/configuration_test.py | 5 +++- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index 76135c3aa..0c653f6dd 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -142,7 +142,7 @@ class AccountFileStorage(interfaces.AccountStorage): def __init__(self, config): self.config = config util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(), - self.config.strict_permissions) + self.config.strict_permissions) def _account_dir_path(self, account_id): return self._account_dir_path_for_server_path(account_id, self.config.server_path) diff --git a/certbot/compat.py b/certbot/compat.py index 7d936aa9d..3b5d068a6 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -1,12 +1,8 @@ """ Compatibility layer to run certbot both on Linux and Windows. -The approach used here is similar to Modernizr for Web browsers. -We do not check the platform type to determine if a particular logic is supported. -Instead, we apply a logic, and then fallback to another logic if first logic -is not supported at runtime. - -Then logic chains are abstracted into single functions to be exposed to certbot. +This module contains all required platform specific code, +allowing the rest of Certbot codebase to be platform agnostic. """ import os import select @@ -27,6 +23,8 @@ except ImportError: UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ 'certificates', 'enhance', 'revoke', 'delete', 'register', 'unregister', 'config_changes', 'plugins'] + + def raise_for_non_administrative_windows_rights(subcommand): """ On Windows, raise if current shell does not have the administrative rights. @@ -50,6 +48,7 @@ def raise_for_non_administrative_windows_rights(subcommand): 'Error, "{0}" subcommand must be run on a shell with administrative rights.' .format(subcommand)) + def os_geteuid(): """ Get current user uid @@ -65,6 +64,7 @@ def os_geteuid(): # Windows specific return 0 + def os_rename(src, dst): """ Rename a file to a destination path and handles situations where the destination exists. @@ -117,6 +117,7 @@ def readline_with_timeout(timeout, prompt): # So no timeout on Windows for now. return sys.stdin.readline() + def lock_file(fd): """ Lock the file linked to the specified file descriptor. @@ -131,6 +132,7 @@ def lock_file(fd): # Windows specific msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + def release_locked_file(fd, path): """ Remove, close, and release a lock file specified by its file descriptor and its path. @@ -164,15 +166,17 @@ def release_locked_file(fd, path): finally: os.close(fd) + def compare_file_modes(mode1, mode2): """Return true if the two modes can be considered as equals for this platform""" - if 'fcntl' in sys.modules: + if os.name != 'nt': # Linux specific: standard compare return oct(stat.S_IMODE(mode1)) == oct(stat.S_IMODE(mode2)) # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights. return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE) + WINDOWS_DEFAULT_FOLDERS = { 'config': 'C:\\Certbot', 'work': 'C:\\Certbot\\lib', @@ -184,6 +188,7 @@ LINUX_DEFAULT_FOLDERS = { 'logs': '/var/log/letsencrypt', } + def get_default_folder(folder_type): """ Return the relevant default folder for the current OS @@ -194,8 +199,25 @@ def get_default_folder(folder_type): :rtype: str """ - if 'fcntl' in sys.modules: + if os.name != 'nt': # Linux specific return LINUX_DEFAULT_FOLDERS[folder_type] # Windows specific return WINDOWS_DEFAULT_FOLDERS[folder_type] + + +def underscores_for_unsupported_characters_in_path(path): + # type: (str) -> str + """ + Replace unsupported characters in path for current OS by underscores. + :param str path: the path to normalize + :return: the normalized path + :rtype: str + """ + if os.name != 'nt': + # Linux specific + return path + + # Windows specific + drive, tail = os.path.splitdrive(path) + return drive + tail.replace(':', '_') diff --git a/certbot/configuration.py b/certbot/configuration.py index daf514be8..15f9fa3e0 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -5,6 +5,7 @@ import os from six.moves.urllib import parse # pylint: disable=import-error import zope.interface +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -69,6 +70,7 @@ class NamespaceConfig(object): def accounts_dir_for_server_path(self, server_path): """Path to accounts directory based on server_path""" + server_path = compat.underscores_for_unsupported_characters_in_path(server_path) return os.path.join( self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 278b0c545..b062f437b 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -12,6 +12,7 @@ import pytz from acme import messages +from certbot import compat from certbot import errors import certbot.tests.util as test_util @@ -114,7 +115,8 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.mock_client.directory.new_authz = new_authzr_uri def test_init_creates_dir(self): - self.assertTrue(os.path.isdir(self.config.accounts_dir)) + self.assertTrue(os.path.isdir( + compat.underscores_for_unsupported_characters_in_path(self.config.accounts_dir))) @test_util.broken_on_windows def test_save_and_restore(self): diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index eb8bcff89..10d9059b3 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -4,6 +4,7 @@ import unittest import mock +from certbot import compat from certbot import constants from certbot import errors @@ -47,9 +48,11 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_constants.KEY_DIR = 'keys' mock_constants.TEMP_CHECKPOINT_DIR = 't' + ref_path = compat.underscores_for_unsupported_characters_in_path( + 'acc/acme-server.org:443/new') self.assertEqual( os.path.normpath(self.config.accounts_dir), - os.path.normpath(os.path.join(self.config.config_dir, 'acc/acme-server.org:443/new'))) + os.path.normpath(os.path.join(self.config.config_dir, ref_path))) self.assertEqual( os.path.normpath(self.config.backup_dir), os.path.normpath(os.path.join(self.config.work_dir, 'backups'))) From a0d47a44c9e8bbeae053eac062e476088a7b872c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Feb 2019 12:16:18 -0800 Subject: [PATCH 551/639] Remove spdy cruft (#6573) --- certbot/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot/constants.py b/certbot/constants.py index 7d0e0c879..c6a80747e 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -146,7 +146,7 @@ RENEWER_DEFAULTS = dict( """Defaults for renewer script.""" -ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling", "spdy"] +ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] """List of possible :class:`certbot.interfaces.IInstaller` enhancements. @@ -154,7 +154,6 @@ List of expected options parameters: - redirect: None - ensure-http-header: name of header (i.e. Strict-Transport-Security) - ocsp-stapling: certificate chain file path -- spdy: TODO """ From bb8222200a8cbd39a3ce9584ce6dfed6c5d05228 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Feb 2019 12:22:11 -0800 Subject: [PATCH 552/639] Remove IValidator (#6572) * Remove unneeded validator usage. * Remove IValidator --- .../certbot_compatibility_test/validator.py | 3 -- certbot/interfaces.py | 50 ------------------- 2 files changed, 53 deletions(-) diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index fd2f95702..3455ce82d 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -2,20 +2,17 @@ import logging import socket import requests -import zope.interface import six from six.moves import xrange # pylint: disable=import-error,redefined-builtin from acme import crypto_util from acme import errors as acme_errors -from certbot import interfaces logger = logging.getLogger(__name__) -@zope.interface.implementer(interfaces.IValidator) class Validator(object): # pylint: disable=no-self-use """Collection of functions to test a live webserver's configuration""" diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 2e837d1d2..bb9a91b0f 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -522,56 +522,6 @@ class IDisplay(zope.interface.Interface): """ -class IValidator(zope.interface.Interface): - """Configuration validator.""" - - def certificate(cert, name, alt_host=None, port=443): - """Verifies the certificate presented at name is cert - - :param OpenSSL.crypto.X509 cert: Expected certificate - :param str name: Server's domain name - :param bytes alt_host: Host to connect to instead of the IP - address of host - :param int port: Port to connect to - - :returns: True if the certificate was verified successfully - :rtype: bool - - """ - - def redirect(name, port=80, headers=None): - """Verify redirect to HTTPS - - :param str name: Server's domain name - :param int port: Port to connect to - :param dict headers: HTTP headers to include in request - - :returns: True if redirect is successfully enabled - :rtype: bool - - """ - - def hsts(name): - """Verify HSTS header is enabled - - :param str name: Server's domain name - - :returns: True if HSTS header is successfully enabled - :rtype: bool - - """ - - def ocsp_stapling(name): - """Verify ocsp stapling for domain - - :param str name: Server's domain name - - :returns: True if ocsp stapling is successfully enabled - :rtype: bool - - """ - - class IReporter(zope.interface.Interface): """Interface to collect and display information to the user.""" From f547521a5b65d2de2328e0f90418e802aeeb8366 Mon Sep 17 00:00:00 2001 From: Samuel Shifterovich Date: Sat, 2 Feb 2019 18:56:38 +0100 Subject: [PATCH 553/639] /var/logs/ -> /var/log/ (#6732) --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 6fbfac04c..531c84e85 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -903,7 +903,7 @@ that by default two instances of Certbot will not be able to run in parallel. Since the directories used by Certbot are configurable, Certbot will write a lock file for all of the directories it uses. This include Certbot's ``--work-dir``, ``--logs-dir``, and ``--config-dir``. By default these are -``/var/lib/letsencrypt``, ``/var/logs/letsencrypt``, and ``/etc/letsencrypt`` +``/var/lib/letsencrypt``, ``/var/log/letsencrypt``, and ``/etc/letsencrypt`` respectively. Additionally if you are using Certbot with Apache or nginx it will lock the configuration folder for that program, which are typically also in the ``/etc`` directory. From 30803f30ba63742cf98f4c6fbfcf890317d558ae Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Mon, 4 Feb 2019 13:03:29 -0500 Subject: [PATCH 554/639] acme: add TLSALPN01Response for initiating tls-alpn-01. (#6689) The existing `acme.TLSALPN01` challenge class did not have a `response_cls`, meaning it was not possible to use a tls-alpn-01 challenge with `client.answer_challenge`. To support the above a simple `TLSALPN01Response` class is added that doesn't provide the ability to solve a tls-alpn-01 challenge end to end (e.g. generating and serving the correct response certificate) but that does allow the challenge to be initiated. This is sufficient for users that have set up the challenge response independent of the `acme` module code. Resolves #6676 --- CHANGELOG.md | 2 ++ acme/acme/challenges.py | 12 ++++++++++++ acme/acme/challenges_test.py | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d45af20e8..cae93c5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Avoid to process again challenges that are already validated when a certificate is issued. +* Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges + with the `acme` module. ### Changed diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index a65768228..501f74881 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -513,6 +513,17 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) +@ChallengeResponse.register +class TLSALPN01Response(KeyAuthorizationChallengeResponse): + """ACME TLS-ALPN-01 challenge response. + + This class only allows initiating a TLS-ALPN-01 challenge returned from the + CA. Full support for responding to TLS-ALPN-01 challenges by generating and + serving the expected response certificate is not currently provided. + """ + typ = "tls-alpn-01" + + @Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): """ACME tls-alpn-01 challenge. @@ -522,6 +533,7 @@ class TLSALPN01(KeyAuthorizationChallenge): """ typ = "tls-alpn-01" + response_cls = TLSALPN01Response def validation(self, account_key, **kwargs): """Generate validation for the challenge.""" diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 9307eb95b..81d39058e 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -402,6 +402,33 @@ class TLSSNI01Test(unittest.TestCase): KEY, cert_key=mock.sentinel.cert_key)) mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) +class TLSALPN01ResponseTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.challenges import TLSALPN01Response + self.msg = TLSALPN01Response(key_authorization=u'foo') + self.jmsg = { + 'resource': 'challenge', + 'type': 'tls-alpn-01', + 'keyAuthorization': u'foo', + } + + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01(token=(b'x' * 16)) + self.response = self.chall.response(KEY) + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.msg.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01Response + self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01Response + hash(TLSALPN01Response.from_json(self.jmsg)) + class TLSALPN01Test(unittest.TestCase): From 9671985885ed8106bf88c1c6250ed3a9b010a094 Mon Sep 17 00:00:00 2001 From: schoen Date: Mon, 4 Feb 2019 13:11:52 -0800 Subject: [PATCH 555/639] Clarify what a "renewal attempt" is (#6735) --- docs/using.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 531c84e85..e17e56b64 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -486,8 +486,9 @@ non-zero exit code. Hooks will only be run if a certificate is due for renewal, so you can run the above command frequently without unnecessarily stopping your webserver. -``--pre-hook`` and ``--post-hook`` hooks run before and after every renewal -attempt. If you want your hook to run only after a successful renewal, use +When Certbot detects that a certificate is due for renewal, ``--pre-hook`` +and ``--post-hook`` hooks run before and after each attempt to renew it. +If you want your hook to run only after a successful renewal, use ``--deploy-hook`` in a command like this. ``certbot renew --deploy-hook /path/to/deploy-hook-script`` From 2ddaf3db043ea8526ae3f9ab2ef120b194b2e506 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 5 Feb 2019 19:45:15 +0100 Subject: [PATCH 556/639] Use built-in support for OCSP in cryptography >= 2.5 (#6603) In response to #6594. [Fixes #6594.] To execute OCSP requests, certbot relies currently on a openssl binary execution. If openssl is not present in the PATH, the OCSP check will be silently ignored. Since version 2.4, cryptography has support for OCSP requests, without the need to have openssl binary available locally. This PR takes advantage of it, and will use the built-in support of OCSP in cryptography for versions >= 2.4. Otherwise, fallback is done do a direct call to openssl binary, allowing oldest requirements to still work with legacy cryptography versions. Update: requirement is now cryptography >= 2.5, to avoid to rely on a private method from cryptography. * Implement logic using cryptography * Working OSCP using pure cryptography * Fix openssl usage in unit tests * Reduce verbosity * Add tests * Improve naive skipIf * Test resiliency * Update ocsp.py * Validate OCSP response. Unify OCSP URL get * Improve resiliency checks, correct lint/mypy * Improve hash selection * Fix warnings when calling openssl bin * Load OCSP tests assets as vectors. * Update ocsp.py * Protect against invalid ocsp response. * Add checks to OCSP response * Add more control on ocsp response * Be lenient about assertion that next_update must be in the future, similarly to openssl. * Construct a more advanced OCSP response mock to trigger more logic in ocsp module. * Add test * Refactor signature process to use crypto_util * Fallback for cryptography 2.4 * Avoid a collision with a meteor. * Correct method signature documentation * Relax OCSP update interval * Trigger built-in ocsp logic from cryptography with 2.5+ * Update pinned version of cryptography * Update certbot/ocsp.py Co-Authored-By: adferrand * Update ocsp.py * Update ocsp_test.py * Update CHANGELOG.md * Update CHANGELOG.md --- CHANGELOG.md | 5 +- certbot/crypto_util.py | 53 +++-- certbot/ocsp.py | 197 ++++++++++++++---- certbot/tests/ocsp_test.py | 172 +++++++++++++-- certbot/tests/testdata/google_certificate.pem | 41 ++++ .../testdata/google_issuer_certificate.pem | 26 +++ certbot/util.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 40 ++-- .../pieces/dependency-requirements.txt | 40 ++-- 9 files changed, 451 insertions(+), 125 deletions(-) create mode 100644 certbot/tests/testdata/google_certificate.pem create mode 100644 certbot/tests/testdata/google_issuer_certificate.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index cae93c5b7..204b7b328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,11 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* Avoid to process again challenges that are already validated +* Avoid reprocessing challenges that are already validated when a certificate is issued. +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index c4a389cd5..66c68eb38 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -19,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey # https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore @@ -226,7 +226,7 @@ def verify_renewable_cert(renewable_cert): def verify_renewable_cert_sig(renewable_cert): - """ Verifies the signature of a `.storage.RenewableCert` object. + """Verifies the signature of a `.storage.RenewableCert` object. :param `.storage.RenewableCert` renewable_cert: cert to verify @@ -239,22 +239,8 @@ def verify_renewable_cert_sig(renewable_cert): cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") + verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, + cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) @@ -262,6 +248,37 @@ def verify_renewable_cert_sig(renewable_cert): raise errors.Error(error_str) +def verify_signed_payload(public_key, signature, payload, signature_hash_algorithm): + """Check the signature of a payload. + + :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature + :param bytes signature: the signature bytes + :param bytes payload: the payload bytes + :param cryptography.hazmat.primitives.hashes.HashAlgorithm + signature_hash_algorithm: algorithm used to hash the payload + + :raises InvalidSignature: If signature verification fails. + :raises errors.Error: If public key type is not supported + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(public_key, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = public_key.verifier( # type: ignore + signature, PKCS1v15(), signature_hash_algorithm + ) + verifier.update(payload) + verifier.verify() + elif isinstance(public_key, EllipticCurvePublicKey): + verifier = public_key.verifier( + signature, ECDSA(signature_hash_algorithm) + ) + verifier.update(payload) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + + def verify_cert_matches_priv_key(cert_path, key_path): """ Verifies that the private key and cert match. diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 049e14827..0abfd3c23 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -1,53 +1,79 @@ """Tools for checking certificate revocation.""" import logging import re - +from datetime import datetime, timedelta from subprocess import Popen, PIPE +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp # pylint: disable=import-error + getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp = None # type: ignore +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +import requests + +from acme.magic_typing import Optional, Tuple # pylint: disable=unused-import, no-name-in-module +from certbot import crypto_util from certbot import errors from certbot import util logger = logging.getLogger(__name__) + class RevocationChecker(object): - "This class figures out OCSP checking on this system, and performs it." + """This class figures out OCSP checking on this system, and performs it.""" - def __init__(self): + def __init__(self, enforce_openssl_binary_usage=False): self.broken = False + self.use_openssl_binary = enforce_openssl_binary_usage or not ocsp - if not util.exe_exists("openssl"): - logger.info("openssl not installed, can't check revocation") - self.broken = True - return - - # New versions of openssl want -header var=val, old ones want -header var val - test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], - stdout=PIPE, stderr=PIPE, universal_newlines=True) - _out, err = test_host_format.communicate() - if "Missing =" in err: - self.host_args = lambda host: ["Host=" + host] - else: - self.host_args = lambda host: ["Host", host] + if self.use_openssl_binary: + if not util.exe_exists("openssl"): + logger.info("openssl not installed, can't check revocation") + self.broken = True + return + # New versions of openssl want -header var=val, old ones want -header var val + test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], + stdout=PIPE, stderr=PIPE, universal_newlines=True) + _out, err = test_host_format.communicate() + if "Missing =" in err: + self.host_args = lambda host: ["Host=" + host] + else: + self.host_args = lambda host: ["Host", host] def ocsp_revoked(self, cert_path, chain_path): + # type: (str, str) -> bool """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call :param str cert_path: Path to certificate :param str chain_path: Path to intermediate cert - :rtype bool or None: :returns: True if revoked; False if valid or the check failed + :rtype: bool """ if self.broken: return False - - url, host = self.determine_ocsp_server(cert_path) - if not host: + url, host = _determine_ocsp_server(cert_path) + if not host or not url: return False + + if self.use_openssl_binary: + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) + else: + return _check_ocsp_cryptography(cert_path, chain_path, url) + + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): + # type: (str, str, str, str) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -65,33 +91,117 @@ class RevocationChecker(object): except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False - return _translate_ocsp_query(cert_path, output, err) - def determine_ocsp_server(self, cert_path): - """Extract the OCSP server host from a certificate. +def _determine_ocsp_server(cert_path): + # type: (str) -> Tuple[Optional[str], Optional[str]] + """Extract the OCSP server host from a certificate. - :param str cert_path: Path to the cert we're checking OCSP for - :rtype tuple: - :returns: (OCSP server URL or None, OCSP server host or None) + :param str cert_path: Path to the cert we're checking OCSP for + :rtype tuple: + :returns: (OCSP server URL or None, OCSP server host or None) - """ - try: - url, _err = util.run_script( - ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logger.debug) - except errors.SubprocessError: - logger.info("Cannot extract OCSP URI from %s", cert_path) - return None, None + """ + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + try: + extension = cert.extensions.get_extension_for_class(x509.AuthorityInformationAccess) + ocsp_oid = x509.AuthorityInformationAccessOID.OCSP + descriptions = [description for description in extension.value + if description.access_method == ocsp_oid] + + url = descriptions[0].access_location.value + except (x509.ExtensionNotFound, IndexError): + logger.info("Cannot extract OCSP URI from %s", cert_path) + return None, None + + url = url.rstrip() + host = url.partition("://")[2].rstrip("/") + + if host: + return url, host + else: + logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + return None, None + + +def _check_ocsp_cryptography(cert_path, chain_path, url): + # type: (str, str, str) -> bool + # Retrieve OCSP response + with open(chain_path, 'rb') as file_handler: + issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + builder = ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + request_binary = request.public_bytes(serialization.Encoding.DER) + response = requests.post(url, data=request_binary, + headers={'Content-Type': 'application/ocsp-request'}) + if response.status_code != 200: + logger.info("OCSP check failed for %s (are we offline?)", cert_path) + return False + response_ocsp = ocsp.load_der_ocsp_response(response.content) + + # Check OCSP response validity + if response_ocsp.response_status != ocsp.OCSPResponseStatus.SUCCESSFUL: + logger.error("Invalid OCSP response status for %s: %s", + cert_path, response_ocsp.response_status) + return False + + # Check OCSP signature + try: + _check_ocsp_response(response_ocsp, request, issuer) + except UnsupportedAlgorithm as e: + logger.error(str(e)) + except errors.Error as e: + logger.error(str(e)) + except InvalidSignature: + logger.error('Invalid signature on OCSP response for %s', cert_path) + except AssertionError as error: + logger.error('Invalid OCSP response for %s: %s.', cert_path, str(error)) + else: + # Check OCSP certificate status + logger.debug("OCSP certificate status for %s is: %s", + cert_path, response_ocsp.certificate_status) + return response_ocsp.certificate_status == ocsp.OCSPCertStatus.REVOKED + + return False + + +def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): + """Verify that the OCSP is valid for serveral criterias""" + # Assert OCSP response corresponds to the certificate we are talking about + if response_ocsp.serial_number != request_ocsp.serial_number: + raise AssertionError('the certificate in response does not correspond ' + 'to the certificate in request') + + # Assert signature is valid + _check_ocsp_response_signature(response_ocsp, issuer_cert) + + # Assert issuer in response is the expected one + if (not isinstance(response_ocsp.hash_algorithm, type(request_ocsp.hash_algorithm)) + or response_ocsp.issuer_key_hash != request_ocsp.issuer_key_hash + or response_ocsp.issuer_name_hash != request_ocsp.issuer_name_hash): + raise AssertionError('the issuer does not correspond to issuer of the certificate.') + + # Assert nextUpdate is in the future, and that thisUpdate is not too old + if response_ocsp.next_update: + if response_ocsp.next_update < datetime.now() - timedelta(minutes=5): + raise AssertionError('next update is in the past.') + interval = response_ocsp.next_update - response_ocsp.this_update + if datetime.now() - response_ocsp.this_update > interval + timedelta(minutes=5): + raise AssertionError('this update is too old.') + + +def _check_ocsp_response_signature(response_ocsp, issuer_cert): + """Verify an OCSP response signature against certificate issuer""" + # Following line may raise UnsupportedAlgorithm + chosen_hash = response_ocsp.signature_hash_algorithm + crypto_util.verify_signed_payload(issuer_cert.public_key(), response_ocsp.signature, + response_ocsp.tbs_response_bytes, chosen_hash) - url = url.rstrip() - host = url.partition("://")[2].rstrip("/") - if host: - return url, host - else: - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) - return None, None def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): """Parse openssl's weird output to work out what it means.""" @@ -102,7 +212,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): warning = good.group(1) if good else None - if (not "Response verify OK" in ocsp_errors) or (good and warning) or unknown: + if ("Response verify OK" not in ocsp_errors) or (good and warning) or unknown: logger.info("Revocation status for %s is unknown", cert_path) logger.debug("Uncertain output:\n%s\nstderr:\n%s", ocsp_output, ocsp_errors) return False @@ -115,6 +225,5 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): return True else: logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", - ocsp_output, ocsp_errors) + ocsp_output, ocsp_errors) return False - diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 55cd24adb..ad3467e5a 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -1,18 +1,33 @@ """Tests for ocsp.py""" # pylint: disable=protected-access - import unittest +from datetime import datetime, timedelta +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +from cryptography import x509 +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp as ocsp_lib # pylint: disable=import-error + getattr(ocsp_lib.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp_lib = None # type: ignore import mock from certbot import errors +from certbot.tests import util as test_util out = """Missing = in header key=value ocsp: Use -help for summary. """ -class OCSPTest(unittest.TestCase): +class OCSPTestOpenSSL(unittest.TestCase): + """ + OCSP revokation tests using OpenSSL binary. + """ def setUp(self): from certbot import ocsp @@ -22,7 +37,7 @@ class OCSPTest(unittest.TestCase): mock_communicate.communicate.return_value = (None, out) mock_popen.return_value = mock_communicate mock_exists.return_value = True - self.checker = ocsp.RevocationChecker() + self.checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) def tearDown(self): pass @@ -37,23 +52,23 @@ class OCSPTest(unittest.TestCase): mock_exists.return_value = True from certbot import ocsp - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 1) self.assertEqual(checker.host_args("x"), ["Host=x"]) mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) self.assertEqual(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) self.assertEqual(checker.broken, True) - @mock.patch('certbot.ocsp.RevocationChecker.determine_ocsp_server') + @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.util.run_script') def test_ocsp_revoked(self, mock_run, mock_determine): self.checker.broken = True @@ -71,21 +86,12 @@ class OCSPTest(unittest.TestCase): self.assertEqual(self.checker.ocsp_revoked("x", "y"), False) self.assertEqual(mock_run.call_count, 2) + def test_determine_ocsp_server(self): + cert_path = test_util.vector_path('google_certificate.pem') - @mock.patch('certbot.ocsp.logger.info') - @mock.patch('certbot.util.run_script') - def test_determine_ocsp_server(self, mock_run, mock_info): - uri = "http://ocsp.stg-int-x1.letsencrypt.org/" - host = "ocsp.stg-int-x1.letsencrypt.org" - mock_run.return_value = uri, "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (uri, host)) - mock_run.return_value = "ftp:/" + host + "/", "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) - self.assertEqual(mock_info.call_count, 1) - - c = "confusion" - mock_run.side_effect = errors.SubprocessError(c) - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) + from certbot import ocsp + result = ocsp._determine_ocsp_server(cert_path) + self.assertEqual(('http://ocsp.digicert.com', 'ocsp.digicert.com'), result) @mock.patch('certbot.ocsp.logger') @mock.patch('certbot.util.run_script') @@ -112,6 +118,129 @@ class OCSPTest(unittest.TestCase): self.assertEqual(mock_log.info.call_count, 1) +@unittest.skipIf(not ocsp_lib, + reason='This class tests functionalities available only on cryptography>=2.5.0') +class OSCPTestCryptography(unittest.TestCase): + """ + OCSP revokation tests using Cryptography >= 2.4.0 + """ + + def setUp(self): + from certbot import ocsp + self.checker = ocsp.RevocationChecker() + self.cert_path = test_util.vector_path('google_certificate.pem') + self.chain_path = test_util.vector_path('google_issuer_certificate.pem') + + @mock.patch('certbot.ocsp._determine_ocsp_server') + @mock.patch('certbot.ocsp._check_ocsp_cryptography') + def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + mock_determine.return_value = ('http://example.com', 'example.com') + self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke(self, mock_ocsp_response, mock_post): + with mock.patch('certbot.ocsp.crypto_util.verify_signed_payload'): + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertTrue(revoked) + + @mock.patch('certbot.ocsp.crypto_util.verify_signed_payload') + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke_resiliency(self, mock_ocsp_response, mock_post, mock_check): + # Server return an invalid HTTP response + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=400) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response in invalid + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response is valid, but certificate status is unknown + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # The OCSP response says that the certificate is revoked, but certificate + # does not contain the OCSP extension. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + with mock.patch('cryptography.x509.Extensions.get_extension_for_class', + side_effect=x509.ExtensionNotFound( + 'Not found', x509.AuthorityInformationAccessOID.OCSP)): + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Valid response, OCSP extension is present, + # but OCSP response uses an unsupported signature. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = UnsupportedAlgorithm('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # And now, the signature itself is invalid. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = InvalidSignature('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Finally, assertion error on OCSP response validity + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = AssertionError('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + +def _construct_mock_ocsp_response(certificate_status, response_status): + cert = x509.load_pem_x509_certificate( + test_util.load_vector('google_certificate.pem'), default_backend()) + issuer = x509.load_pem_x509_certificate( + test_util.load_vector('google_issuer_certificate.pem'), default_backend()) + builder = ocsp_lib.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + + return mock.Mock( + response_status=response_status, + certificate_status=certificate_status, + serial_number=request.serial_number, + issuer_key_hash=request.issuer_key_hash, + issuer_name_hash=request.issuer_name_hash, + hash_algorithm=hashes.SHA1(), + next_update=datetime.now() + timedelta(days=1), + this_update=datetime.now() - timedelta(days=1), + signature_algorithm_oid=x509.oid.SignatureAlgorithmOID.RSA_WITH_SHA1, + ) + + # pylint: disable=line-too-long openssl_confused = ("", """ /etc/letsencrypt/live/example.org/cert.pem: good @@ -165,5 +294,6 @@ revoked """, """Response verify OK""") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/testdata/google_certificate.pem b/certbot/tests/testdata/google_certificate.pem new file mode 100644 index 000000000..c26fea0b1 --- /dev/null +++ b/certbot/tests/testdata/google_certificate.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1 +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk +IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy +MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB +BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF +Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG +A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD +VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ +w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj +/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE +sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ +xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM +fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag +re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG +A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE +AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0 +oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy +LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy +dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB +FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF +BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS +BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0 +U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA +MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY +BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT +Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg +U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE +AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51 +vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV +CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN +8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG +eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v +ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN +HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB +Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk +myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq +WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik= +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/google_issuer_certificate.pem b/certbot/tests/testdata/google_issuer_certificate.pem new file mode 100644 index 000000000..50db47bc4 --- /dev/null +++ b/certbot/tests/testdata/google_issuer_certificate.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEXDCCA0SgAwIBAgINAeOpMBz8cgY4P5pTHTANBgkqhkiG9w0BAQsFADBMMSAw +HgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFs +U2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEy +MTUwMDAwNDJaMFQxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3Qg +U2VydmljZXMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzMw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKUkvqHv/OJGuo2nIYaNVW +XQ5IWi01CXZaz6TIHLGp/lOJ+600/4hbn7vn6AAB3DVzdQOts7G5pH0rJnnOFUAK +71G4nzKMfHCGUksW/mona+Y2emJQ2N+aicwJKetPKRSIgAuPOB6Aahh8Hb2XO3h9 +RUk2T0HNouB2VzxoMXlkyW7XUR5mw6JkLHnA52XDVoRTWkNty5oCINLvGmnRsJ1z +ouAqYGVQMc/7sy+/EYhALrVJEA8KbtyX+r8snwU5C1hUrwaW6MWOARa8qBpNQcWT +kaIeoYvy/sGIJEmjR0vFEwHdp1cSaWIr6/4g72n7OqXwfinu7ZYW97EfoOSQJeAz +AgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH +AwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHfCuFCa +Z3Z2sS3ChtCDoH6mfrpLMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYu +MDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdv +b2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dz +cjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYc +aHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEA +HLeJluRT7bvs26gyAZ8so81trUISd7O45skDUmAge1cnxhG1P2cNmSxbWsoiCt2e +ux9LSD+PAj2LIYRFHW31/6xoic1k4tbWXkDCjir37xTTNqRAMPUyFRWSdvt+nlPq +wnb8Oa2I/maSJukcxDjNSfpDh/Bd1lZNgdd/8cLdsE3+wypufJ9uXO1iQpnh9zbu +FIwsIONGl1p3A8CgxkqI/UAih3JaGOqcpcdaCIzkBaR9uYQ1X4k2Vg5APRLouzVy +7a8IVk6wuy6pm+T7HT4LY8ibS5FEZlfAFLSW8NwsVz9SBK2Vqn1N0PIMn5xA6NZV +c7o835DLAFshEWfC7TIe3g== +-----END CERTIFICATE----- diff --git a/certbot/util.py b/certbot/util.py index 416075ce8..097593e9d 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -62,7 +62,7 @@ def run_script(params, log=logger.error): """Run the script with the given params. :param list params: List of parameters to pass to Popen - :param logging.Logger log: Logger to use for errors + :param callable log: Logger method to use for errors """ try: diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 48ddfb570..8aaa1e3fb 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1034,26 +1034,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1fac78836..dff57dfd5 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -60,26 +60,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 From 7e6a1f248866df8b581f372f3e57355ca4ab4b1c Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 6 Feb 2019 20:02:35 +0200 Subject: [PATCH 557/639] Apache plugin: configure all matching domain names to be able to answer HTTP challenge. (#6729) Attempts to configure all of the following VirtualHosts for answering the HTTP challenge: * VirtualHosts that have the requested domain name in either `ServerName` or `ServerAlias` directive. * VirtualHosts that have a wildcard name that would match the requested domain name. This also applies to HTTPS VirtualHosts, making Apache plugin able to handle cases where HTTP redirection takes place in reverse proxy or similar, before reaching the Apache HTTPD. Even though also HTTPS VirtualHosts are selected, Apache plugin tries to ensure that at least one of the selected VirtualHosts listens to HTTP-01 port (configured with `--http-01-port` CLI option). So in a case where only HTTPS VirtualHosts exist, but user wants to configure those, `--http-01-port` parameter needs to be set for the port configured to the HTTPS VirtualHost(s). Fixes: #6730 * Select all matching VirtualHosts for HTTP-01 challenges instead of just one * Finalize PR and add tests * Changelog entry --- CHANGELOG.md | 2 + certbot-apache/certbot_apache/configurator.py | 9 ++-- certbot-apache/certbot_apache/http_01.py | 44 +++++++++++++++---- .../certbot_apache/tests/configurator_test.py | 22 +++++----- .../certbot_apache/tests/http_01_test.py | 34 +++++++++----- .../certbot_apache/tests/parser_test.py | 2 +- .../sites-available/duplicatehttp.conf | 9 ++++ .../sites-available/duplicatehttps.conf | 14 ++++++ .../apache2/sites-enabled/duplicatehttp.conf | 1 + .../apache2/sites-enabled/duplicatehttps.conf | 1 + certbot-apache/certbot_apache/tests/util.py | 12 ++++- 11 files changed, 112 insertions(+), 38 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttp.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttps.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttp.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttps.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 204b7b328..c529f3bb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support on 2.x branch is maintained). +* Apache plugin now attempts to configure all VirtualHosts matching requested + domain name instead of only a single one when answering the HTTP-01 challenge. ### Fixed diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 16de3a3d8..efd766e63 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -577,8 +577,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc[target_name] = vhost return vhost - def included_in_wildcard(self, names, target_name): - """Is target_name covered by a wildcard? + def domain_in_names(self, names, target_name): + """Checks if target domain is covered by one or more of the provided + names. The target name is matched by wildcard as well as exact match. :param names: server aliases :type names: `collections.Iterable` of `str` @@ -649,7 +650,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): names = vhost.get_names() if target_name in names: points = 3 - elif self.included_in_wildcard(names, target_name): + elif self.domain_in_names(names, target_name): points = 2 elif any(addr.get_addr() == target_name for addr in vhost.addrs): points = 1 @@ -1463,7 +1464,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): matches = self.parser.find_dir( "ServerAlias", start=vh_path, exclude=False) aliases = (self.aug.get(match) for match in matches) - return self.included_in_wildcard(aliases, target_name) + return self.domain_in_names(aliases, target_name) def _add_name_vhost_if_necessary(self, vhost): """Add NameVirtualHost Directives if necessary for new vhost. diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 22598baca..962433415 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -2,7 +2,7 @@ import logging import os -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List, Set # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.plugins import common from certbot_apache.obj import VirtualHost # pylint: disable=unused-import @@ -89,15 +89,27 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): + selected_vhosts = [] # type: List[VirtualHost] + http_port = str(self.configurator.config.http01_port) for chall in self.achalls: - vh = self.configurator.find_best_http_vhost( - chall.domain, filter_defaults=False, - port=str(self.configurator.config.http01_port)) - if vh: - self._set_up_include_directives(vh) - else: - for vh in self._relevant_vhosts(): - self._set_up_include_directives(vh) + # Search for matching VirtualHosts + for vh in self._matching_vhosts(chall.domain): + selected_vhosts.append(vh) + + # Ensure that we have one or more VirtualHosts that we can continue + # with. (one that listens to port configured with --http-01-port) + found = False + for vhost in selected_vhosts: + if any(a.is_wildcard() or a.get_port() == http_port for a in vhost.addrs): + found = True + + if not found: + for vh in self._relevant_vhosts(): + selected_vhosts.append(vh) + + # Add the challenge configuration + for vh in selected_vhosts: + self._set_up_include_directives(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf_pre) @@ -121,6 +133,20 @@ class ApacheHttp01(common.TLSSNI01): with open(self.challenge_conf_post, "w") as new_conf: new_conf.write(config_text_post) + def _matching_vhosts(self, domain): + """Return all VirtualHost objects that have the requested domain name or + a wildcard name that would match the domain in ServerName or ServerAlias + directive. + """ + matching_vhosts = [] + for vhost in self.configurator.vhosts: + if self.configurator.domain_in_names(vhost.get_names(), domain): + # domain_in_names also matches the exact names, so no need + # to check "domain in vhost.get_names()" explicitly here + matching_vhosts.append(vhost) + + return matching_vhosts + def _relevant_vhosts(self): http01_port = str(self.configurator.config.http01_port) relevant_vhosts = [] diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 4aaa23ea4..ff93682b6 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -139,7 +139,8 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() self.assertEqual(names, set( ["certbot.demo", "ocspvhost.com", "encryption-example.demo", - "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"] + "nonsym.link", "vhost.in.rootconf", "www.certbot.demo", + "duplicate.example.com"] )) @certbot_util.patch_get_utility() @@ -158,8 +159,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.vhosts.append(vhost) names = self.config.get_all_names() - # Names get filtered, only 5 are returned - self.assertEqual(len(names), 8) + self.assertEqual(len(names), 9) self.assertTrue("zombo.com" in names) self.assertTrue("google.com" in names) self.assertTrue("certbot.demo" in names) @@ -200,7 +200,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_get_virtual_hosts(self): """Make sure all vhosts are being properly found.""" vhs = self.config.get_virtual_hosts() - self.assertEqual(len(vhs), 10) + self.assertEqual(len(vhs), 12) found = 0 for vhost in vhs: @@ -211,7 +211,7 @@ class MultipleVhostsTest(util.ApacheTest): else: raise Exception("Missed: %s" % vhost) # pragma: no cover - self.assertEqual(found, 10) + self.assertEqual(found, 12) # Handle case of non-debian layout get_virtual_hosts with mock.patch( @@ -219,7 +219,7 @@ class MultipleVhostsTest(util.ApacheTest): ) as mock_conf: mock_conf.return_value = False vhs = self.config.get_virtual_hosts() - self.assertEqual(len(vhs), 10) + self.assertEqual(len(vhs), 12) @mock.patch("certbot_apache.display_ops.select_vhost") def test_choose_vhost_none_avail(self, mock_select): @@ -322,7 +322,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.vhosts = [ vh for vh in self.config.vhosts if vh.name not in ["certbot.demo", "nonsym.link", - "encryption-example.demo", + "encryption-example.demo", "duplicate.example.com", "ocspvhost.com", "vhost.in.rootconf"] and "*.blue.purple.com" not in vh.aliases ] @@ -333,7 +333,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_non_default_vhosts(self): # pylint: disable=protected-access vhosts = self.config._non_default_vhosts(self.config.vhosts) - self.assertEqual(len(vhosts), 8) + self.assertEqual(len(vhosts), 10) def test_deploy_cert_enable_new_vhost(self): # Create @@ -688,7 +688,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]), self.config.is_name_vhost(ssl_vhost)) - self.assertEqual(len(self.config.vhosts), 11) + self.assertEqual(len(self.config.vhosts), 13) def test_clean_vhost_ssl(self): # pylint: disable=protected-access @@ -1269,7 +1269,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") - self.assertEqual(len(self.config.vhosts), 11) + self.assertEqual(len(self.config.vhosts), 13) def test_create_own_redirect_for_old_apache_version(self): self.config.parser.modules.add("rewrite_module") @@ -1280,7 +1280,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") - self.assertEqual(len(self.config.vhosts), 11) + self.assertEqual(len(self.config.vhosts), 13) def test_sift_rewrite_rule(self): # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 9c729b08c..bf7a3719a 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -27,8 +27,8 @@ class ApacheHttp01Test(util.ApacheTest): self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") - # Takes the vhosts for encryption-example.demo, certbot.demo, and - # vhost.in.rootconf + # Takes the vhosts for encryption-example.demo, certbot.demo + # and vhost.in.rootconf self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]] for i in range(NUM_ACHALLS): @@ -39,7 +39,7 @@ class ApacheHttp01Test(util.ApacheTest): "pending"), domain=self.vhosts[i].name, account_key=self.account_key)) - modules = ["rewrite", "authz_core", "authz_host"] + modules = ["ssl", "rewrite", "authz_core", "authz_host"] for mod in modules: self.config.parser.modules.add("mod_{0}.c".format(mod)) self.config.parser.modules.add(mod + "_module") @@ -111,6 +111,17 @@ class ApacheHttp01Test(util.ApacheTest): domain="something.nonexistent", account_key=self.account_key)] self.common_perform_test(achalls, vhosts) + def test_configure_multiple_vhosts(self): + vhosts = [v for v in self.config.vhosts if "duplicate.example.com" in v.get_names()] + self.assertEqual(len(vhosts), 2) + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="duplicate.example.com", account_key=self.account_key)] + self.common_perform_test(achalls, vhosts) + def test_no_vhost(self): for achall in self.achalls: self.http.add_chall(achall) @@ -176,15 +187,14 @@ class ApacheHttp01Test(util.ApacheTest): self._test_challenge_file(achall) for vhost in vhosts: - if not vhost.ssl: - matches = self.config.parser.find_dir("Include", - self.http.challenge_conf_pre, - vhost.path) - self.assertEqual(len(matches), 1) - matches = self.config.parser.find_dir("Include", - self.http.challenge_conf_post, - vhost.path) - self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, + vhost.path) + self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index a089ec471..5d692eeac 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -52,7 +52,7 @@ class BasicParserTest(util.ParserTest): test2 = self.parser.find_dir("documentroot") self.assertEqual(len(test), 1) - self.assertEqual(len(test2), 7) + self.assertEqual(len(test2), 8) def test_add_dir(self): aug_default = "/files" + self.parser.loc["default"] diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttp.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttp.conf new file mode 100644 index 000000000..5684651fb --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttp.conf @@ -0,0 +1,9 @@ + + ServerName duplicate.example.com + + ServerAdmin webmaster@certbot.demo + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttps.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttps.conf new file mode 100644 index 000000000..e3ac21fac --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttps.conf @@ -0,0 +1,14 @@ + + + ServerName duplicate.example.com + + ServerAdmin webmaster@certbot.demo + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + +SSLCertificateFile /etc/apache2/certs/certbot-cert_5.pem +SSLCertificateKeyFile /etc/apache2/ssl/key-certbot_15.pem + + diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttp.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttp.conf new file mode 120000 index 000000000..a69ee3c1d --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttp.conf @@ -0,0 +1 @@ +../sites-available/duplicatehttp.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttps.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttps.conf new file mode 120000 index 000000000..a52ee1ccb --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttps.conf @@ -0,0 +1 @@ +../sites-available/duplicatehttps.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 9329ccb20..02de6ada4 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -196,7 +196,17 @@ def get_vh_truth(temp_dir, config_name): "/files" + os.path.join(temp_dir, config_name, "apache2/apache2.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - "vhost.in.rootconf")] + "vhost.in.rootconf"), + obj.VirtualHost( + os.path.join(prefix, "duplicatehttp.conf"), + os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"), + set([obj.Addr.fromstring("10.2.3.4:80")]), False, True, + "duplicate.example.com"), + obj.VirtualHost( + os.path.join(prefix, "duplicatehttps.conf"), + os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"), + set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + "duplicate.example.com")] return vh_truth if config_name == "debian_apache_2_4/multi_vhosts": prefix = os.path.join( From 2560ef0ffa5aebf5ef54c9a543dfc3edb28314d0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Feb 2019 12:47:56 -0800 Subject: [PATCH 558/639] Test all on push events to tested non-master branches. (#6741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We always run a full set of CI tests before beginning the release process. The way this would work previously is we would either trigger tests on the `test-everything` branch to run through Travis' web UI or if it was a point release, create a new branch based on `test-everything` but modify `.travis.yml` so the branch that was pulled in to be tested was the point release branch instead of `master`. This no longer works because the former `test-everything` tests are now only run when Travis automatically runs our tests nightly. We could create and maintain a separate branch for the purpose of manually running all tests or remove the conditionals from the latest `.travis.yml` file every time before we want to run these tests, but there must be A Better Wayâ„¢. This PR makes the change that in addition to running all tests nightly, they would also run on pushes to tested branches other than master. These changes do not affect the tests run on PRs or on commits to `master`. What is affected is commits to point release branches and branches named `test-*`. (See [.travis.yml](https://github.com/certbot/certbot/blob/2ddaf3db043ea8526ae3f9ab2ef120b194b2e506/.travis.yml#L177) for what branches we run tests on.) Running all tests on point release branches automates the step of running our full test suite before doing a point release. The changes to `test-*` could be a mixed bag, however, since we switched to travis-ci.com over 3 weeks ago, I'm the only one who has used this functionality and I personally prefer things this way. At the very least, since these branches don't seem to be widely used, I think we can make this change and reevaluate if it becomes a problem. * Test all on push events to non-master branches. * Move branches section up. * expand comment --- .travis.yml | 64 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 16cb6f23f..0d59ec9e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,16 @@ before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' - export TOX_TESTENV_PASSENV=TRAVIS +# Only build pushes to the master branch, PRs, and branches beginning with +# `test-` or of the form `digit(s).digit(s).x`. This reduces the number of +# simultaneous Travis runs, which speeds turnaround time on review since there +# is a cap of on the number of simultaneous runs. +branches: + only: + - master + - /^\d+\.\d+\.x$/ + - /^test-.*$/ + matrix: include: # These environments are always executed @@ -62,85 +72,87 @@ matrix: - python: "2.7" env: TOXENV=nginxroundtrip - # These environments are executed on cron events only + # These environments are executed on cron events and commits to tested + # branches other than master. Which branches are tested is controlled by + # the "branches" section earlier in this file. - python: "3.7" dist: xenial env: TOXENV=py37 CERTBOT_NO_PIN=1 - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - sudo: required env: TOXENV=le_auto_xenial services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - sudo: required env: TOXENV=le_auto_jessie services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - sudo: required env: TOXENV=le_auto_centos6 services: docker - if: type = cron + if: type = cron OR (type = push AND branch != master) - sudo: required env: TOXENV=docker_dev services: docker @@ -148,7 +160,7 @@ matrix: apt: packages: # don't install nginx and apache - libaugeas0 - if: type = cron + if: type = cron OR (type = push AND branch != master) - language: generic env: TOXENV=py27 os: osx @@ -157,7 +169,7 @@ matrix: packages: - augeas - python2 - if: type = cron + if: type = cron OR (type = push AND branch != master) - language: generic env: TOXENV=py3 os: osx @@ -166,19 +178,7 @@ matrix: packages: - augeas - python3 - if: type = cron - - - -# Only build pushes to the master branch, PRs, and branches beginning with -# `test-` or of the form `digit(s).digit(s).x`. This reduces the number of -# simultaneous Travis runs, which speeds turnaround time on review since there -# is a cap of on the number of simultaneous runs. -branches: - only: - - master - - /^\d+\.\d+\.x$/ - - /^test-.*$/ + if: type = cron OR (type = push AND branch != master) # container-based infrastructure sudo: false From c5baf035df064c854e0c16be3479b3bbaa778b41 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 6 Feb 2019 14:51:52 -0800 Subject: [PATCH 559/639] Update CHANGELOG.md (#6745) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c529f3bb4..c119323cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ package with changes other than its version number was: * acme * certbot +* certbot-apache * certbot-dns-cloudxns * certbot-dns-dnsimple * certbot-dns-dnsmadeeasy From ab79d1d44a8fe44c1ec90b71355b550480dbd042 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Feb 2019 16:36:32 -0800 Subject: [PATCH 560/639] Revert "Use built-in support for OCSP in cryptography >= 2.5 (#6603) (#6747) I think this is causing failures in some of our tests so this PR reverts the change until we can fix the problem. The 2nd commit is to keep the change using more idiomatic wording in the changelog for another change that got included in this PR. * Revert "Use built-in support for OCSP in cryptography >= 2.5 (#6603)" This reverts commit 2ddaf3db043ea8526ae3f9ab2ef120b194b2e506. * keep changelog correction --- CHANGELOG.md | 3 - certbot/crypto_util.py | 53 ++--- certbot/ocsp.py | 197 ++++-------------- certbot/tests/ocsp_test.py | 174 ++-------------- certbot/tests/testdata/google_certificate.pem | 41 ---- .../testdata/google_issuer_certificate.pem | 26 --- certbot/util.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 40 ++-- .../pieces/dependency-requirements.txt | 40 ++-- 9 files changed, 125 insertions(+), 451 deletions(-) delete mode 100644 certbot/tests/testdata/google_certificate.pem delete mode 100644 certbot/tests/testdata/google_issuer_certificate.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index c119323cc..140d5df40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,6 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Avoid reprocessing challenges that are already validated when a certificate is issued. -* If possible, Certbot uses built-in support for OCSP from recent cryptography - versions instead of the OpenSSL binary: as a consequence Certbot does not need - the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 66c68eb38..c4a389cd5 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -19,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey # https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore @@ -226,7 +226,7 @@ def verify_renewable_cert(renewable_cert): def verify_renewable_cert_sig(renewable_cert): - """Verifies the signature of a `.storage.RenewableCert` object. + """ Verifies the signature of a `.storage.RenewableCert` object. :param `.storage.RenewableCert` renewable_cert: cert to verify @@ -239,8 +239,22 @@ def verify_renewable_cert_sig(renewable_cert): cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): - verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, - cert.signature_hash_algorithm) + warnings.simplefilter("ignore") + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) @@ -248,37 +262,6 @@ def verify_renewable_cert_sig(renewable_cert): raise errors.Error(error_str) -def verify_signed_payload(public_key, signature, payload, signature_hash_algorithm): - """Check the signature of a payload. - - :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature - :param bytes signature: the signature bytes - :param bytes payload: the payload bytes - :param cryptography.hazmat.primitives.hashes.HashAlgorithm - signature_hash_algorithm: algorithm used to hash the payload - - :raises InvalidSignature: If signature verification fails. - :raises errors.Error: If public key type is not supported - """ - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if isinstance(public_key, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = public_key.verifier( # type: ignore - signature, PKCS1v15(), signature_hash_algorithm - ) - verifier.update(payload) - verifier.verify() - elif isinstance(public_key, EllipticCurvePublicKey): - verifier = public_key.verifier( - signature, ECDSA(signature_hash_algorithm) - ) - verifier.update(payload) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") - - def verify_cert_matches_priv_key(cert_path, key_path): """ Verifies that the private key and cert match. diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 0abfd3c23..049e14827 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -1,79 +1,53 @@ """Tools for checking certificate revocation.""" import logging import re -from datetime import datetime, timedelta + from subprocess import Popen, PIPE -try: - # Only cryptography>=2.5 has ocsp module - # and signature_hash_algorithm attribute in OCSPResponse class - from cryptography.x509 import ocsp # pylint: disable=import-error - getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') -except (ImportError, AttributeError): # pragma: no cover - ocsp = None # type: ignore -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature -import requests - -from acme.magic_typing import Optional, Tuple # pylint: disable=unused-import, no-name-in-module -from certbot import crypto_util from certbot import errors from certbot import util logger = logging.getLogger(__name__) - class RevocationChecker(object): - """This class figures out OCSP checking on this system, and performs it.""" + "This class figures out OCSP checking on this system, and performs it." - def __init__(self, enforce_openssl_binary_usage=False): + def __init__(self): self.broken = False - self.use_openssl_binary = enforce_openssl_binary_usage or not ocsp - if self.use_openssl_binary: - if not util.exe_exists("openssl"): - logger.info("openssl not installed, can't check revocation") - self.broken = True - return + if not util.exe_exists("openssl"): + logger.info("openssl not installed, can't check revocation") + self.broken = True + return + + # New versions of openssl want -header var=val, old ones want -header var val + test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], + stdout=PIPE, stderr=PIPE, universal_newlines=True) + _out, err = test_host_format.communicate() + if "Missing =" in err: + self.host_args = lambda host: ["Host=" + host] + else: + self.host_args = lambda host: ["Host", host] - # New versions of openssl want -header var=val, old ones want -header var val - test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], - stdout=PIPE, stderr=PIPE, universal_newlines=True) - _out, err = test_host_format.communicate() - if "Missing =" in err: - self.host_args = lambda host: ["Host=" + host] - else: - self.host_args = lambda host: ["Host", host] def ocsp_revoked(self, cert_path, chain_path): - # type: (str, str) -> bool """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call :param str cert_path: Path to certificate :param str chain_path: Path to intermediate cert + :rtype bool or None: :returns: True if revoked; False if valid or the check failed - :rtype: bool """ if self.broken: return False - url, host = _determine_ocsp_server(cert_path) - if not host or not url: + + url, host = self.determine_ocsp_server(cert_path) + if not host: return False - - if self.use_openssl_binary: - return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) - else: - return _check_ocsp_cryptography(cert_path, chain_path, url) - - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): - # type: (str, str, str, str) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -91,117 +65,33 @@ class RevocationChecker(object): except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False + return _translate_ocsp_query(cert_path, output, err) -def _determine_ocsp_server(cert_path): - # type: (str) -> Tuple[Optional[str], Optional[str]] - """Extract the OCSP server host from a certificate. + def determine_ocsp_server(self, cert_path): + """Extract the OCSP server host from a certificate. - :param str cert_path: Path to the cert we're checking OCSP for - :rtype tuple: - :returns: (OCSP server URL or None, OCSP server host or None) + :param str cert_path: Path to the cert we're checking OCSP for + :rtype tuple: + :returns: (OCSP server URL or None, OCSP server host or None) - """ - with open(cert_path, 'rb') as file_handler: - cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) - try: - extension = cert.extensions.get_extension_for_class(x509.AuthorityInformationAccess) - ocsp_oid = x509.AuthorityInformationAccessOID.OCSP - descriptions = [description for description in extension.value - if description.access_method == ocsp_oid] - - url = descriptions[0].access_location.value - except (x509.ExtensionNotFound, IndexError): - logger.info("Cannot extract OCSP URI from %s", cert_path) - return None, None - - url = url.rstrip() - host = url.partition("://")[2].rstrip("/") - - if host: - return url, host - else: - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) - return None, None - - -def _check_ocsp_cryptography(cert_path, chain_path, url): - # type: (str, str, str) -> bool - # Retrieve OCSP response - with open(chain_path, 'rb') as file_handler: - issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) - with open(cert_path, 'rb') as file_handler: - cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) - builder = ocsp.OCSPRequestBuilder() - builder = builder.add_certificate(cert, issuer, hashes.SHA1()) - request = builder.build() - request_binary = request.public_bytes(serialization.Encoding.DER) - response = requests.post(url, data=request_binary, - headers={'Content-Type': 'application/ocsp-request'}) - if response.status_code != 200: - logger.info("OCSP check failed for %s (are we offline?)", cert_path) - return False - response_ocsp = ocsp.load_der_ocsp_response(response.content) - - # Check OCSP response validity - if response_ocsp.response_status != ocsp.OCSPResponseStatus.SUCCESSFUL: - logger.error("Invalid OCSP response status for %s: %s", - cert_path, response_ocsp.response_status) - return False - - # Check OCSP signature - try: - _check_ocsp_response(response_ocsp, request, issuer) - except UnsupportedAlgorithm as e: - logger.error(str(e)) - except errors.Error as e: - logger.error(str(e)) - except InvalidSignature: - logger.error('Invalid signature on OCSP response for %s', cert_path) - except AssertionError as error: - logger.error('Invalid OCSP response for %s: %s.', cert_path, str(error)) - else: - # Check OCSP certificate status - logger.debug("OCSP certificate status for %s is: %s", - cert_path, response_ocsp.certificate_status) - return response_ocsp.certificate_status == ocsp.OCSPCertStatus.REVOKED - - return False - - -def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): - """Verify that the OCSP is valid for serveral criterias""" - # Assert OCSP response corresponds to the certificate we are talking about - if response_ocsp.serial_number != request_ocsp.serial_number: - raise AssertionError('the certificate in response does not correspond ' - 'to the certificate in request') - - # Assert signature is valid - _check_ocsp_response_signature(response_ocsp, issuer_cert) - - # Assert issuer in response is the expected one - if (not isinstance(response_ocsp.hash_algorithm, type(request_ocsp.hash_algorithm)) - or response_ocsp.issuer_key_hash != request_ocsp.issuer_key_hash - or response_ocsp.issuer_name_hash != request_ocsp.issuer_name_hash): - raise AssertionError('the issuer does not correspond to issuer of the certificate.') - - # Assert nextUpdate is in the future, and that thisUpdate is not too old - if response_ocsp.next_update: - if response_ocsp.next_update < datetime.now() - timedelta(minutes=5): - raise AssertionError('next update is in the past.') - interval = response_ocsp.next_update - response_ocsp.this_update - if datetime.now() - response_ocsp.this_update > interval + timedelta(minutes=5): - raise AssertionError('this update is too old.') - - -def _check_ocsp_response_signature(response_ocsp, issuer_cert): - """Verify an OCSP response signature against certificate issuer""" - # Following line may raise UnsupportedAlgorithm - chosen_hash = response_ocsp.signature_hash_algorithm - crypto_util.verify_signed_payload(issuer_cert.public_key(), response_ocsp.signature, - response_ocsp.tbs_response_bytes, chosen_hash) + """ + try: + url, _err = util.run_script( + ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], + log=logger.debug) + except errors.SubprocessError: + logger.info("Cannot extract OCSP URI from %s", cert_path) + return None, None + url = url.rstrip() + host = url.partition("://")[2].rstrip("/") + if host: + return url, host + else: + logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + return None, None def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): """Parse openssl's weird output to work out what it means.""" @@ -212,7 +102,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): warning = good.group(1) if good else None - if ("Response verify OK" not in ocsp_errors) or (good and warning) or unknown: + if (not "Response verify OK" in ocsp_errors) or (good and warning) or unknown: logger.info("Revocation status for %s is unknown", cert_path) logger.debug("Uncertain output:\n%s\nstderr:\n%s", ocsp_output, ocsp_errors) return False @@ -225,5 +115,6 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): return True else: logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", - ocsp_output, ocsp_errors) + ocsp_output, ocsp_errors) return False + diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index ad3467e5a..55cd24adb 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -1,33 +1,18 @@ """Tests for ocsp.py""" # pylint: disable=protected-access -import unittest -from datetime import datetime, timedelta -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature -from cryptography import x509 -try: - # Only cryptography>=2.5 has ocsp module - # and signature_hash_algorithm attribute in OCSPResponse class - from cryptography.x509 import ocsp as ocsp_lib # pylint: disable=import-error - getattr(ocsp_lib.OCSPResponse, 'signature_hash_algorithm') -except (ImportError, AttributeError): # pragma: no cover - ocsp_lib = None # type: ignore +import unittest + import mock from certbot import errors -from certbot.tests import util as test_util out = """Missing = in header key=value ocsp: Use -help for summary. """ +class OCSPTest(unittest.TestCase): -class OCSPTestOpenSSL(unittest.TestCase): - """ - OCSP revokation tests using OpenSSL binary. - """ def setUp(self): from certbot import ocsp @@ -37,7 +22,7 @@ class OCSPTestOpenSSL(unittest.TestCase): mock_communicate.communicate.return_value = (None, out) mock_popen.return_value = mock_communicate mock_exists.return_value = True - self.checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) + self.checker = ocsp.RevocationChecker() def tearDown(self): pass @@ -52,23 +37,23 @@ class OCSPTestOpenSSL(unittest.TestCase): mock_exists.return_value = True from certbot import ocsp - checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) + checker = ocsp.RevocationChecker() self.assertEqual(mock_popen.call_count, 1) self.assertEqual(checker.host_args("x"), ["Host=x"]) mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) - checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) + checker = ocsp.RevocationChecker() self.assertEqual(checker.host_args("x"), ["Host", "x"]) self.assertEqual(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 - checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) + checker = ocsp.RevocationChecker() self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) self.assertEqual(checker.broken, True) - @mock.patch('certbot.ocsp._determine_ocsp_server') + @mock.patch('certbot.ocsp.RevocationChecker.determine_ocsp_server') @mock.patch('certbot.util.run_script') def test_ocsp_revoked(self, mock_run, mock_determine): self.checker.broken = True @@ -86,12 +71,21 @@ class OCSPTestOpenSSL(unittest.TestCase): self.assertEqual(self.checker.ocsp_revoked("x", "y"), False) self.assertEqual(mock_run.call_count, 2) - def test_determine_ocsp_server(self): - cert_path = test_util.vector_path('google_certificate.pem') - from certbot import ocsp - result = ocsp._determine_ocsp_server(cert_path) - self.assertEqual(('http://ocsp.digicert.com', 'ocsp.digicert.com'), result) + @mock.patch('certbot.ocsp.logger.info') + @mock.patch('certbot.util.run_script') + def test_determine_ocsp_server(self, mock_run, mock_info): + uri = "http://ocsp.stg-int-x1.letsencrypt.org/" + host = "ocsp.stg-int-x1.letsencrypt.org" + mock_run.return_value = uri, "" + self.assertEqual(self.checker.determine_ocsp_server("beep"), (uri, host)) + mock_run.return_value = "ftp:/" + host + "/", "" + self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) + self.assertEqual(mock_info.call_count, 1) + + c = "confusion" + mock_run.side_effect = errors.SubprocessError(c) + self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) @mock.patch('certbot.ocsp.logger') @mock.patch('certbot.util.run_script') @@ -118,129 +112,6 @@ class OCSPTestOpenSSL(unittest.TestCase): self.assertEqual(mock_log.info.call_count, 1) -@unittest.skipIf(not ocsp_lib, - reason='This class tests functionalities available only on cryptography>=2.5.0') -class OSCPTestCryptography(unittest.TestCase): - """ - OCSP revokation tests using Cryptography >= 2.4.0 - """ - - def setUp(self): - from certbot import ocsp - self.checker = ocsp.RevocationChecker() - self.cert_path = test_util.vector_path('google_certificate.pem') - self.chain_path = test_util.vector_path('google_issuer_certificate.pem') - - @mock.patch('certbot.ocsp._determine_ocsp_server') - @mock.patch('certbot.ocsp._check_ocsp_cryptography') - def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): - mock_determine.return_value = ('http://example.com', 'example.com') - self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') - - @mock.patch('certbot.ocsp.requests.post') - @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') - def test_revoke(self, mock_ocsp_response, mock_post): - with mock.patch('certbot.ocsp.crypto_util.verify_signed_payload'): - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertTrue(revoked) - - @mock.patch('certbot.ocsp.crypto_util.verify_signed_payload') - @mock.patch('certbot.ocsp.requests.post') - @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') - def test_revoke_resiliency(self, mock_ocsp_response, mock_post, mock_check): - # Server return an invalid HTTP response - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=400) - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # OCSP response in invalid - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED) - mock_post.return_value = mock.Mock(status_code=200) - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # OCSP response is valid, but certificate status is unknown - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # The OCSP response says that the certificate is revoked, but certificate - # does not contain the OCSP extension. - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - with mock.patch('cryptography.x509.Extensions.get_extension_for_class', - side_effect=x509.ExtensionNotFound( - 'Not found', x509.AuthorityInformationAccessOID.OCSP)): - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # Valid response, OCSP extension is present, - # but OCSP response uses an unsupported signature. - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - mock_check.side_effect = UnsupportedAlgorithm('foo') - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # And now, the signature itself is invalid. - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - mock_check.side_effect = InvalidSignature('foo') - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - # Finally, assertion error on OCSP response validity - mock_ocsp_response.return_value = _construct_mock_ocsp_response( - ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) - mock_post.return_value = mock.Mock(status_code=200) - mock_check.side_effect = AssertionError('foo') - revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) - - self.assertFalse(revoked) - - -def _construct_mock_ocsp_response(certificate_status, response_status): - cert = x509.load_pem_x509_certificate( - test_util.load_vector('google_certificate.pem'), default_backend()) - issuer = x509.load_pem_x509_certificate( - test_util.load_vector('google_issuer_certificate.pem'), default_backend()) - builder = ocsp_lib.OCSPRequestBuilder() - builder = builder.add_certificate(cert, issuer, hashes.SHA1()) - request = builder.build() - - return mock.Mock( - response_status=response_status, - certificate_status=certificate_status, - serial_number=request.serial_number, - issuer_key_hash=request.issuer_key_hash, - issuer_name_hash=request.issuer_name_hash, - hash_algorithm=hashes.SHA1(), - next_update=datetime.now() + timedelta(days=1), - this_update=datetime.now() - timedelta(days=1), - signature_algorithm_oid=x509.oid.SignatureAlgorithmOID.RSA_WITH_SHA1, - ) - - # pylint: disable=line-too-long openssl_confused = ("", """ /etc/letsencrypt/live/example.org/cert.pem: good @@ -294,6 +165,5 @@ revoked """, """Response verify OK""") - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/testdata/google_certificate.pem b/certbot/tests/testdata/google_certificate.pem deleted file mode 100644 index c26fea0b1..000000000 --- a/certbot/tests/testdata/google_certificate.pem +++ /dev/null @@ -1,41 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1 -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk -IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy -MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB -BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF -Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG -A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD -VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA -xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ -w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj -/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE -sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ -xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM -fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag -re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG -A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE -AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0 -oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy -LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy -dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB -FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF -BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS -BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0 -U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA -MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY -BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT -Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg -U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE -AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51 -vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV -CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN -8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG -eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v -ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN -HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB -Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk -myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq -WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik= ------END CERTIFICATE----- diff --git a/certbot/tests/testdata/google_issuer_certificate.pem b/certbot/tests/testdata/google_issuer_certificate.pem deleted file mode 100644 index 50db47bc4..000000000 --- a/certbot/tests/testdata/google_issuer_certificate.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEXDCCA0SgAwIBAgINAeOpMBz8cgY4P5pTHTANBgkqhkiG9w0BAQsFADBMMSAw -HgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFs -U2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEy -MTUwMDAwNDJaMFQxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3Qg -U2VydmljZXMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzMw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKUkvqHv/OJGuo2nIYaNVW -XQ5IWi01CXZaz6TIHLGp/lOJ+600/4hbn7vn6AAB3DVzdQOts7G5pH0rJnnOFUAK -71G4nzKMfHCGUksW/mona+Y2emJQ2N+aicwJKetPKRSIgAuPOB6Aahh8Hb2XO3h9 -RUk2T0HNouB2VzxoMXlkyW7XUR5mw6JkLHnA52XDVoRTWkNty5oCINLvGmnRsJ1z -ouAqYGVQMc/7sy+/EYhALrVJEA8KbtyX+r8snwU5C1hUrwaW6MWOARa8qBpNQcWT -kaIeoYvy/sGIJEmjR0vFEwHdp1cSaWIr6/4g72n7OqXwfinu7ZYW97EfoOSQJeAz -AgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH -AwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHfCuFCa -Z3Z2sS3ChtCDoH6mfrpLMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYu -MDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdv -b2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dz -cjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYc -aHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEA -HLeJluRT7bvs26gyAZ8so81trUISd7O45skDUmAge1cnxhG1P2cNmSxbWsoiCt2e -ux9LSD+PAj2LIYRFHW31/6xoic1k4tbWXkDCjir37xTTNqRAMPUyFRWSdvt+nlPq -wnb8Oa2I/maSJukcxDjNSfpDh/Bd1lZNgdd/8cLdsE3+wypufJ9uXO1iQpnh9zbu -FIwsIONGl1p3A8CgxkqI/UAih3JaGOqcpcdaCIzkBaR9uYQ1X4k2Vg5APRLouzVy -7a8IVk6wuy6pm+T7HT4LY8ibS5FEZlfAFLSW8NwsVz9SBK2Vqn1N0PIMn5xA6NZV -c7o835DLAFshEWfC7TIe3g== ------END CERTIFICATE----- diff --git a/certbot/util.py b/certbot/util.py index 097593e9d..416075ce8 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -62,7 +62,7 @@ def run_script(params, log=logger.error): """Run the script with the given params. :param list params: List of parameters to pass to Popen - :param callable log: Logger method to use for errors + :param logging.Logger log: Logger to use for errors """ try: diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8aaa1e3fb..48ddfb570 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1034,26 +1034,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.5 \ - --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ - --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ - --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ - --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ - --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ - --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ - --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ - --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ - --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ - --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ - --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ - --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ - --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ - --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ - --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ - --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ - --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ - --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ - --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index dff57dfd5..1fac78836 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -60,26 +60,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.5 \ - --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ - --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ - --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ - --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ - --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ - --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ - --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ - --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ - --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ - --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ - --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ - --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ - --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ - --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ - --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ - --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ - --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ - --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ - --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 +cryptography==2.2.2 \ + --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ + --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ + --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ + --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ + --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ + --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ + --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ + --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ + --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ + --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ + --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ + --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ + --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ + --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ + --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ + --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ + --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ + --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ + --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 From 67828562a0349421488c4fce1e1953c34f4e1b53 Mon Sep 17 00:00:00 2001 From: J0WI Date: Thu, 7 Feb 2019 18:06:04 +0100 Subject: [PATCH 561/639] Upgrade to Alpine 3.9 (#6743) Alpine 3.9 comes with OpenSSL 1.1.1. --- CHANGELOG.md | 2 ++ Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 140d5df40..8022e90cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* Certbot's official Docker images are now based on Alpine Linux 3.9 rather + than 3.7. The new version comes with OpenSSL 1.1.1. * Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support on 2.x branch is maintained). * Apache plugin now attempts to configure all VirtualHosts matching requested diff --git a/Dockerfile b/Dockerfile index f3626dc8d..8f434e89c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2-alpine3.7 +FROM python:2-alpine3.9 ENTRYPOINT [ "certbot" ] EXPOSE 80 443 @@ -12,7 +12,7 @@ COPY certbot src/certbot RUN apk add --no-cache --virtual .certbot-deps \ libffi \ - libssl1.0 \ + libssl1.1 \ openssl \ ca-certificates \ binutils From 432e18d943782b308f77eb8225ba03617cb12c6d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Feb 2019 12:40:45 -0800 Subject: [PATCH 562/639] Revert "Call atexit handlers before test tearDown to remove errors on Windows (#6667)" (#6752) This reverts commit ca25d1b66a24e351739a6b4dbfadd8f471857721. --- certbot/tests/lock_test.py | 7 ++----- certbot/tests/util.py | 31 +++++++++++++++---------------- certbot/tests/util_test.py | 24 +++++++++++++++--------- certbot/util.py | 6 +++--- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2ade47827..aa82701f3 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -10,6 +10,7 @@ from certbot import errors from certbot.tests import util as test_util +@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -24,6 +25,7 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) +@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod @@ -35,7 +37,6 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') - @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -53,7 +54,6 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) - @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) locked_repr = repr(lock_file) @@ -71,7 +71,6 @@ class LockFileTest(test_util.TempDirTestCase): self.assertTrue(lock_file.__class__.__name__ in lock_repr) self.assertTrue(self.lock_path in lock_repr) - @test_util.broken_on_windows def test_race(self): should_delete = [True, False] stat = os.stat @@ -87,13 +86,11 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) - @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @test_util.broken_on_windows @mock.patch('certbot.compat.fcntl.lockf') def test_unexpected_lockf_err(self, mock_lockf): msg = 'hi there' diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 38a9075c1..8c5db2c2f 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,15 +3,14 @@ .. warning:: This module is not part of the public API. """ -import logging import multiprocessing import os import pkg_resources import shutil -import stat import tempfile import unittest import sys +import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -329,22 +328,22 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # Cleanup opened resources after a test. This is usually done through atexit handlers in - # Certbot, but during tests, atexit will not run registered functions before tearDown is - # called and instead will run them right before the entire test process exits. - # It is a problem on Windows, that does not accept to clean resources before closing them. - logging.shutdown() - util._release_locks() # pylint: disable=protected-access - - def handle_rw_files(_, path, __): - """Handle read-only files, that will fail to be removed on Windows.""" - os.chmod(path, stat.S_IWRITE) - os.remove(path) - shutil.rmtree(self.tempdir, onerror=handle_rw_files) - + # On Windows we have various files which are not correctly closed at the time of tearDown. + # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. + def onerror_handler(_, path, excinfo): + """On error handler""" + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) + shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): - """Test class which sets up a NamespaceConfig object.""" + """Test class which sets up a NamespaceConfig object. + + """ def setUp(self): super(ConfigTestCase, self).setUp() self.config = configuration.NamespaceConfig( diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 81d0629c8..6685b88c6 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -191,12 +191,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_wrong_mode(self): os.chmod(self.tempdir, 0o400) - try: - self.assertFalse(self._call(0o600)) - finally: - # Without proper write permissions, Windows is unable to delete a folder, - # even with admin permissions. Write access must be explicitly set first. - os.chmod(self.tempdir, 0o700) + self.assertFalse(self._call(0o600)) class UniqueFileTest(test_util.TempDirTestCase): @@ -282,9 +277,20 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for f, _ in items: f.close() - def test_failure(self): - with mock.patch("certbot.util.os.open", side_effect=OSError(errno.EIO)): - self.assertRaises(OSError, self._call, "wow") + @mock.patch("certbot.util.os.fdopen") + def test_failure(self, mock_fdopen): + err = OSError("whoops") + err.errno = errno.EIO + mock_fdopen.side_effect = err + self.assertRaises(OSError, self._call, "wow") + + @mock.patch("certbot.util.os.fdopen") + def test_subsequent_failure(self, mock_fdopen): + self._call("wow") + err = OSError("whoops") + err.errno = errno.EIO + mock_fdopen.side_effect = err + self.assertRaises(OSError, self._call, "wow") class SafelyRemoveTest(test_util.TempDirTestCase): diff --git a/certbot/util.py b/certbot/util.py index 416075ce8..d7c542465 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -142,7 +142,6 @@ def _release_locks(): except: # pylint: disable=bare-except msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) logger.debug(msg, exc_info=True) - _LOCKS.clear() def set_up_core_dir(directory, mode, uid, strict): @@ -226,8 +225,9 @@ def safe_open(path, mode="w", chmod=None, buffering=None): fdopen_args = () # type: Union[Tuple[()], Tuple[int]] if buffering is not None: fdopen_args = (buffering,) - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) - return os.fdopen(fd, mode, *fdopen_args) + return os.fdopen( + os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), + mode, *fdopen_args) def _unique_file(path, filename_pat, count, chmod, mode): From ee3c14cbab9ed2df7ef69aca9171936918f95438 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Feb 2019 13:20:30 -0800 Subject: [PATCH 563/639] Update changelog for 0.31.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8022e90cc..30f813ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.31.0 - master +## 0.31.0 - 2019-02-07 ### Added From 75499277be6699fd5a9b884837546391950a3ec9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Feb 2019 13:27:10 -0800 Subject: [PATCH 564/639] Release 0.31.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 84 ++++-------------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 6 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 6 +- certbot-dns-dnsmadeeasy/setup.py | 6 +- certbot-dns-gehirn/setup.py | 6 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 6 +- certbot-dns-luadns/setup.py | 6 +- certbot-dns-nsone/setup.py | 6 +- certbot-dns-ovh/setup.py | 6 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 6 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 20 ++--- letsencrypt-auto | 84 ++++-------------- letsencrypt-auto-source/certbot-auto.asc | 16 ++-- letsencrypt-auto-source/letsencrypt-auto | 26 +++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++--- 26 files changed, 112 insertions(+), 216 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index eac3974fa..1a7d54704 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.31.0.dev0' +version = '0.31.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 14d6cacb6..e7270cb34 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index a79a9c5ae..03c2994e3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.2" +LE_AUTO_VERSION="0.31.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ @@ -1140,9 +1088,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ @@ -1232,18 +1180,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.2 \ - --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ - --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e -acme==0.30.2 \ - --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ - --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 -certbot-apache==0.30.2 \ - --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ - --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 -certbot-nginx==0.30.2 \ - --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ - --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 +certbot==0.31.0 \ + --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ + --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 +acme==0.31.0 \ + --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ + --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf +certbot-apache==0.31.0 \ + --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ + --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd +certbot-nginx==0.31.0 \ + --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ + --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index f519ed422..649054661 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index ff33293fe..ab98e04e1 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 1a6f900d8..e52d9ac06 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 2f7fa37d6..f918bf622 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 1a2ce5d92..a4225aec0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0a99f452d..732ec768e 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index f4a75379c..f6356e2a0 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,12 +2,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.1.22', 'mock', 'setuptools', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index c99ad38aa..e2d335cbd 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 31c2c20bc..52af79c4f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,12 +1,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', 'mock', 'setuptools', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 31472e8cf..cee34eb9f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 41b99cc73..ed4dbedfd 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 5b3329568..cd0214429 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,13 +2,13 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index edf7b6ba6..0faa2d4a0 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 69c2c7ed3..3f5dfe76a 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 4ebfc6e1d..098dd9dad 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,12 +2,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.31.0.dev0', - 'certbot>=0.31.0.dev0', + 'acme>=0.31.0', + 'certbot>=0.31.0', 'dns-lexicon>=2.1.23', 'mock', 'setuptools', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 70e11f62b..2b4941147 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0.dev0' +version = '0.31.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index bf68034c8..51c53a9dd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.31.0.dev0' +__version__ = '0.31.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index cd6d431b3..e95b2fcd5 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.30.2 + "". (default: CertbotACMEClient/0.31.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -351,8 +351,9 @@ revoke: Specify reason for revoking certificate. (default: unspecified) --delete-after-revoke - Delete certificates after revoking them. (default: - None) + Delete certificates after revoking them, along with + all previous and later versions of those certificates. + (default: None) --no-delete-after-revoke Do not delete certificates after revoking them. This option should be used with caution because the 'renew' @@ -479,10 +480,9 @@ apache: Apache Web Server plugin --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -496,16 +496,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index a79a9c5ae..03c2994e3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.2" +LE_AUTO_VERSION="0.31.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ @@ -1140,9 +1088,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ @@ -1232,18 +1180,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.2 \ - --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ - --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e -acme==0.30.2 \ - --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ - --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 -certbot-apache==0.30.2 \ - --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ - --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 -certbot-nginx==0.30.2 \ - --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ - --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 +certbot==0.31.0 \ + --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ + --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 +acme==0.31.0 \ + --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ + --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf +certbot-apache==0.31.0 \ + --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ + --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd +certbot-nginx==0.31.0 \ + --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ + --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index a60ccd8bb..d95f9abcb 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxLcw8ACgkQTRfJlc2X -dfK35gf+PoxtrJJIjvybNqd3lb8HOg2ntIVmXcYJGuuUo6m09fzai+XI6cOm5Dpu -l2D5OrbLqmez8tYkCkEWHV0OfwyVWw+m8T3sXlcrv14eA1RfgMnZ+cmmlpDskzHU -EOtaXo1/IkLDwBRrsl8IUbwD2XxbjuLsA2Sevoa59NlfTXJUApfAzohl3epRiJjB -gugdqcsfjRRAqQqOz+iJCKBCWSTIrr/g6Y9aZu9V93t/WDSLRFjehxO1GQrLnCnX -17JGlr0/AXd67jOKS1OWmORPPAFfLIXezUMtgrz5hE7T5UviaUu9ySV8UCxq1N79 -cfSBb/HIUxZ0wf1CkTUMRFQpA7cGtw== -=cNcT +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxcop8ACgkQTRfJlc2X +dfIbZwf/faKu7IjLi0qFQ+kw8zaAnV47JDgfWqbR5GSdwWPqld+QyHlcRfPgwYma +fKj9+g/FvPNPSfjHRRCoFrYvpZ4lZ+f4HPN9+OjydfM77rdDhVDwzs8dbKIk02yU +0IEJhXj5Q9hF3TSDZcyXAJdBU1lz51ohtVIXelMBPmzhYPCZF47iE9/k9pApQi86 +RTji7hxPcF/n7mzXrbyTvk+kDxSdDlE0Eg9syK7XaFDBTa2lqgG8wTnMPVqhc/hm +WM/uwkzbYarjy05ffV1kM683nP0rECnHlYT38pYcT2puw2kn/QthwR5j/jB/DWSc +94Kw7BeMH651V8EaNwYIiouylnVH3A== +=U+Qh -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 48ddfb570..03c2994e3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0.dev0" +LE_AUTO_VERSION="0.31.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1180,18 +1180,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.30.2 \ - --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ - --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e -acme==0.30.2 \ - --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ - --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 -certbot-apache==0.30.2 \ - --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ - --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 -certbot-nginx==0.30.2 \ - --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ - --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 +certbot==0.31.0 \ + --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ + --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 +acme==0.31.0 \ + --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ + --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf +certbot-apache==0.31.0 \ + --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ + --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd +certbot-nginx==0.31.0 \ + --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ + --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index f5175187a4b2a398ef8639581b0b06a34afd69cd..3ecca151022170a559f05135a0b3ec86bc998e96 100644 GIT binary patch literal 256 zcmV+b0ssCTra?=}eAKv_CUSuoF#>1oCdpGMSAJlPA0}OyX15pk1>`;NPfsRIoA7989)8hXTrsI)?K2n+{4(K0$wW- z?Xt|aQ|bMz{LJj09XOy^>A@XGdSH*2KX{+1-dg&>BqH~3rK zY2Wn@jjzL2vZ@UiY;SJVRrZ9^Fl|&kDuWGZ&Hbdn^_%ay=HGR_ZI}eiM&euM+yK@f-}Y-l4zbB>$FT0s`7CQBL5;Qw4P+7T};do=%aiAKY_QqraO&Q28SPJ=~B4FsLptn;Y%xS zbn`p(WozSZX|IcfjH8k)+!NL|y;AE>!UN&Oq)?#jR(u}Y6+7~5EDy2pR~+twU3%mB zs>Id05oxeYXG6CWTI*3n<}(!x9|qK)DpgGWW?xS~uV23*oWjVp6Y4hey9IeMZ`Uj7 G&Tzlk=6^>3 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 80249cd9a..514d3271b 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.30.2 \ - --hash=sha256:e411b72fa86eec1018e6de28e649e8c9c71191a7431dcc77f207b57ca9484c11 \ - --hash=sha256:534487cb552ced8e47948ba3d2e7ca12c3a439133fc609485012b1a02fc7776e -acme==0.30.2 \ - --hash=sha256:68982576492dfa99c7e2be0fce4371adc9344740b05420ce0ab53238d2bb9b3b \ - --hash=sha256:295a5b7fce9f908e6e5cff8c40be1a3daf3e1ebabd2e139a4c87274e68eeb8f2 -certbot-apache==0.30.2 \ - --hash=sha256:3b7fa4e59772da7c9975ef2a49ceff157c9d7cb31eb9475928b5986d89701a3a \ - --hash=sha256:32fa915a8a51810fdfe828ac1361da4425c231d7384891e49e6338e4741464b2 -certbot-nginx==0.30.2 \ - --hash=sha256:7dc785f6f0c0c57b19cea8d98f9ea8feef53945613967b52c9348c81327010e2 \ - --hash=sha256:6ba4dd772d0c7cdfb3383ca325b35639e01ac9e142e4baa6445cd85c7fb59552 +certbot==0.31.0 \ + --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ + --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 +acme==0.31.0 \ + --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ + --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf +certbot-apache==0.31.0 \ + --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ + --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd +certbot-nginx==0.31.0 \ + --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ + --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 From 917dc16b302d7bac421c19472f1554c8e6991cab Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Feb 2019 13:27:12 -0800 Subject: [PATCH 565/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f813ac6..8f0b2d04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.32.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.31.0 - 2019-02-07 ### Added From 381d097895182b2bfd3ef5a9ecd17d7c7bea6ea4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Feb 2019 13:27:13 -0800 Subject: [PATCH 566/639] Bump version to 0.32.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/local-oldest-requirements.txt | 4 ++-- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/local-oldest-requirements.txt | 4 ++-- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/local-oldest-requirements.txt | 4 ++-- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/local-oldest-requirements.txt | 4 ++-- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/local-oldest-requirements.txt | 4 ++-- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/local-oldest-requirements.txt | 4 ++-- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/local-oldest-requirements.txt | 4 ++-- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/local-oldest-requirements.txt | 4 ++-- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/local-oldest-requirements.txt | 4 ++-- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 29 files changed, 38 insertions(+), 38 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 1a7d54704..6ec226d26 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.31.0' +version = '0.32.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index e7270cb34..52528a536 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 649054661..e1c82b063 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index ab98e04e1..d6a16f468 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-cloudxns/local-oldest-requirements.txt +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index e52d9ac06..f0f2446e1 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f918bf622..c61f05339 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index a4225aec0..7d39a10d0 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 732ec768e..8a5f234b5 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/local-oldest-requirements.txt b/certbot-dns-gehirn/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-gehirn/local-oldest-requirements.txt +++ b/certbot-dns-gehirn/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index f6356e2a0..67ae1d4eb 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e2d335cbd..429702d75 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 52af79c4f..6797501b9 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-luadns/local-oldest-requirements.txt +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index cee34eb9f..07d9decdf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-nsone/local-oldest-requirements.txt +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index ed4dbedfd..314b0a16c 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 01cbcb317..b7e6072af 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,3 +1,3 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 dns-lexicon==2.7.14 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index cd0214429..a878eac47 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 0faa2d4a0..39f35e953 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 3f5dfe76a..e915c6b86 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/local-oldest-requirements.txt b/certbot-dns-sakuracloud/local-oldest-requirements.txt index 65f5a758e..45b6b2291 100644 --- a/certbot-dns-sakuracloud/local-oldest-requirements.txt +++ b/certbot-dns-sakuracloud/local-oldest-requirements.txt @@ -1,2 +1,2 @@ --e acme[dev] --e .[dev] +acme[dev]==0.31.0 +certbot[dev]==0.31.0 diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 098dd9dad..429f960f7 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 2b4941147..f78473ada 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.31.0' +version = '0.32.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 51c53a9dd..767448a12 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.31.0' +__version__ = '0.32.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 03c2994e3..02656521f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 66c9767623a97c44974b0aafbd234915e8822002 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Feb 2019 14:59:34 -0800 Subject: [PATCH 567/639] Fix #6501 (#6761) --- CHANGELOG.md | 6 ++++-- acme/setup.py | 4 +++- setup.py | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f0b2d04d..ac0f60eb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Certbot and its acme module now depend on josepy>=1.1.0 to avoid printing the + warnings described at https://github.com/certbot/josepy/issues/13. ### Fixed @@ -20,7 +21,8 @@ Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme +* certbot More details about these changes can be found on our GitHub repo. diff --git a/acme/setup.py b/acme/setup.py index 6ec226d26..79d6d3389 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -11,7 +11,9 @@ install_requires = [ # rsa_recover_prime_factors (>=0.8) 'cryptography>=1.2.3', # formerly known as acme.jose: - 'josepy>=1.0.0', + # 1.1.0+ is required to avoid the warnings described at + # https://github.com/certbot/josepy/issues/13. + 'josepy>=1.1.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13.1', diff --git a/setup.py b/setup.py index 9e6af2d4f..14fef37f3 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,9 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2.3', # load_pem_x509_certificate - 'josepy', + # 1.1.0+ is required to avoid the warnings described at + # https://github.com/certbot/josepy/issues/13. + 'josepy>=1.1.0', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', From a0a8292ff26a2d062e75b865d9b9b10977dc1f80 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 13 Feb 2019 00:36:27 +0100 Subject: [PATCH 568/639] Correct the Content-Type used in the POST-as-GET request to retrieve a cert (#6757) --- acme/acme/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 41338e17e..76b6a7788 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -739,8 +739,7 @@ class ClientV2(ClientBase): if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self._post_as_get(body.certificate, - content_type=DER_CONTENT_TYPE).text + certificate_response = self._post_as_get(body.certificate).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() From f10f98fec536caca74017af2c51a6b4868ddc50c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Feb 2019 16:54:04 -0800 Subject: [PATCH 569/639] More carefully check for certbot --version output. (#6762) --- tests/letstest/scripts/test_leauto_upgrades.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 0c2b374f2..e08a0710c 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -23,7 +23,7 @@ fi # started failing on newer distros with newer versions of OpenSSL. INITIAL_VERSION="0.17.0" git checkout -f "v$INITIAL_VERSION" letsencrypt-auto -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep "$INITIAL_VERSION" ; then +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then echo initial installation appeared to fail exit 1 fi @@ -85,7 +85,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; fi # Create a 2nd venv at the new path to ensure we properly handle this case export VENV_PATH="/opt/eff.org/certbot/venv" - if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep "$INITIAL_VERSION" ; then + if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then echo second installation appeared to fail exit 1 fi @@ -98,7 +98,7 @@ if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python fi EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) -if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then echo upgrade appeared to fail exit 1 fi From cff8769db737f19a93fe26cd1d5247904c4b188a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 13 Feb 2019 18:37:01 +0200 Subject: [PATCH 570/639] Apache: respect CERTBOT_DOCS environment variable (#6598) Apache plugin will now use command line default values from `ApacheConfingurator.OS_DEFAULTS` instead of respective distribution override when `CERTBOT_DOCS=1` environment variable is present. Fixes: #6234 * Apache: respect CERTBOT_DOCS environment variable * Move the tests to apache plugin --- CHANGELOG.md | 3 ++ certbot-apache/certbot_apache/configurator.py | 31 +++++++++++++------ .../certbot_apache/tests/configurator_test.py | 31 +++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0f60eb8..57a895e07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Certbot and its acme module now depend on josepy>=1.1.0 to avoid printing the warnings described at https://github.com/certbot/josepy/issues/13. +* Apache plugin now respects CERTBOT_DOCS environment variable when adding + command line defaults. ### Fixed @@ -23,6 +25,7 @@ package with changes other than its version number was: * acme * certbot +* certbot-apache More details about these changes can be found on our GitHub repo. diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index efd766e63..f7f4ff925 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -92,6 +92,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ description = "Apache Web Server plugin" + if os.environ.get("CERTBOT_DOCS") == "1": + description += ( # pragma: no cover + " (Please note that the default values of the Apache plugin options" + " change depending on the operating system Certbot is run on.)" + ) OS_DEFAULTS = dict( server_root="/etc/apache2", @@ -141,28 +146,36 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # When adding, modifying or deleting command line arguments, be sure to # include the changes in the list used in method _prepare_options() to # ensure consistent behavior. - add("enmod", default=cls.OS_DEFAULTS["enmod"], + + # Respect CERTBOT_DOCS environment variable and use default values from + # base class regardless of the underlying distribution (overrides). + if os.environ.get("CERTBOT_DOCS") == "1": + DEFAULTS = ApacheConfigurator.OS_DEFAULTS + else: + # cls.OS_DEFAULTS can be distribution specific, see override classes + DEFAULTS = cls.OS_DEFAULTS + add("enmod", default=DEFAULTS["enmod"], help="Path to the Apache 'a2enmod' binary") - add("dismod", default=cls.OS_DEFAULTS["dismod"], + add("dismod", default=DEFAULTS["dismod"], help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], help="SSL vhost configuration extension") - add("server-root", default=cls.OS_DEFAULTS["server_root"], + add("server-root", default=DEFAULTS["server_root"], help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=cls.OS_DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", - default=cls.OS_DEFAULTS["challenge_location"], + default=DEFAULTS["challenge_location"], help="Directory path for challenge configuration") - add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS["handle_modules"], help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS["handle_sites"], help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=cls.OS_DEFAULTS["ctl"], + add("ctl", default=DEFAULTS["ctl"], help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ff93682b6..7c281d707 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,6 +115,37 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_docs_parser_arguments(self): + os.environ["CERTBOT_DOCS"] = "1" + from certbot_apache.configurator import ApacheConfigurator + mock_add = mock.MagicMock() + ApacheConfigurator.add_parser_arguments(mock_add) + parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", + "vhost_root", "logs_root", "challenge_location", + "handle_modules", "handle_sites", "ctl"] + exp = dict() + + for k in ApacheConfigurator.OS_DEFAULTS: + if k in parserargs: + exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + # Special cases + exp["vhost-root"] = None + exp["init-script"] = None + + found = set() + for call in mock_add.call_args_list: + # init-script is a special case: deprecated argument + if call[0][0] != "init-script": + self.assertEqual(exp[call[0][0]], call[1]['default']) + found.add(call[0][0]) + + # Make sure that all (and only) the expected values exist + self.assertEqual(len(mock_add.call_args_list), len(found)) + for e in exp: + self.assertTrue(e in found) + + del os.environ["CERTBOT_DOCS"] + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use from certbot_apache.entrypoint import OVERRIDE_CLASSES for cls in OVERRIDE_CLASSES.values(): From acc0b1e773de076e71f628b0b64e69f85fc9a39b Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 14 Feb 2019 19:43:27 +0100 Subject: [PATCH 571/639] Fix the pebble fetch script (#6765) This PR updates and fixes `pebble-fetch.sh` considering latest improvements done on Pebble, to start a working instance. * Fix the pebble fetch script * Update pebble-fetch.sh * Update tox.ini --- tests/pebble-fetch.sh | 33 +++++++++++++++++++++++++-------- tox.ini | 1 - 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/pebble-fetch.sh b/tests/pebble-fetch.sh index b0ba08961..6b562eec2 100755 --- a/tests/pebble-fetch.sh +++ b/tests/pebble-fetch.sh @@ -2,7 +2,7 @@ # Download and run Pebble instance for integration testing set -xe -PEBBLE_VERSION=2018-11-02 +PEBBLE_VERSION=v1.0.1 # We reuse the same GOPATH-style directory than for Boulder. # Pebble does not need it, but it will make the installation consistent with Boulder's one. @@ -13,15 +13,32 @@ mkdir -p ${PEBBLEPATH} cat << UNLIKELY_EOF > "$PEBBLEPATH/docker-compose.yml" version: '3' - services: - pebble: - image: letsencrypt/pebble:${PEBBLE_VERSION} - command: pebble -strict ${PEBBLE_STRICT:-false} -dnsserver 10.77.77.1 - ports: - - 14000:14000 - environment: + pebble: + image: letsencrypt/pebble:${PEBBLE_VERSION} + command: pebble -dnsserver 10.30.50.3:8053 + environment: - PEBBLE_VA_NOSLEEP=1 + ports: + - 14000:14000 + networks: + acmenet: + ipv4_address: 10.30.50.2 + challtestsrv: + image: letsencrypt/pebble-challtestsrv:${PEBBLE_VERSION} + command: pebble-challtestsrv -defaultIPv6 "" -defaultIPv4 10.30.50.1 + ports: + - 8055:8055 + networks: + acmenet: + ipv4_address: 10.30.50.3 +networks: + acmenet: + driver: bridge + ipam: + driver: default + config: + - subnet: 10.30.50.0/24 UNLIKELY_EOF docker-compose -f "$PEBBLEPATH/docker-compose.yml" up -d pebble diff --git a/tox.ini b/tox.ini index b66e330da..b386ebc86 100644 --- a/tox.ini +++ b/tox.ini @@ -166,7 +166,6 @@ passenv = HOME GOPATH PEBBLEPATH - PEBBLE_STRICT setenv = SERVER=https://localhost:14000/dir From 583d40f5cf5bc22799eadb99b9900e4cff17b4a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Feb 2019 15:26:44 -0800 Subject: [PATCH 572/639] Pin pytest in test_sdists.sh. (#6764) * pin pytest in test_sdists.sh. * Use pip_install.py in test_tests.sh. --- tests/letstest/scripts/test_sdists.sh | 2 ++ tests/letstest/scripts/test_tests.sh | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index 0b9a91ffd..260a0acfb 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,6 +12,8 @@ export VENV_ARGS="-p $PYTHON" # setup venv tools/_venv_common.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate +# pytest is needed to run tests on some of our packages so we install a pinned version here. +tools/pip_install.py pytest # build sdists for pkg_dir in acme . $PLUGINS; do diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index e6ab836b8..d5fd6e14a 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,18 +1,20 @@ #!/bin/sh -xe -LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +REPO_ROOT="letsencrypt" +LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" +PIP_INSTALL="$REPO_ROOT/tools/pip_install.py" VENV_NAME=venv # *-auto respects VENV_PATH $LE_AUTO --os-packages-only LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate +"$PIP_INSTALL" pytest # change to an empty directory to ensure CWD doesn't affect tests cd $(mktemp -d) -pip install pytest==3.2.5 for module in $MODULES ; do echo testing $module From e40d929e804137566c0e04f87f9d5c62a7966a4a Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 15 Feb 2019 01:55:27 +0100 Subject: [PATCH 573/639] [Windows|Unix] New platform independent locking mechanism - the revenge (#6663) First PR about this issue, #6440, involved to much refactoring to ensure a correct behavior on Linux and installer plugins. This PR proposes a new implementation of a lock mechanism for Linux and Windows, without implying a refactoring. It takes strictly the existing behavior for Linux, and add the appropriate logic for Windows. The `lock` module formalizes two independant mechanism dedicated to each platform, to improve maintainability. Tests related to locking are re-activated for Windows, or definitively skipped because of irrelevancy for this platform. 6 more tests are enabled overall. * Reimplement lock file on basic level * Remove unused code * Re-activate some tests * Update doc * Reactivate tests relevant to locks in Windows. Correct a test that was not testing what is supposed to test. * Clean compat. * Move close sooner in Windows lock implementation * Add strong mypy types * Use os.name * Refactor lock mechanism logic * Enable more tests * Update lock.py * Update lock_test.py --- certbot/compat.py | 56 ---------- certbot/lock.py | 213 +++++++++++++++++++++++++++++-------- certbot/tests/lock_test.py | 57 ++++++---- certbot/tests/util_test.py | 16 +-- 4 files changed, 216 insertions(+), 126 deletions(-) diff --git a/certbot/compat.py b/certbot/compat.py index 3b5d068a6..f533f5954 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -13,13 +13,6 @@ import stat from certbot import errors -try: - # Linux specific - import fcntl # pylint: disable=import-error -except ImportError: - # Windows specific - import msvcrt # pylint: disable=import-error - UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ 'certificates', 'enhance', 'revoke', 'delete', 'register', 'unregister', 'config_changes', 'plugins'] @@ -118,55 +111,6 @@ def readline_with_timeout(timeout, prompt): return sys.stdin.readline() -def lock_file(fd): - """ - Lock the file linked to the specified file descriptor. - - :param int fd: The file descriptor of the file to lock. - - """ - if 'fcntl' in sys.modules: - # Linux specific - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - else: - # Windows specific - msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - - -def release_locked_file(fd, path): - """ - Remove, close, and release a lock file specified by its file descriptor and its path. - - :param int fd: The file descriptor of the lock file. - :param str path: The path of the lock file. - - """ - # Linux specific - # - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - try: - os.remove(path) - except OSError as err: - if err.errno == errno.EACCES: - # Windows specific - # We will not be able to remove a file before closing it. - # To avoid race conditions described for Linux, we will not delete the lockfile, - # just close it to be reused on the next Certbot call. - pass - else: - raise - finally: - os.close(fd) - - def compare_file_modes(mode1, mode2): """Return true if the two modes can be considered as equals for this platform""" if os.name != 'nt': diff --git a/certbot/lock.py b/certbot/lock.py index 3ff46518d..760a12b8f 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,15 +1,23 @@ -"""Implements file locks for locking files and directories in UNIX.""" +"""Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging import os +try: + import fcntl # pylint: disable=import-error +except ImportError: + import msvcrt # pylint: disable=import-error + POSIX_MODE = False +else: + POSIX_MODE = True -from certbot import compat from certbot import errors +from acme.magic_typing import Optional, Callable # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) def lock_dir(dir_path): + # type: (str) -> LockFile """Place a lock file on the directory at dir_path. The lock file is placed in the root of dir_path with the name @@ -27,34 +35,99 @@ def lock_dir(dir_path): class LockFile(object): - """A UNIX lock file. - - This lock file is released when the locked file is closed or the - process exits. It cannot be used to provide synchronization between - threads. It is based on the lock_file package by Martin Horcicka. - + """ + Platform independent file lock system. + LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, + instance is created, the associated file is 'locked from the point of view of the OS, + meaning that if another instance of Certbot try at the same time to acquire the same lock, + it will raise an Exception. Calling release method will release the lock, and make it + available to every other instance. + Upon exit, Certbot will also release all the locks. + This allows us to protect a file or directory from being concurrently accessed + or modified by two Certbot instances. + LockFile is platform independent: it will proceed to the appropriate OS lock mechanism + depending on Linux or Windows. """ def __init__(self, path): - """Initialize and acquire the lock file. - - :param str path: path to the file to lock - - :raises errors.LockError: if unable to acquire the lock - + # type: (str) -> None + """ + Create a LockFile instance on the given file path, and acquire lock. + :param str path: the path to the file that will hold a lock """ - super(LockFile, self).__init__() self._path = path - self._fd = None + mechanism = _UnixLockMechanism if POSIX_MODE else _WindowsLockMechanism + self._lock_mechanism = mechanism(path) self.acquire() + def __repr__(self): + # type: () -> str + repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) + if self.is_locked(): + repr_str += 'acquired>' + else: + repr_str += 'released>' + return repr_str + def acquire(self): - """Acquire the lock file. - - :raises errors.LockError: if lock is already held - :raises OSError: if unable to open or stat the lock file - + # type: () -> None """ + Acquire the lock on the file, forbidding any other Certbot instance to acquire it. + :raises errors.LockError: if unable to acquire the lock + """ + self._lock_mechanism.acquire() + + def release(self): + # type: () -> None + """ + Release the lock on the file, allowing any other Certbot instance to acquire it. + """ + self._lock_mechanism.release() + + def is_locked(self): + # type: () -> bool + """ + Check if the file is currently locked. + :return: True if the file is locked, False otherwise + """ + return self._lock_mechanism.is_locked() + + +class _BaseLockMechanism(object): + def __init__(self, path): + # type: (str) -> None + """ + Create a lock file mechanism for Unix. + :param str path: the path to the lock file + """ + self._path = path + self._fd = None # type: Optional[int] + + def is_locked(self): + # type: () -> bool + """Check if lock file is currently locked. + :return: True if the lock file is locked + :rtype: bool + """ + return self._fd is not None + + def acquire(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + def release(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + +class _UnixLockMechanism(_BaseLockMechanism): + """ + A UNIX lock file mechanism. + This lock file is released when the locked file is closed or the + process exits. It cannot be used to provide synchronization between + threads. It is based on the lock_file package by Martin Horcicka. + """ + def acquire(self): + # type: () -> None + """Acquire the lock.""" while self._fd is None: # Open the file fd = os.open(self._path, os.O_CREAT | os.O_WRONLY, 0o600) @@ -68,33 +141,29 @@ class LockFile(object): os.close(fd) def _try_lock(self, fd): - """Try to acquire the lock file without blocking. - + # type: (int) -> None + """ + Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock - """ try: - compat.lock_file(fd) + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): - logger.debug( - "A lock on %s is held by another process.", self._path) - raise errors.LockError( - "Another instance of Certbot is already running.") + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') raise def _lock_success(self, fd): - """Did we successfully grab the lock? - + # type: (int) -> bool + """ + Did we successfully grab the lock? Because this class deletes the locked file when the lock is released, it is possible another process removed and recreated the file between us opening the file and acquiring the lock. - :param int fd: file descriptor of the opened file to lock - :returns: True if the lock was successfully acquired :rtype: bool - """ try: stat1 = os.stat(self._path) @@ -108,17 +177,75 @@ class LockFile(object): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def __repr__(self): - repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) - if self._fd is None: - repr_str += 'released>' - else: - repr_str += 'acquired>' - return repr_str + def release(self): + # type: () -> None + """Remove, close, and release the lock file.""" + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(self._path) + finally: + # Following check is done to make mypy happy: it ensure that self._fd, marked + # as Optional[int] is effectively int to make it compatible with os.close signature. + if self._fd is None: # pragma: no cover + raise TypeError('Error, self._fd is None.') + try: + os.close(self._fd) + finally: + self._fd = None + + +class _WindowsLockMechanism(_BaseLockMechanism): + """ + A Windows lock file mechanism. + By default on Windows, acquiring a file handler gives exclusive access to the process + and results in an effective lock. However, it is possible to explicitly acquire the + file handler in shared access in terms of read and write, and this is done by os.open + and io.open in Python. So an explicit lock needs to be done through the call of + msvcrt.locking, that will lock the first byte of the file. In theory, it is also + possible to access a file in shared delete access, allowing other processes to delete an + opened file. But this needs also to be done explicitly by all processes using the Windows + low level APIs, and Python does not do it. As of Python 3.7 and below, Python developers + state that deleting a file opened by a process from another process is not possible with + os.open and io.open. + Consequently, mscvrt.locking is sufficient to obtain an effective lock, and the race + condition encountered on Linux is not possible on Windows, leading to a simpler workflow. + """ + def acquire(self): + """Acquire the lock""" + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + + fd = os.open(self._path, open_mode, 0o600) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except (IOError, OSError) as err: + os.close(fd) + # Anything except EACCES is unexpected. Raise directly the error in that case. + if err.errno != errno.EACCES: + raise + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') + + self._fd = fd def release(self): - """Remove, close, and release the lock file.""" + """Release the lock.""" try: - compat.release_locked_file(self._fd, self._path) + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + os.close(self._fd) + + try: + os.remove(self._path) + except OSError as e: + # If the lock file cannot be removed, it is not a big deal. + # Likely another instance is acquiring the lock we just released. + logger.debug(str(e)) finally: self._fd = None diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa82701f3..aa1de299b 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,6 +3,12 @@ import functools import multiprocessing import os import unittest +try: + import fcntl # pylint: disable=import-error,unused-import +except ImportError: + POSIX_MODE = False +else: + POSIX_MODE = True import mock @@ -10,7 +16,6 @@ from certbot import errors from certbot.tests import util as test_util -@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -18,6 +23,7 @@ class LockDirTest(test_util.TempDirTestCase): from certbot.lock import lock_dir return lock_dir(*args, **kwargs) + @test_util.broken_on_windows def test_it(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.tempdir) @@ -25,7 +31,6 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) -@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod @@ -49,6 +54,7 @@ class LockFileTest(test_util.TempDirTestCase): # Test we're still able to properly acquire and release the lock self.test_removed() + @test_util.broken_on_windows def test_contention(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.lock_path) @@ -71,6 +77,8 @@ class LockFileTest(test_util.TempDirTestCase): self.assertTrue(lock_file.__class__.__name__ in lock_repr) self.assertTrue(self.lock_path in lock_repr) + @test_util.skip_on_windows( + 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') def test_race(self): should_delete = [True, False] stat = os.stat @@ -91,27 +99,36 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.compat.fcntl.lockf') - def test_unexpected_lockf_err(self, mock_lockf): + def test_unexpected_lockf_or_locking_err(self): + if POSIX_MODE: + mocked_function = 'certbot.lock.fcntl.lockf' + else: + mocked_function = 'certbot.lock.msvcrt.locking' msg = 'hi there' - mock_lockf.side_effect = IOError(msg) - try: - self._call(self.lock_path) - except IOError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('IOError not raised') + with mock.patch(mocked_function) as mock_lock: + mock_lock.side_effect = IOError(msg) + try: + self._call(self.lock_path) + except IOError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('IOError not raised') - @mock.patch('certbot.lock.os.stat') - def test_unexpected_stat_err(self, mock_stat): + def test_unexpected_os_err(self): + if POSIX_MODE: + mock_function = 'certbot.lock.os.stat' + else: + mock_function = 'certbot.lock.msvcrt.locking' + # The only expected errno are ENOENT and EACCES in lock module. msg = 'hi there' - mock_stat.side_effect = OSError(msg) - try: - self._call(self.lock_path) - except OSError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('OSError not raised') + with mock.patch(mock_function) as mock_os: + mock_os.side_effect = OSError(msg) + try: + self._call(self.lock_path) + except OSError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('OSError not raised') if __name__ == "__main__": diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6685b88c6..c85009c62 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -2,7 +2,6 @@ import argparse import errno import os -import shutil import unittest import mock @@ -88,7 +87,6 @@ class LockDirUntilExit(test_util.TempDirTestCase): import certbot.util reload_module(certbot.util) - @test_util.broken_on_windows @mock.patch('certbot.util.logger') @mock.patch('certbot.util.atexit_register') def test_it(self, mock_register, mock_logger): @@ -100,11 +98,15 @@ class LockDirUntilExit(test_util.TempDirTestCase): self.assertEqual(mock_register.call_count, 1) registered_func = mock_register.call_args[0][0] - shutil.rmtree(subdir) - registered_func() # exception not raised - # logger.debug is only called once because the second call - # to lock subdir was ignored because it was already locked - self.assertEqual(mock_logger.debug.call_count, 1) + + from certbot import util + # Despite lock_dir_until_exit has been called twice to subdir, its lock should have been + # added only once. So we expect to have two lock references: for self.tempdir and subdir + self.assertTrue(len(util._LOCKS) == 2) # pylint: disable=protected-access + registered_func() # Exception should not be raised + # Logically, logger.debug, that would be invoked in case of unlock failure, + # should never been called. + self.assertEqual(mock_logger.debug.call_count, 0) class SetUpCoreDirTest(test_util.TempDirTestCase): From 0489ca588851430d03c45fecec56fc1ca8223c83 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 16 Feb 2019 03:51:22 +0100 Subject: [PATCH 574/639] [Windows] Fixes lock_and_call test method (#6772) The method `lock_and_call`, in `certbot.tests.util` is designed to acquire a lock on a foreign process, then execute a callable in the current process. This is done to closely reproduce the lock mechanism involved between two certbot instances that are running in parallel. This method uses the `multiprocessing` module. But its implementation in `lock_and_call` is broken for Windows: the two processes fail to communicate, leading to a deadlock. In fact, `multiprocessing` module is using the fork mechanism on Linux, and the spawn mechanism on Windows, leading to behavior inconsistencies between the two platforms. As this method is for tests, and not for production code, I did not try to make two implementations "by the book", one suitable for Windows, the other for Linux, like for the `certbot.lock` module. Instead, I use a `subprocess` approach with a trigger file allowing to coordinate the current process and the subprocess. With this, `lock_and_call` is running from the same code both on Linux and Windows. Relevant tests in the `certbot.tests.lock_test` test module are now enabled for Windows. * Implement new lock_and_call method * Reactivate tests for Windows --- certbot/tests/lock_test.py | 2 - certbot/tests/util.py | 93 ++++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa1de299b..8658443d0 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -23,7 +23,6 @@ class LockDirTest(test_util.TempDirTestCase): from certbot.lock import lock_dir return lock_dir(*args, **kwargs) - @test_util.broken_on_windows def test_it(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.tempdir) @@ -54,7 +53,6 @@ class LockFileTest(test_util.TempDirTestCase): # Test we're still able to properly acquire and release the lock self.test_removed() - @test_util.broken_on_windows def test_contention(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.lock_path) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8c5db2c2f..953d36536 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,7 +3,6 @@ .. warning:: This module is not part of the public API. """ -import multiprocessing import os import pkg_resources import shutil @@ -11,6 +10,8 @@ import tempfile import unittest import sys import warnings +import subprocess +import time from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -23,8 +24,8 @@ from six.moves import reload_module # pylint: disable=import-error from certbot import constants from certbot import interfaces from certbot import storage -from certbot import util from certbot import configuration +from certbot import util from certbot.display import util as display_util @@ -211,7 +212,7 @@ class FreezableMock(object): """ def __init__(self, frozen=False, func=None, return_value=mock.sentinel.DEFAULT): - self._frozen_set = set() if frozen else set(('freeze',)) + self._frozen_set = set() if frozen else {'freeze', } self._func = func self._mock = mock.MagicMock() if return_value != mock.sentinel.DEFAULT: @@ -340,6 +341,7 @@ class TempDirTestCase(unittest.TestCase): warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) + class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object. @@ -358,47 +360,58 @@ class ConfigTestCase(TempDirTestCase): self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.server = "https://example.com" -def lock_and_call(func, lock_path): - """Grab a lock for lock_path and call func. - - :param callable func: object to call after acquiring the lock - :param str lock_path: path to file or directory to lock +def lock_and_call(callback, path_to_lock): + """Grab a lock on path_to_lock from a foreign process and call the callback. + :param callable callback: object to call after acquiring the lock + :param str path_to_lock: path to file or directory to lock """ - # Reload module to reset internal _LOCKS dictionary + script = """\ +import os +import sys +import time +from certbot import lock + +path_to_lock = sys.argv[1] +trigger = sys.argv[2] + +if os.path.isdir(path_to_lock): + my_lock = lock.lock_dir(path_to_lock) +else: + my_lock = lock.LockFile(path_to_lock) +try: + open(trigger, 'w').close() + while os.path.exists(trigger): + time.sleep(1) +finally: + my_lock.release() +""" + # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) - # start child and wait for it to grab the lock - cv = multiprocessing.Condition() - cv.acquire() - child_args = (cv, lock_path,) - child = multiprocessing.Process(target=hold_lock, args=child_args) - child.start() - cv.wait() + workspace = tempfile.mkdtemp() + try: + tmp_script = os.path.join(workspace, 'test_script.py') + with open(tmp_script, 'w') as file_handle: + file_handle.write(script) - # call func and terminate the child - func() - cv.notify() - cv.release() - child.join() - assert child.exitcode == 0 + # Trigger file is used to coordinate current process and its subprocess. + trigger = os.path.join(workspace, 'trigger') + process = subprocess.Popen([sys.executable, tmp_script, path_to_lock, trigger]) + try: + # Poll and wait for the lock to be acquired, spotted by the trigger file creation. + while not os.path.exists(trigger): + time.sleep(1) + # Then execute the callback. + callback() + finally: + # This will trigger the lock release in subprocess. + os.remove(trigger) + process.communicate() + assert process.returncode == 0 + finally: + shutil.rmtree(workspace) -def hold_lock(cv, lock_path): # pragma: no cover - """Acquire a file lock at lock_path and wait to release it. - - :param multiprocessing.Condition cv: condition for synchronization - :param str lock_path: path to the file lock - - """ - from certbot import lock - if os.path.isdir(lock_path): - my_lock = lock.lock_dir(lock_path) - else: - my_lock = lock.LockFile(lock_path) - cv.acquire() - cv.notify() - cv.wait() - my_lock.release() def skip_on_windows(reason): """Decorator to skip permanently a test on Windows. A reason is required.""" @@ -407,6 +420,7 @@ def skip_on_windows(reason): return unittest.skipIf(sys.platform == 'win32', reason)(function) return wrapper + def broken_on_windows(function): """Decorator to skip temporarily a broken test on Windows.""" reason = 'Test is broken and ignored on windows but should be fixed.' @@ -415,9 +429,10 @@ def broken_on_windows(function): and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', reason)(function) + def temp_join(path): """ Return the given path joined to the tempdir path for the current platform Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows) """ - return os.path.join(tempfile.gettempdir(), path) + return os.path.join(tempfile.gettempdir(), path) From 209a0c4d2ce6ddd553326e10e71a012b05027bfe Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 20 Feb 2019 00:15:06 +0100 Subject: [PATCH 575/639] [Windows] Refactor lock_and_call using queues (#6778) * Refactor lock_and_call using queues * Update util.py * Replace queue by event * Add comments * Update certbot/tests/util.py Co-Authored-By: adferrand * Update certbot/tests/util.py Co-Authored-By: adferrand * Add control on timeout --- certbot/tests/util.py | 81 ++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 953d36536..289f2dd9b 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -10,8 +10,7 @@ import tempfile import unittest import sys import warnings -import subprocess -import time +from multiprocessing import Process, Event from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -25,6 +24,7 @@ from certbot import constants from certbot import interfaces from certbot import storage from certbot import configuration +from certbot import lock from certbot import util from certbot.display import util as display_util @@ -361,56 +361,49 @@ class ConfigTestCase(TempDirTestCase): self.config.server = "https://example.com" +def _handle_lock(event_in, event_out, path): + """ + Acquire a file lock on given path, then wait to release it. This worker is coordinated + using events to signal when the lock should be acquired and released. + :param multiprocessing.Event event_in: event object to signal when to release the lock + :param multiprocessing.Event event_out: event object to signal when the lock is acquired + :param path: the path to lock + """ + if os.path.isdir(path): + my_lock = lock.lock_dir(path) + else: + my_lock = lock.LockFile(path) + try: + event_out.set() + assert event_in.wait(timeout=20), 'Timeout while waiting to release the lock.' + finally: + my_lock.release() + + def lock_and_call(callback, path_to_lock): - """Grab a lock on path_to_lock from a foreign process and call the callback. + """ + Grab a lock on path_to_lock from a foreign process then execute the callback. :param callable callback: object to call after acquiring the lock :param str path_to_lock: path to file or directory to lock """ - script = """\ -import os -import sys -import time -from certbot import lock - -path_to_lock = sys.argv[1] -trigger = sys.argv[2] - -if os.path.isdir(path_to_lock): - my_lock = lock.lock_dir(path_to_lock) -else: - my_lock = lock.LockFile(path_to_lock) -try: - open(trigger, 'w').close() - while os.path.exists(trigger): - time.sleep(1) -finally: - my_lock.release() -""" # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) - workspace = tempfile.mkdtemp() - try: - tmp_script = os.path.join(workspace, 'test_script.py') - with open(tmp_script, 'w') as file_handle: - file_handle.write(script) + emit_event = Event() + receive_event = Event() + process = Process(target=_handle_lock, args=(emit_event, receive_event, path_to_lock)) + process.start() - # Trigger file is used to coordinate current process and its subprocess. - trigger = os.path.join(workspace, 'trigger') - process = subprocess.Popen([sys.executable, tmp_script, path_to_lock, trigger]) - try: - # Poll and wait for the lock to be acquired, spotted by the trigger file creation. - while not os.path.exists(trigger): - time.sleep(1) - # Then execute the callback. - callback() - finally: - # This will trigger the lock release in subprocess. - os.remove(trigger) - process.communicate() - assert process.returncode == 0 - finally: - shutil.rmtree(workspace) + # Wait confirmation that lock is acquired + assert receive_event.wait(timeout=10), 'Timeout while waiting to acquire the lock.' + # Execute the callback + callback() + # Trigger unlock from foreign process + emit_event.set() + + # Wait for process termination + process.join(timeout=10) + assert process.exitcode == 0 def skip_on_windows(reason): From bda840b3eec938829306956a0b3e7d4f691ddac2 Mon Sep 17 00:00:00 2001 From: sblondon Date: Wed, 20 Feb 2019 01:08:20 +0100 Subject: [PATCH 576/639] add version parameter when the help message is displayed (#6780) --- certbot/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 31f55711f..398124510 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -108,7 +108,7 @@ manage your account with Let's Encrypt: # This is the short help for certbot --help, where we disable argparse # altogether -HELP_USAGE = """ +HELP_AND_VERSION_USAGE = """ More detailed help: -h, --help [TOPIC] print this message, or detailed help on a topic; @@ -117,6 +117,8 @@ More detailed help: all, automation, commands, paths, security, testing, or any of the subcommands or plugins (certonly, renew, install, register, nginx, apache, standalone, webroot, etc.) + + --version print the version number """ @@ -566,7 +568,7 @@ class HelpfulArgumentParser(object): usage = SHORT_USAGE if help_arg == True: - self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_USAGE) + self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_AND_VERSION_USAGE) sys.exit(0) elif help_arg in self.COMMANDS_TOPICS: self.notify(usage + self._list_subcommands()) From 7c731599a0140868acdb609bc20345f0bd0bd3aa Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 18 Jan 2019 17:09:19 -0800 Subject: [PATCH 577/639] Generate constraints file to pin deps in Docker images Dockerfiles pin versions using constraints file Pulling out strip_hashes and add --no-deps flag --- Dockerfile | 8 +++++++- certbot-dns-cloudflare/Dockerfile | 2 +- certbot-dns-cloudxns/Dockerfile | 2 +- certbot-dns-digitalocean/Dockerfile | 2 +- certbot-dns-dnsimple/Dockerfile | 2 +- certbot-dns-dnsmadeeasy/Dockerfile | 2 +- certbot-dns-gehirn/Dockerfile | 2 +- certbot-dns-google/Dockerfile | 2 +- certbot-dns-linode/Dockerfile | 2 +- certbot-dns-luadns/Dockerfile | 2 +- certbot-dns-nsone/Dockerfile | 2 +- certbot-dns-ovh/Dockerfile | 2 +- certbot-dns-rfc2136/Dockerfile | 2 +- certbot-dns-route53/Dockerfile | 2 +- certbot-dns-sakuracloud/Dockerfile | 2 +- tools/dev_constraints.txt | 2 +- tools/pip_install.py | 6 ++---- tools/strip_hashes.py | 16 ++++++++++++++++ 18 files changed, 40 insertions(+), 20 deletions(-) create mode 100755 tools/strip_hashes.py diff --git a/Dockerfile b/Dockerfile index 8f434e89c..828f5ec94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,13 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ + +# Generate constraints file to pin dependency versions COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . +COPY tools /opt/certbot/tools +RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt' +RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt' + COPY acme src/acme COPY certbot src/certbot @@ -23,7 +29,7 @@ RUN apk add --no-cache --virtual .build-deps \ musl-dev \ libffi-dev \ && pip install -r /opt/certbot/dependency-requirements.txt \ - && pip install --no-cache-dir \ + && pip install --no-cache-dir --no-deps \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ && apk del .build-deps diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile index 27dcc8751..adbf715fa 100644 --- a/certbot-dns-cloudflare/Dockerfile +++ b/certbot-dns-cloudflare/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudflare -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile index cc84ea65b..48c88c35c 100644 --- a/certbot-dns-cloudxns/Dockerfile +++ b/certbot-dns-cloudxns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudxns -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile index 8bdd0619f..342e0e876 100644 --- a/certbot-dns-digitalocean/Dockerfile +++ b/certbot-dns-digitalocean/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-digitalocean -RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile index 38d2be80e..724675339 100644 --- a/certbot-dns-dnsimple/Dockerfile +++ b/certbot-dns-dnsimple/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsimple -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile index ff7936925..1480baf4f 100644 --- a/certbot-dns-dnsmadeeasy/Dockerfile +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsmadeeasy -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile index 48ad902b5..7dce0e521 100644 --- a/certbot-dns-gehirn/Dockerfile +++ b/certbot-dns-gehirn/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-gehirn -RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile index 4a258d0ee..5750b31d9 100644 --- a/certbot-dns-google/Dockerfile +++ b/certbot-dns-google/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-google -RUN pip install --no-cache-dir --editable src/certbot-dns-google +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile index 2e237b521..6db8b59fb 100644 --- a/certbot-dns-linode/Dockerfile +++ b/certbot-dns-linode/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-linode -RUN pip install --no-cache-dir --editable src/certbot-dns-linode +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile index 6efb4d777..efc9f36d6 100644 --- a/certbot-dns-luadns/Dockerfile +++ b/certbot-dns-luadns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-luadns -RUN pip install --no-cache-dir --editable src/certbot-dns-luadns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile index 88fc13c57..de541e850 100644 --- a/certbot-dns-nsone/Dockerfile +++ b/certbot-dns-nsone/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-nsone -RUN pip install --no-cache-dir --editable src/certbot-dns-nsone +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile index e8da96d95..37e488dc4 100644 --- a/certbot-dns-ovh/Dockerfile +++ b/certbot-dns-ovh/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-ovh -RUN pip install --no-cache-dir --editable src/certbot-dns-ovh +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile index 1b8feb2f8..3ebb6a72e 100644 --- a/certbot-dns-rfc2136/Dockerfile +++ b/certbot-dns-rfc2136/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-rfc2136 -RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile index a1b8d6caf..e1825c11d 100644 --- a/certbot-dns-route53/Dockerfile +++ b/certbot-dns-route53/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-route53 -RUN pip install --no-cache-dir --editable src/certbot-dns-route53 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-route53 diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile index 694773f61..9fa9b3c22 100644 --- a/certbot-dns-sakuracloud/Dockerfile +++ b/certbot-dns-sakuracloud/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-sakuracloud -RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-sakuracloud diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..f9ad7329b 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,4 +1,4 @@ -# Specifies Python package versions for development. +# Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). diff --git a/tools/pip_install.py b/tools/pip_install.py index dd6302b48..475799eb1 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -19,6 +19,7 @@ import tempfile import merge_requirements as merge_module import readlink +import strip_hashes def find_tools_path(): @@ -47,10 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - for line in data: - search = re.search(r'^(\S*==\S*).*$', line) - if search: - fd.write('{0}{1}'.format(search.group(1), os.linesep)) + fd.write(strip_hashes.main(data)) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py new file mode 100755 index 000000000..c5591e2f4 --- /dev/null +++ b/tools/strip_hashes.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import os +import re +import sys + +def main(args): + out_lines = [] + for line in args: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + out_lines.append(search.group(1)) + return os.linesep.join(out_lines) + +if __name__ == '__main__': + print(main(sys.argv[1:])) From 8bda10541a67fc66c0b4da0ccc5bc8bacf741f25 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 20 Feb 2019 17:00:59 +0200 Subject: [PATCH 578/639] Add stdin option for merge_requirements Add stdin and file path support to strip_hashes --- tools/merge_requirements.py | 43 ++++++++++++++++++++++++++----------- tools/pip_install.py | 3 ++- tools/strip_hashes.py | 39 ++++++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 4205e6bcf..521d28b8f 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,27 +10,36 @@ from __future__ import print_function import sys -def read_file(file_path): - """Reads in a Python requirements file. +def process_entries(entries): + """ Ignore empty lines, comments and editable requirements - :param str file_path: path to requirements file + :param list entries: List of entries :returns: mapping from a project to its pinned version :rtype: dict - """ data = {} - with open(file_path) as file_h: - for line in file_h: - line = line.strip() - if line and not line.startswith('#') and not line.startswith('-e'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - data[project] = version + for e in entries: + e = e.strip() + if e and not e.startswith('#') and not e.startswith('-e'): + project, version = e.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(e)) + data[project] = version return data +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: list of entries in the file + :rtype: list + + """ + with open(file_path) as file_h: + return file_h.readlines() def output_requirements(requirements): """Prepare print requirements to stdout. @@ -53,7 +62,15 @@ def main(*paths): """ data = {} for path in paths: - data.update(read_file(path)) + data.update(process_entries(read_file(path))) + + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) + return output_requirements(data) diff --git a/tools/pip_install.py b/tools/pip_install.py index 475799eb1..8f0437d9c 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -48,7 +48,8 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - fd.write(strip_hashes.main(data)) + data = os.linesep.join(strip_hashes.process_entries(data)) + fd.write(data) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py index c5591e2f4..7e7809458 100755 --- a/tools/strip_hashes.py +++ b/tools/strip_hashes.py @@ -1,16 +1,45 @@ #!/usr/bin/env python +"""Removes hash information from requirement files passed to it as file path +arguments or simply piped to stdin.""" import os import re import sys -def main(args): + +def process_entries(entries): + """Strips off hash strings from dependencies. + + :param list entries: List of entries + + :returns: list of dependencies without hashes + :rtype: list + """ out_lines = [] - for line in args: - search = re.search(r'^(\S*==\S*).*$', line) + for e in entries: + e = e.strip() + search = re.search(r'^(\S*==\S*).*$', e) if search: out_lines.append(search.group(1)) - return os.linesep.join(out_lines) + return out_lines + +def main(*paths): + """Reads dependency definitions from a (list of) file(s) or stdin and + removes hashes from returned entries""" + + deps = [] + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) + + return os.linesep.join(deps) if __name__ == '__main__': - print(main(sys.argv[1:])) + print(main(*sys.argv[1:])) # pylint: disable=star-args From eef4c476335a9338dc71509d3d7c21de7d81b486 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 20 Feb 2019 15:20:44 -0800 Subject: [PATCH 579/639] Add failure message if test farm tests do not run the correct number of tests. (#6771) Fixes #6748. --- tests/letstest/multitester.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 8babc67b3..b8ae937ad 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -564,6 +564,11 @@ try: ii, target, status = outq print('%d %s %s'%(ii, target['name'], status)) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + if len(outputs) != num_processes: + failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ + 'Tests should be rerun.' + print(failure_message) + results_file.write(failure_message + '\n') results_file.close() finally: From eb5c4eca877baf52c0c39778002fce1e9482cff7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 21 Feb 2019 01:20:16 +0100 Subject: [PATCH 580/639] [Windows] Working unit tests for certbot-nginx (#6782) This PR fixes certbot-nginx and relevant tests to make them succeed on Windows. Next step will be to enable integration tests through certbot-ci in a future PR. * Fix tests and incompabilities in certbot-nginx for Windows * Fix lint, fix oldest local dependencies --- certbot-nginx/certbot_nginx/configurator.py | 7 +++---- certbot-nginx/certbot_nginx/parser.py | 4 ++-- .../certbot_nginx/tests/configurator_test.py | 19 ++++++++----------- .../certbot_nginx/tests/http_01_test.py | 6 ------ .../certbot_nginx/tests/parser_test.py | 12 +++++++++--- .../certbot_nginx/tests/tls_sni_01_test.py | 6 ------ certbot-nginx/certbot_nginx/tests/util.py | 18 ++++++++++++++++++ certbot-nginx/local-oldest-requirements.txt | 4 ++-- tools/install_and_test.py | 2 +- tox.cover.py | 5 ++++- 10 files changed, 47 insertions(+), 36 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index dd0bf9e8b..ffe1ddac7 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -13,6 +13,7 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util +from certbot import compat from certbot import constants as core_constants from certbot import crypto_util from certbot import errors @@ -164,9 +165,7 @@ class NginxConfigurator(common.Installer): util.lock_dir_until_exit(self.conf('server-root')) except (OSError, errors.LockError): logger.debug('Encountered error:', exc_info=True) - raise errors.PluginError( - 'Unable to lock %s', self.conf('server-root')) - + raise errors.PluginError('Unable to lock {0}'.format(self.conf('server-root'))) # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, @@ -899,7 +898,7 @@ class NginxConfigurator(common.Installer): have permissions of root. """ - uid = os.geteuid() + uid = compat.os_geteuid() util.make_or_verify_dir( self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) util.make_or_verify_dir( diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 622eb8d55..c5f780d94 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -81,9 +81,9 @@ class NginxParser(object): """ if not os.path.isabs(path): - return os.path.join(self.root, path) + return os.path.normpath(os.path.join(self.root, path)) else: - return path + return os.path.normpath(path) def _build_addr_to_ssl(self): """Builds a map from address to whether it listens on ssl in any server block diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 957588e2a..706e2637a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -1,7 +1,6 @@ # pylint: disable=too-many-public-methods """Test for certbot_nginx.configurator.""" import os -import shutil import unittest import mock @@ -33,12 +32,6 @@ class NginxConfiguratorTest(util.NginxTest): self.config = util.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - shutil.rmtree(self.logs_dir) - @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): mock_exe_exists.return_value = False @@ -69,8 +62,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare_locked(self): server_root = self.config.conf("server-root") + + from certbot import util as certbot_util + certbot_util._LOCKS[server_root].release() # pylint: disable=protected-access + self.config.config_test = mock.Mock() - os.remove(os.path.join(server_root, ".certbot.lock")) certbot_test_util.lock_and_call(self._test_prepare_locked, server_root) @mock.patch("certbot_nginx.configurator.util.exe_exists") @@ -88,11 +84,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) names = self.config.get_all_names() - self.assertEqual(names, set( - ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", + self.assertEqual(names, { + "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com"])) + "headers.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -171,6 +167,7 @@ class NginxConfiguratorTest(util.NginxTest): 'abc.www.foo.com': "etc_nginx/foo.conf", 'www.bar.co.uk': "etc_nginx/nginx.conf", 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + conf_path = {key: os.path.normpath(value) for key, value in conf_path.items()} vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index ed3c257ee..41c4b95fc 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.http_01""" import unittest -import shutil import mock import six @@ -54,11 +53,6 @@ class HttpPerformTest(util.NginxTest): from certbot_nginx import http_01 self.http01 = http_01.NginxHttp01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - def test_perform0(self): responses = self.http01.perform() self.assertEqual([], responses) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index f6f28e42b..3a68f7f24 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -67,9 +67,15 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods def test_abs_path(self): nparser = parser.NginxParser(self.config_path) - self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) - self.assertEqual(os.path.join(self.config_path, 'foo/bar/'), - nparser.abs_path('foo/bar/')) + if os.name != 'nt': + self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) + self.assertEqual(os.path.join(self.config_path, 'foo/bar'), + nparser.abs_path('foo/bar')) + else: + self.assertEqual('C:\\etc\\nginx\\*', nparser.abs_path('C:\\etc\\nginx\\*')) + self.assertEqual(os.path.join(self.config_path, 'foo\\bar'), + nparser.abs_path('foo\\bar')) + def test_filedump(self): nparser = parser.NginxParser(self.config_path) diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 72b65911c..62ca085ef 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.tls_sni_01""" import unittest -import shutil import mock import six @@ -55,11 +54,6 @@ class TlsSniPerformTest(util.NginxTest): from certbot_nginx import tls_sni_01 self.sni = tls_sni_01.NginxTlsSni01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - @mock.patch("certbot_nginx.configurator" ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index ad1af2b96..ef669dac0 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -4,6 +4,8 @@ import os import pkg_resources import tempfile import unittest +import shutil +import warnings import josepy as jose import mock @@ -33,6 +35,22 @@ class NginxTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + def tearDown(self): + # On Windows we have various files which are not correctly closed at the time of tearDown. + # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. + def onerror_handler(_, path, excinfo): + """On error handler""" + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting path {0}' + 'during tearDown process: {1}'.format(path, str(excinfo))) + warnings.warn(message) + + shutil.rmtree(self.temp_dir, onerror=onerror_handler) + shutil.rmtree(self.config_dir, onerror=onerror_handler) + shutil.rmtree(self.work_dir, onerror=onerror_handler) + shutil.rmtree(self.logs_dir, onerror=onerror_handler) + def get_data_filename(filename): """Gets the filename of a test data file.""" diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index bcd02d197..db6b261f0 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.26.0 -certbot[dev]==0.22.0 +acme[dev]==0.29.0 +-e .[dev] diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b15c8eca5..288226527 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -15,7 +15,7 @@ import subprocess import re SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] def call_with_print(command, cwd=None): diff --git a/tox.cover.py b/tox.cover.py index 008424641..e323ba255 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -35,7 +35,8 @@ COVER_THRESHOLDS = { } SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] + def cover(package): threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] @@ -54,6 +55,7 @@ def cover(package): sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) + def main(): description = """ This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order @@ -77,5 +79,6 @@ Option -e makes sure we fail fast and don't submit to codecov.""" for package in packages: cover(package) + if __name__ == '__main__': main() From b10ceb7d907cdcc36ec2d745e7802127919ec043 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 22 Feb 2019 01:55:08 +0100 Subject: [PATCH 581/639] Fix test sdists with atexit handlers (#6769) So merging the study from @bmw and me, here is what happened. Each invocation of `certbot.logger.post_arg_parse_setup` create a file handler on `letsencrypt.log`. This function also set an atexit handler invoking `logger.shutdown()`, that have the effect to close all logger file handler not already closed at this point. This method is supposed to be called when a python process is close to exit, because it makes all logger unable to write new logs on any handler. Before #6667 and this PR, for tests, the atexit handle would be triggered only at the end of the pytest process. It means that each test that launches `certbot.logger.post_arg_parse_setup` add a new file handler. These tests were typically connecting the file handler on a `letsencrypt.log` located in a temporary directory, and this directory and content was wipped out at each test tearDown. As a consequence, the file handles, not cleared from the logger, were accumulating in the logger, with all of them connected to a deleted file log, except the last one that was just created by the current test. Considering the number of tests concerned, there were ~300 file handler at the end of pytest execution. One can see that, on prior #6667, by calling `print(logger.getLogger().handlers` on the `tearDown` of these tests, and see the array growing at each test execution. Even if this represent a memory leak, this situation was not really a problem on Linux: because a file can be deleted before it is closed, it was only meaning that a given invocation of `logger.debug` for instance, during the tests, was written in 300 log files. The overhead is negligeable. On Windows however, the file handlers were failing because you cannot delete a file before it is closed. It was one of the reason for #6667, that added a call to `logging.shutdown()` at each test tearDown, with the consequence to close all file handlers. At this point, Linux is not happy anymore. Any call to `logger.warn` will generate an error for each closed file handler. As a file handler is added for each test, the number of errors grows on each test, following an arithmetical suite divergence. On `test_sdists.py`, that is using the bare setuptools test suite without output capturing, we can see the damages. The total output takes 216000 lines, and 23000 errors are generated. A decent machine can support this load, but a not a small AWS instance, that is crashing during the execution. Even with pytest, the captured output and the memory leak become so large that segfaults are generated. On the current PR, the problem is solved, by resetting the file handlers array on the logging system on each test tearDown. So each fileHandler is properly closed, and removed from the stack. They do not participate anymore in the logging system, and can be garbage collected. Then we stay on always one file handler opened at any time, and tests can succeed on AWS instances. For the record, here is all the places where the logging system is called and fail if there is still file handlers closed but not cleaned (extracted from the original huge output before correction): ``` Logged from file account.py, line 116 Logged from file account.py, line 178 Logged from file client.py, line 166 Logged from file client.py, line 295 Logged from file client.py, line 415 Logged from file client.py, line 422 Logged from file client.py, line 480 Logged from file client.py, line 503 Logged from file client.py, line 540 Logged from file client.py, line 601 Logged from file client.py, line 622 Logged from file client.py, line 750 Logged from file cli.py, line 220 Logged from file cli.py, line 226 Logged from file crypto_util.py, line 101 Logged from file crypto_util.py, line 127 Logged from file crypto_util.py, line 147 Logged from file crypto_util.py, line 261 Logged from file crypto_util.py, line 283 Logged from file crypto_util.py, line 307 Logged from file crypto_util.py, line 336 Logged from file disco.py, line 116 Logged from file disco.py, line 124 Logged from file disco.py, line 134 Logged from file disco.py, line 138 Logged from file disco.py, line 141 Logged from file dns_common_lexicon.py, line 45 Logged from file dns_common_lexicon.py, line 61 Logged from file dns_common_lexicon.py, line 67 Logged from file dns_common.py, line 316 Logged from file dns_common.py, line 64 Logged from file eff.py, line 60 Logged from file eff.py, line 73 Logged from file error_handler.py, line 105 Logged from file error_handler.py, line 110 Logged from file error_handler.py, line 87 Logged from file hooks.py, line 248 Logged from file main.py, line 1071 Logged from file main.py, line 1075 Logged from file main.py, line 1189 Logged from file ops.py, line 122 Logged from file ops.py, line 325 Logged from file ops.py, line 338 Logged from file reporter.py, line 55 Logged from file selection.py, line 110 Logged from file selection.py, line 118 Logged from file selection.py, line 123 Logged from file selection.py, line 176 Logged from file selection.py, line 231 Logged from file selection.py, line 310 Logged from file selection.py, line 66 Logged from file standalone.py, line 101 Logged from file standalone.py, line 88 Logged from file standalone.py, line 97 Logged from file standalone.py, line 98 Logged from file storage.py, line 52 Logged from file storage.py, line 59 Logged from file storage.py, line 75 Logged from file util.py, line 56 Logged from file webroot.py, line 165 Logged from file webroot.py, line 186 Logged from file webroot.py, line 187 Logged from file webroot.py, line 204 Logged from file webroot.py, line 223 Logged from file webroot.py, line 234 Logged from file webroot.py, line 235 Logged from file webroot.py, line 237 Logged from file webroot.py, line 91 ``` * Reapply #6667 * Make setuptools delegates tests execution to pytest, like in acme module. * Clean handlers at each tearDown to avoid memory leaks. * Update changelog --- CHANGELOG.md | 2 ++ acme/setup.py | 4 +++- certbot-apache/setup.py | 20 ++++++++++++++++++++ certbot-nginx/setup.py | 20 ++++++++++++++++++++ certbot/tests/lock_test.py | 3 +++ certbot/tests/util.py | 33 ++++++++++++++++++--------------- certbot/tests/util_test.py | 24 +++++++++--------------- certbot/util.py | 6 +++--- setup.py | 20 ++++++++++++++++++++ 9 files changed, 98 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a895e07..7744e3745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). warnings described at https://github.com/certbot/josepy/issues/13. * Apache plugin now respects CERTBOT_DOCS environment variable when adding command line defaults. +* Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. ### Fixed @@ -26,6 +27,7 @@ package with changes other than its version number was: * acme * certbot * certbot-apache +* certbot-nginx More details about these changes can be found on our GitHub repo. diff --git a/acme/setup.py b/acme/setup.py index 79d6d3389..6cb5c3f92 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -36,6 +36,7 @@ docs_extras = [ 'sphinx_rtd_theme', ] + class PyTest(TestCommand): user_options = [] @@ -50,6 +51,7 @@ class PyTest(TestCommand): errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) + setup( name='acme', version=version, @@ -82,7 +84,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, - tests_require=["pytest"], test_suite='acme', + tests_require=["pytest"], cmdclass={"test": PyTest}, ) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 52528a536..5d15611dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,7 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys version = '0.32.0.dev0' @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-apache', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_apache', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index f78473ada..8984704a6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,7 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys version = '0.32.0.dev0' @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-nginx', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_nginx', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 8658443d0..6379693ae 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -41,6 +41,7 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') + @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -58,6 +59,7 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) + @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) locked_repr = repr(lock_file) @@ -92,6 +94,7 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) + @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 289f2dd9b..4c0c66a42 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,13 +3,14 @@ .. warning:: This module is not part of the public API. """ +import logging import os import pkg_resources import shutil +import stat import tempfile import unittest import sys -import warnings from multiprocessing import Process, Event from cryptography.hazmat.backends import default_backend @@ -329,23 +330,25 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # On Windows we have various files which are not correctly closed at the time of tearDown. - # For know, we log them until a proper file close handling is written. - # Useful for development only, so no warning when we are on a CI process. - def onerror_handler(_, path, excinfo): - """On error handler""" - if not os.environ.get('APPVEYOR'): # pragma: no cover - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) - shutil.rmtree(self.tempdir, onerror=onerror_handler) + # Cleanup opened resources after a test. This is usually done through atexit handlers in + # Certbot, but during tests, atexit will not run registered functions before tearDown is + # called and instead will run them right before the entire test process exits. + # It is a problem on Windows, that does not accept to clean resources before closing them. + logging.shutdown() + # Remove logging handlers that have been closed so they won't be + # accidentally used in future tests. + logging.getLogger().handlers = [] + util._release_locks() # pylint: disable=protected-access + + def handle_rw_files(_, path, __): + """Handle read-only files, that will fail to be removed on Windows.""" + os.chmod(path, stat.S_IWRITE) + os.remove(path) + shutil.rmtree(self.tempdir, onerror=handle_rw_files) class ConfigTestCase(TempDirTestCase): - """Test class which sets up a NamespaceConfig object. - - """ + """Test class which sets up a NamespaceConfig object.""" def setUp(self): super(ConfigTestCase, self).setUp() self.config = configuration.NamespaceConfig( diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index c85009c62..cac587995 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -193,7 +193,12 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_wrong_mode(self): os.chmod(self.tempdir, 0o400) - self.assertFalse(self._call(0o600)) + try: + self.assertFalse(self._call(0o600)) + finally: + # Without proper write permissions, Windows is unable to delete a folder, + # even with admin permissions. Write access must be explicitly set first. + os.chmod(self.tempdir, 0o700) class UniqueFileTest(test_util.TempDirTestCase): @@ -279,20 +284,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for f, _ in items: f.close() - @mock.patch("certbot.util.os.fdopen") - def test_failure(self, mock_fdopen): - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") - - @mock.patch("certbot.util.os.fdopen") - def test_subsequent_failure(self, mock_fdopen): - self._call("wow") - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") + def test_failure(self): + with mock.patch("certbot.util.os.open", side_effect=OSError(errno.EIO)): + self.assertRaises(OSError, self._call, "wow") class SafelyRemoveTest(test_util.TempDirTestCase): diff --git a/certbot/util.py b/certbot/util.py index d7c542465..416075ce8 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -142,6 +142,7 @@ def _release_locks(): except: # pylint: disable=bare-except msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) logger.debug(msg, exc_info=True) + _LOCKS.clear() def set_up_core_dir(directory, mode, uid, strict): @@ -225,9 +226,8 @@ def safe_open(path, mode="w", chmod=None, buffering=None): fdopen_args = () # type: Union[Tuple[()], Tuple[int]] if buffering is not None: fdopen_args = (buffering,) - return os.fdopen( - os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), - mode, *fdopen_args) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) + return os.fdopen(fd, mode, *fdopen_args) def _unique_file(path, filename_pat, count, chmod, mode): diff --git a/setup.py b/setup.py index 14fef37f3..26a1c4293 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ import codecs import os import re +import sys from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand # Workaround for http://bugs.python.org/issue8876, see # http://bugs.python.org/issue8876#msg208792 @@ -77,6 +79,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot', version=version, @@ -123,6 +141,8 @@ setup( # to test all packages run "python setup.py test -s # {acme,certbot_apache,certbot_nginx}" test_suite='certbot', + tests_require=["pytest"], + cmdclass={"test": PyTest}, entry_points={ 'console_scripts': [ From 31b4b8e57c542577d35a20f87eaad60d5ce37193 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 22 Feb 2019 06:42:01 -0800 Subject: [PATCH 582/639] Log the execution of manual hooks (#6788) * Move logging to execute and fix tests. * update changelog --- CHANGELOG.md | 2 ++ certbot/hooks.py | 36 ++++++++++++++++++++---------------- certbot/plugins/manual.py | 7 +++++-- certbot/tests/hook_test.py | 34 +++++++++++++++++++--------------- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7744e3745..ce7eac2e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). warnings described at https://github.com/certbot/josepy/issues/13. * Apache plugin now respects CERTBOT_DOCS environment variable when adding command line defaults. +* The running of manual plugin hooks is now always included in Certbot's log + output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. ### Fixed diff --git a/certbot/hooks.py b/certbot/hooks.py index d5239a437..7d2e42fcd 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -93,8 +93,7 @@ def _run_pre_hook_if_necessary(command): if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: - logger.info("Running pre-hook command: %s", command) - _run_hook(command) + _run_hook("pre-hook", command) executed_pre_hooks.add(command) @@ -126,8 +125,7 @@ def post_hook(config): _run_eventually(cmd) # certonly / run elif cmd: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) post_hooks = [] # type: List[str] @@ -149,8 +147,7 @@ def _run_eventually(command): def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" for cmd in post_hooks: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) def deploy_hook(config, domains, lineage_path): @@ -220,23 +217,30 @@ def _run_deploy_hook(command, domains, lineage_path, dry_run): os.environ["RENEWED_DOMAINS"] = " ".join(domains) os.environ["RENEWED_LINEAGE"] = lineage_path - logger.info("Running deploy-hook command: %s", command) - _run_hook(command) + _run_hook("deploy-hook", command) -def _run_hook(shell_cmd): +def _run_hook(cmd_name, shell_cmd): """Run a hook command. - :returns: stderr if there was any""" + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` - err, _ = execute(shell_cmd) + :returns: stderr if there was any""" + err, _ = execute(cmd_name, shell_cmd) return err -def execute(shell_cmd): +def execute(cmd_name, shell_cmd): """Run a command. + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` + :returns: `tuple` (`str` stderr, `str` stdout)""" + logger.info("Running %s command: %s", cmd_name, shell_cmd) # universal_newlines causes Popen.communicate() # to return str objects instead of bytes in Python 3 @@ -245,12 +249,12 @@ def execute(shell_cmd): out, err = cmd.communicate() base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) if out: - logger.info('Output from %s:\n%s', base_cmd, out) + logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out) if cmd.returncode != 0: - logger.error('Hook command "%s" returned error code %d', - shell_cmd, cmd.returncode) + logger.error('%s command "%s" returned error code %d', + cmd_name, shell_cmd, cmd.returncode) if err: - logger.error('Error output from %s:\n%s', base_cmd, err) + logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err) return (err, out) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 8723a1c62..123a5bfea 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -202,7 +202,7 @@ permitted by DNS standards.) os.environ.pop('CERTBOT_KEY_PATH', None) os.environ.pop('CERTBOT_SNI_DOMAIN', None) os.environ.update(env) - _, out = hooks.execute(self.conf('auth-hook')) + _, out = self._execute_hook('auth-hook') env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env @@ -243,5 +243,8 @@ permitted by DNS standards.) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) - hooks.execute(self.conf('cleanup-hook')) + self._execute_hook('cleanup-hook') self.reverter.recovery_routine() + + def _execute_hook(self, hook_name): + return hooks.execute(self.option_name(hook_name), self.conf(hook_name)) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index f5bb0c8b5..90f639958 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -121,7 +121,7 @@ class PreHookTest(HookTest): def _test_nonrenew_common(self): mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_no_hooks(self): @@ -137,21 +137,21 @@ class PreHookTest(HookTest): def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_no_overlap(self): self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.pre_hook) + mock_execute.assert_any_call("pre-hook", self.dir_hook) + mock_execute.assert_called_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_with_overlap(self): self.config.pre_hook = self.dir_hook self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("pre-hook", self.dir_hook) self._test_no_executions_common() def _test_no_executions_common(self): @@ -193,7 +193,7 @@ class PostHookTest(HookTest): for verb in ("certonly", "run",): self.config.verb = verb mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.post_hook) + mock_execute.assert_called_once_with("post-hook", self.config.post_hook) self.assertFalse(self._get_eventually()) def test_cert_only_and_run_without_hook(self): @@ -277,12 +277,12 @@ class RunSavedPostHooksTest(HookTest): calls = mock_execute.call_args_list for actual_call, expected_arg in zip(calls, self.eventually): - self.assertEqual(actual_call[0][0], expected_arg) + self.assertEqual(actual_call[0][1], expected_arg) def test_single(self): self.eventually = ["foo"] mock_execute = self._call_with_mock_execute_and_eventually() - mock_execute.assert_called_once_with(self.eventually[0]) + mock_execute.assert_called_once_with("post-hook", self.eventually[0]) class RenewalHookTest(HookTest): @@ -360,7 +360,7 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = "foo" mock_execute = self._call_with_mock_execute( self.config, domains, lineage) - mock_execute.assert_called_once_with(self.config.deploy_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.deploy_hook) class RenewHookTest(RenewalHookTest): @@ -384,7 +384,7 @@ class RenewHookTest(RenewalHookTest): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.config.renew_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.renew_hook) @mock.patch("certbot.hooks.logger") def test_dry_run(self, mock_logger): @@ -408,13 +408,13 @@ class RenewHookTest(RenewalHookTest): self.config.renew_hook = self.dir_hook mock_execute = self._call_with_mock_execute( self.config, ["example.net", "example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("deploy-hook", self.dir_hook) def test_no_overlap(self): mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.renew_hook) + mock_execute.assert_any_call("deploy-hook", self.dir_hook) + mock_execute.assert_called_with("deploy-hook", self.config.renew_hook) class ExecuteTest(unittest.TestCase): @@ -433,18 +433,22 @@ class ExecuteTest(unittest.TestCase): def _test_common(self, returncode, stdout, stderr): given_command = "foo" + given_name = "foo-hook" with mock.patch("certbot.hooks.Popen") as mock_popen: mock_popen.return_value.communicate.return_value = (stdout, stderr) mock_popen.return_value.returncode = returncode with mock.patch("certbot.hooks.logger") as mock_logger: - self.assertEqual(self._call(given_command), (stderr, stdout)) + self.assertEqual(self._call(given_name, given_command), (stderr, stdout)) executed_command = mock_popen.call_args[1].get( "args", mock_popen.call_args[0][0]) self.assertEqual(executed_command, given_command) + mock_logger.info.assert_any_call("Running %s command: %s", + given_name, given_command) if stdout: - self.assertTrue(mock_logger.info.called) + mock_logger.info.assert_any_call(mock.ANY, mock.ANY, + mock.ANY, stdout) if stderr or returncode: self.assertTrue(mock_logger.error.called) From f105aedc9258d7a1de55b7de56802535d3b1b4dc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 22 Feb 2019 16:55:50 -0800 Subject: [PATCH 583/639] Remove display.py. --- tests/display.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/display.py diff --git a/tests/display.py b/tests/display.py deleted file mode 100644 index 1f548e33d..000000000 --- a/tests/display.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Manual test of display functions.""" -import sys - -from certbot.display import util -from certbot.tests.display import util_test - - -def test_visual(displayer, choices): - """Visually test all of the display functions.""" - displayer.notification("Random notification!") - displayer.menu("Question?", choices, - ok_label="O", cancel_label="Can", help_label="??") - displayer.menu("Question?", [choice[1] for choice in choices], - ok_label="O", cancel_label="Can", help_label="??") - displayer.input("Input Message") - displayer.yesno("YesNo Message", yes_label="Yessir", no_label="Nosir") - displayer.checklist("Checklist Message", [choice[0] for choice in choices]) - - -if __name__ == "__main__": - displayer = util.FileDisplay(sys.stdout, False) - test_visual(displayer, util_test.CHOICES) From 401045be89d241bad01ebb8e933bdb83f836bc20 Mon Sep 17 00:00:00 2001 From: Julie R Date: Sat, 23 Feb 2019 03:02:43 +0100 Subject: [PATCH 584/639] Add acme library usage example (http-01) (#5494) * Add acme library usage example Create, edit and deactivate account. Setup and perform http-01 challenge. Issue, renew and revoke certificate. * Adapt example to ACME-v2 and exclude data persistence The code to persist/load data would length this example and distract from what is actually important. * Fix domain names and e-mail addresses * Remove unnecessary license header This usage example is under the license for the acme package. * Remove logging information The code will be mostly read by developers, so simplify the logging info into comments. * Revert abstraction of simple methods All methods that are used only once in this example were expanded into the main code in order to make the process more explicit. * Fix missing URL suffix * Improve aesthetics and reorganize workflow Also make words capitalization consistent and improve comments. No complaints from pep8. --- acme/examples/http01_example.py | 240 ++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 acme/examples/http01_example.py diff --git a/acme/examples/http01_example.py b/acme/examples/http01_example.py new file mode 100644 index 000000000..79508f1b4 --- /dev/null +++ b/acme/examples/http01_example.py @@ -0,0 +1,240 @@ +"""Example ACME-V2 API for HTTP-01 challenge. + +Brief: + +This a complete usage example of the python-acme API. + +Limitations of this example: + - Works for only one Domain name + - Performs only HTTP-01 challenge + - Uses ACME-v2 + +Workflow: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account +""" +from contextlib import contextmanager +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL + +from acme import challenges +from acme import client +from acme import crypto_util +from acme import errors +from acme import messages +from acme import standalone +import josepy as jose + +# Constants: + +# This is the staging point for ACME-V2 within Let's Encrypt. +DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' + +USER_AGENT = 'python-acme-example' + +# Account key size +ACC_KEY_BITS = 2048 + +# Certificate private key size +CERT_PKEY_BITS = 2048 + +# Domain name for the certificate. +DOMAIN = 'client.example.com' + +# If you are running Boulder locally, it is possible to configure any port +# number to execute the challenge, but real CA servers will always use port +# 80, as described in the ACME specification. +PORT = 80 + + +# Useful methods and classes: + + +def new_csr_comp(domain_name, pkey_pem=None): + """Create certificate signing request.""" + if pkey_pem is None: + # Create private key. + pkey = OpenSSL.crypto.PKey() + pkey.generate_key(OpenSSL.crypto.TYPE_RSA, CERT_PKEY_BITS) + pkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, + pkey) + csr_pem = crypto_util.make_csr(pkey_pem, [domain_name]) + return pkey_pem, csr_pem + + +def select_http01_chall(orderr): + """Extract authorization resource from within order resource.""" + # Authorization Resource: authz. + # This object holds the offered challenges by the server and their status. + authz_list = orderr.authorizations + + for authz in authz_list: + # Choosing challenge. + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + return i + + raise Exception('HTTP-01 challenge was not offered by the CA server.') + + +@contextmanager +def challenge_server(http_01_resources): + """Manage standalone server set up and shutdown.""" + + # Setting up a fake server that binds at PORT and any address. + address = ('', PORT) + try: + servers = standalone.HTTP01DualNetworkedServers(address, + http_01_resources) + # Start client standalone web server. + servers.serve_forever() + yield servers + finally: + # Shutdown client web server and unbind from PORT + servers.shutdown_and_server_close() + + +def perform_http01(client_acme, challb, orderr): + """Set up standalone webserver and perform HTTP-01 challenge.""" + + response, validation = challb.response_and_validation(client_acme.net.key) + + resource = standalone.HTTP01RequestHandler.HTTP01Resource( + chall=challb.chall, response=response, validation=validation) + + with challenge_server({resource}): + # Let the CA server know that we are ready for the challenge. + client_acme.answer_challenge(challb, response) + + # Wait for challenge status and then issue a certificate. + # It is possible to set a deadline time. + finalized_orderr = client_acme.poll_and_finalize(orderr) + + return finalized_orderr.fullchain_pem + + +# Main examples: + + +def example_http(): + """This example executes the whole process of fulfilling a HTTP-01 + challenge for one specific domain. + + The workflow consists of: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account + + """ + # Create account key + + acc_key = jose.JWKRSA( + key=rsa.generate_private_key(public_exponent=65537, + key_size=ACC_KEY_BITS, + backend=default_backend())) + + # Register account and accept TOS + + net = client.ClientNetwork(acc_key, user_agent=USER_AGENT) + directory = messages.Directory.from_json(net.get(DIRECTORY_URL).json()) + client_acme = client.ClientV2(directory, net=net) + + # Terms of Service URL is in client_acme.directory.meta.terms_of_service + # Registration Resource: regr + # Creates account with contact information. + email = ('fake@example.com') + regr = client_acme.new_account( + messages.NewRegistration.from_data( + email=email, terms_of_service_agreed=True)) + + # Create domain private key and CSR + pkey_pem, csr_pem = new_csr_comp(DOMAIN) + + # Issue certificate + + orderr = client_acme.new_order(csr_pem) + + # Select HTTP-01 within offered challenges by the CA server + challb = select_http01_chall(orderr) + + # The certificate is ready to be used in the variable "fullchain_pem". + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Renew certificate + + _, csr_pem = new_csr_comp(DOMAIN, pkey_pem) + + orderr = client_acme.new_order(csr_pem) + + challb = select_http01_chall(orderr) + + # Performing challenge + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Revoke certificate + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + + try: + client_acme.revoke(fullchain_com, 0) # revocation reason = 0 + except errors.ConflictError: + # Certificate already revoked. + pass + + # Query registration status. + client_acme.net.account = regr + try: + regr = client_acme.query_registration(regr) + except errors.Error as err: + if err.typ == messages.OLD_ERROR_PREFIX + 'unauthorized' \ + or err.typ == messages.ERROR_PREFIX + 'unauthorized': + # Status is deactivated. + pass + raise + + # Change contact information + + email = 'newfake@example.com' + regr = client_acme.update_registration( + regr.update( + body=regr.body.update( + contact=('mailto:' + email,) + ) + ) + ) + + # Deactivate account/registration + + regr = client_acme.deactivate_registration(regr) + + +if __name__ == "__main__": + example_http() From 339d034d6a5a57d296607795a4706203f81d7059 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 27 Feb 2019 18:21:47 +0100 Subject: [PATCH 585/639] Remove keyAuthorization field from the challenge response JWS token (#6758) Fixes #6755. POSTing the `keyAuthorization` in a JWS token when answering an ACME challenge, has been deprecated for some time now. Indeed, this is superfluous as the request is already authentified by the JWS signature. Boulder still accepts to see this field in the JWS token, and ignore it. Pebble in non strict mode also. But Pebble in strict mode refuses the request, to prepare complete removal of this field in ACME v2. Certbot still sends the `keyAuthorization` field. This PR removes it, and makes Certbot compliant with current ACME v2 protocol, and so Pebble in strict mode. See also [letsencrypt/pebble#192](https://github.com/letsencrypt/pebble/issues/192) for implementation details server side. * New implementation, with a fallback. * Add deprecation on changelog * Update acme/acme/client.py Co-Authored-By: adferrand * Fix an instance parameter * Update changelog, extend coverage * Update comment * Add unit tests on keyAuthorization dump * Update acme/acme/client.py Co-Authored-By: adferrand * Restrict the magic of setting a variable in immutable object in one place. Make a soon to be removed method private. --- CHANGELOG.md | 7 +++++++ acme/acme/challenges.py | 20 ++++++++++++++++++++ acme/acme/challenges_test.py | 12 ++++++++++++ acme/acme/client.py | 26 ++++++++++++++++++++------ acme/acme/client_test.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7eac2e9..53c158f8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The running of manual plugin hooks is now always included in Certbot's log output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. +* The `acme` module avoids sending the `keyAuthorization` field in the JWS + payload when responding to a challenge as the field is not included in the + current ACME protocol. To ease the migration path for ACME CA servers, + Certbot and its `acme` module will first try the request without the + `keyAuthorization` field but will temporarily retry the request with the + field included if a `malformed` error is received. This fallback will be + removed in version 0.34.0. ### Fixed diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 501f74881..6f2b3757b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -108,6 +108,10 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): key_authorization = jose.Field("keyAuthorization") thumbprint_hash_function = hashes.SHA256 + def __init__(self, *args, **kwargs): + super(KeyAuthorizationChallengeResponse, self).__init__(*args, **kwargs) + self._dump_authorization_key(False) + def verify(self, chall, account_public_key): """Verify the key authorization. @@ -140,6 +144,22 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True + def _dump_authorization_key(self, dump): + # type: (bool) -> None + """ + Set if keyAuthorization is dumped in the JSON representation of this ChallengeResponse. + NB: This method is declared as private because it will eventually be removed. + :param bool dump: True to dump the keyAuthorization, False otherwise + """ + object.__setattr__(self, '_dump_auth_key', dump) + + def to_partial_json(self): + jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + if not self._dump_auth_key: # pylint: disable=no-member + jobj.pop('keyAuthorization', None) + + return jobj + @six.add_metaclass(abc.ABCMeta) class KeyAuthorizationChallenge(_TokenChallenge): diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 81d39058e..4b905c1e5 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -94,6 +94,9 @@ class DNS01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -165,6 +168,9 @@ class HTTP01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -285,6 +291,9 @@ class TLSSNI01ResponseTest(unittest.TestCase): self.assertEqual(self.z_domain, self.response.z_domain) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.response.to_partial_json()) + self.response._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.response.to_partial_json()) def test_from_json(self): @@ -419,6 +428,9 @@ class TLSALPN01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): diff --git a/acme/acme/client.py b/acme/acme/client.py index 76b6a7788..d74713a7e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -17,6 +17,7 @@ import requests from requests.adapters import HTTPAdapter import sys +from acme import challenges from acme import crypto_util from acme import errors from acme import jws @@ -155,7 +156,23 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self._post(challb.uri, response) + # Because sending keyAuthorization in a response challenge has been removed from the ACME + # spec, it is not included in the KeyAuthorizationResponseChallenge JSON by default. + # However as a migration path, we temporarily expect a malformed error from the server, + # and fallback by resending the challenge response with the keyAuthorization field. + # TODO: Remove this fallback for Certbot 0.34.0 + try: + response = self._post(challb.uri, response) + except messages.Error as error: + if (error.code == 'malformed' + and isinstance(response, challenges.KeyAuthorizationChallengeResponse)): + logger.debug('Error while responding to a challenge without keyAuthorization ' + 'in the JWS, your ACME CA server may not support it:\n%s', error) + logger.debug('Retrying request with keyAuthorization set.') + response._dump_authorization_key(True) # pylint: disable=protected-access + response = self._post(challb.uri, response) + else: + raise try: authzr_uri = response.links['up']['url'] except KeyError: @@ -781,7 +798,7 @@ class ClientV2(ClientBase): except messages.Error as error: if error.code == 'malformed': logger.debug('Error during a POST-as-GET request, ' - 'your ACME CA may not support it:\n%s', error) + 'your ACME CA server may not support it:\n%s', error) logger.debug('Retrying request with GET.') else: # pragma: no cover raise @@ -1191,10 +1208,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, acme_version=1, **kwargs): - try: - new_nonce_url = kwargs.pop('new_nonce_url') - except KeyError: - new_nonce_url = None + new_nonce_url = kwargs.pop('new_nonce_url', None) data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index b3d0f1921..dc8cc0c93 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -463,6 +463,34 @@ class ClientTest(ClientTestBase): errors.ClientError, self.client.answer_challenge, self.challr.body, challenges.DNSResponse(validation=None)) + def test_answer_challenge_key_authorization_fallback(self): + self.response.links['up'] = {'url': self.challr.authzr_uri} + self.response.json.return_value = self.challr.body.to_json() + + def _wrapper_post(url, obj, *args, **kwargs): # pylint: disable=unused-argument + """ + Simulate an old ACME CA server, that would respond a 'malformed' + error if keyAuthorization is missing. + """ + jobj = obj.to_partial_json() + if 'keyAuthorization' not in jobj: + raise messages.Error.with_code('malformed') + return self.response + self.net.post.side_effect = _wrapper_post + + # This challenge response is of type KeyAuthorizationChallengeResponse, so the fallback + # should be triggered, and avoid an exception. + http_chall_response = challenges.HTTP01Response(key_authorization='test', + resource=mock.MagicMock()) + self.client.answer_challenge(self.challr.body, http_chall_response) + + # This challenge response is not of type KeyAuthorizationChallengeResponse, so the fallback + # should not be triggered, leading to an exception. + dns_chall_response = challenges.DNSResponse(validation=None) + self.assertRaises( + errors.Error, self.client.answer_challenge, + self.challr.body, dns_chall_response) + def test_retry_after_date(self): self.response.headers['Retry-After'] = 'Fri, 31 Dec 1999 23:59:59 GMT' self.assertEqual( From 9c405a3cd12588ba27cdd8f79c8a101831bd6ad3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 00:16:52 +0100 Subject: [PATCH 586/639] Fix cryptography OCSP support (#6751) * Reenabling OCSP cryptography support * Refactor the validation logic of OCSP response to match the OpenSSL one * Prepare runtime for OCSP response test * Move unrelated test to another relevant place * Reimplement OCSP status checks in integration tests * Clean script * Protect OCSP check against connection errors * Update tests/certbot-boulder-integration.sh Co-Authored-By: adferrand * Cleaning * Add a specific script for letsencrypt-auto install+help * Remove inconsistent assertion * Add executable permissions * Remove unused variable * Move testdata * Corrected cleanup code * Empty commit --- CHANGELOG.md | 3 + certbot/crypto_util.py | 53 +++-- certbot/ocsp.py | 211 ++++++++++++++---- certbot/tests/ocsp_test.py | 172 ++++++++++++-- certbot/tests/testdata/google_certificate.pem | 41 ++++ .../testdata/google_issuer_certificate.pem | 26 +++ certbot/util.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 40 ++-- .../pieces/dependency-requirements.txt | 40 ++-- tests/certbot-boulder-integration.sh | 55 +++++ .../meta.json | 0 .../private_key.json | 0 .../regr.json | 0 .../a.encryption-example.com/cert1.pem | 0 .../a.encryption-example.com/chain1.pem | 0 .../a.encryption-example.com/fullchain1.pem | 0 .../a.encryption-example.com/privkey1.pem | 0 .../b.encryption-example.com/cert1.pem | 0 .../b.encryption-example.com/chain1.pem | 0 .../b.encryption-example.com/fullchain1.pem | 0 .../b.encryption-example.com/privkey1.pem | 0 .../sample-config/csr/0000_csr-certbot.pem | 0 .../sample-config/csr/0001_csr-certbot.pem | 0 .../sample-config/csr/0002_csr-certbot.pem | 0 .../sample-config/csr/0003_csr-certbot.pem | 0 .../sample-config/keys/0000_key-certbot.pem | 0 .../sample-config/keys/0001_key-certbot.pem | 0 .../sample-config/keys/0002_key-certbot.pem | 0 .../sample-config/keys/0003_key-certbot.pem | 0 .../live/a.encryption-example.com/README | 0 .../live/a.encryption-example.com/cert.pem | 0 .../live/a.encryption-example.com/chain.pem | 0 .../a.encryption-example.com/fullchain.pem | 0 .../live/a.encryption-example.com/privkey.pem | 0 .../live/b.encryption-example.com/README | 0 .../live/b.encryption-example.com/cert.pem | 0 .../live/b.encryption-example.com/chain.pem | 0 .../b.encryption-example.com/fullchain.pem | 0 .../live/b.encryption-example.com/privkey.pem | 0 .../sample-config/options-ssl-apache.conf | 0 .../renewal/a.encryption-example.com.conf | 0 .../renewal/b.encryption-example.com.conf | 0 ...st_letsencrypt_auto_certonly_standalone.sh | 21 -- 43 files changed, 519 insertions(+), 145 deletions(-) create mode 100644 certbot/tests/testdata/google_certificate.pem create mode 100644 certbot/tests/testdata/google_issuer_certificate.pem rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json (100%) rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json (100%) rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/cert1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/chain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/fullchain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/privkey1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/cert1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/chain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/fullchain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/privkey1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0000_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0001_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0002_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0003_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0000_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0001_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0002_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0003_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/README (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/cert.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/chain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/fullchain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/privkey.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/README (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/cert.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/chain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/fullchain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/privkey.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/options-ssl-apache.conf (100%) rename tests/{letstest/testdata => integration}/sample-config/renewal/a.encryption-example.com.conf (100%) rename tests/{letstest/testdata => integration}/sample-config/renewal/b.encryption-example.com.conf (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53c158f8f..3eea1923d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,9 @@ More details about these changes can be found on our GitHub repo. * Avoid reprocessing challenges that are already validated when a certificate is issued. +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index c4a389cd5..66c68eb38 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -19,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey # https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore @@ -226,7 +226,7 @@ def verify_renewable_cert(renewable_cert): def verify_renewable_cert_sig(renewable_cert): - """ Verifies the signature of a `.storage.RenewableCert` object. + """Verifies the signature of a `.storage.RenewableCert` object. :param `.storage.RenewableCert` renewable_cert: cert to verify @@ -239,22 +239,8 @@ def verify_renewable_cert_sig(renewable_cert): cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") + verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, + cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) @@ -262,6 +248,37 @@ def verify_renewable_cert_sig(renewable_cert): raise errors.Error(error_str) +def verify_signed_payload(public_key, signature, payload, signature_hash_algorithm): + """Check the signature of a payload. + + :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature + :param bytes signature: the signature bytes + :param bytes payload: the payload bytes + :param cryptography.hazmat.primitives.hashes.HashAlgorithm + signature_hash_algorithm: algorithm used to hash the payload + + :raises InvalidSignature: If signature verification fails. + :raises errors.Error: If public key type is not supported + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(public_key, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = public_key.verifier( # type: ignore + signature, PKCS1v15(), signature_hash_algorithm + ) + verifier.update(payload) + verifier.verify() + elif isinstance(public_key, EllipticCurvePublicKey): + verifier = public_key.verifier( + signature, ECDSA(signature_hash_algorithm) + ) + verifier.update(payload) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + + def verify_cert_matches_priv_key(cert_path, key_path): """ Verifies that the private key and cert match. diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 049e14827..6f4c0b0fb 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -1,53 +1,79 @@ """Tools for checking certificate revocation.""" import logging import re - +from datetime import datetime, timedelta from subprocess import Popen, PIPE +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp # pylint: disable=import-error + getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp = None # type: ignore +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +import requests + +from acme.magic_typing import Optional, Tuple # pylint: disable=unused-import, no-name-in-module +from certbot import crypto_util from certbot import errors from certbot import util logger = logging.getLogger(__name__) + class RevocationChecker(object): - "This class figures out OCSP checking on this system, and performs it." + """This class figures out OCSP checking on this system, and performs it.""" - def __init__(self): + def __init__(self, enforce_openssl_binary_usage=False): self.broken = False + self.use_openssl_binary = enforce_openssl_binary_usage or not ocsp - if not util.exe_exists("openssl"): - logger.info("openssl not installed, can't check revocation") - self.broken = True - return - - # New versions of openssl want -header var=val, old ones want -header var val - test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], - stdout=PIPE, stderr=PIPE, universal_newlines=True) - _out, err = test_host_format.communicate() - if "Missing =" in err: - self.host_args = lambda host: ["Host=" + host] - else: - self.host_args = lambda host: ["Host", host] + if self.use_openssl_binary: + if not util.exe_exists("openssl"): + logger.info("openssl not installed, can't check revocation") + self.broken = True + return + # New versions of openssl want -header var=val, old ones want -header var val + test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], + stdout=PIPE, stderr=PIPE, universal_newlines=True) + _out, err = test_host_format.communicate() + if "Missing =" in err: + self.host_args = lambda host: ["Host=" + host] + else: + self.host_args = lambda host: ["Host", host] def ocsp_revoked(self, cert_path, chain_path): + # type: (str, str) -> bool """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call :param str cert_path: Path to certificate :param str chain_path: Path to intermediate cert - :rtype bool or None: :returns: True if revoked; False if valid or the check failed + :rtype: bool """ if self.broken: return False - - url, host = self.determine_ocsp_server(cert_path) - if not host: + url, host = _determine_ocsp_server(cert_path) + if not host or not url: return False + + if self.use_openssl_binary: + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) + else: + return _check_ocsp_cryptography(cert_path, chain_path, url) + + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): + # type: (str, str, str, str) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -65,33 +91,131 @@ class RevocationChecker(object): except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False - return _translate_ocsp_query(cert_path, output, err) - def determine_ocsp_server(self, cert_path): - """Extract the OCSP server host from a certificate. +def _determine_ocsp_server(cert_path): + # type: (str) -> Tuple[Optional[str], Optional[str]] + """Extract the OCSP server host from a certificate. - :param str cert_path: Path to the cert we're checking OCSP for - :rtype tuple: - :returns: (OCSP server URL or None, OCSP server host or None) + :param str cert_path: Path to the cert we're checking OCSP for + :rtype tuple: + :returns: (OCSP server URL or None, OCSP server host or None) - """ - try: - url, _err = util.run_script( - ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logger.debug) - except errors.SubprocessError: - logger.info("Cannot extract OCSP URI from %s", cert_path) - return None, None + """ + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + try: + extension = cert.extensions.get_extension_for_class(x509.AuthorityInformationAccess) + ocsp_oid = x509.AuthorityInformationAccessOID.OCSP + descriptions = [description for description in extension.value + if description.access_method == ocsp_oid] + + url = descriptions[0].access_location.value + except (x509.ExtensionNotFound, IndexError): + logger.info("Cannot extract OCSP URI from %s", cert_path) + return None, None + + url = url.rstrip() + host = url.partition("://")[2].rstrip("/") + + if host: + return url, host + else: + logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + return None, None + + +def _check_ocsp_cryptography(cert_path, chain_path, url): + # type: (str, str, str) -> bool + # Retrieve OCSP response + with open(chain_path, 'rb') as file_handler: + issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + builder = ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + request_binary = request.public_bytes(serialization.Encoding.DER) + try: + response = requests.post(url, data=request_binary, + headers={'Content-Type': 'application/ocsp-request'}) + except requests.exceptions.RequestException: + logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) + return False + if response.status_code != 200: + logger.info("OCSP check failed for %s (HTTP status: %d)", cert_path, response.status_code) + return False + + response_ocsp = ocsp.load_der_ocsp_response(response.content) + + # Check OCSP response validity + if response_ocsp.response_status != ocsp.OCSPResponseStatus.SUCCESSFUL: + logger.error("Invalid OCSP response status for %s: %s", + cert_path, response_ocsp.response_status) + return False + + # Check OCSP signature + try: + _check_ocsp_response(response_ocsp, request, issuer) + except UnsupportedAlgorithm as e: + logger.error(str(e)) + except errors.Error as e: + logger.error(str(e)) + except InvalidSignature: + logger.error('Invalid signature on OCSP response for %s', cert_path) + except AssertionError as error: + logger.error('Invalid OCSP response for %s: %s.', cert_path, str(error)) + else: + # Check OCSP certificate status + logger.debug("OCSP certificate status for %s is: %s", + cert_path, response_ocsp.certificate_status) + return response_ocsp.certificate_status == ocsp.OCSPCertStatus.REVOKED + + return False + + +def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): + """Verify that the OCSP is valid for serveral criterias""" + # Assert OCSP response corresponds to the certificate we are talking about + if response_ocsp.serial_number != request_ocsp.serial_number: + raise AssertionError('the certificate in response does not correspond ' + 'to the certificate in request') + + # Assert signature is valid + _check_ocsp_response_signature(response_ocsp, issuer_cert) + + # Assert issuer in response is the expected one + if (not isinstance(response_ocsp.hash_algorithm, type(request_ocsp.hash_algorithm)) + or response_ocsp.issuer_key_hash != request_ocsp.issuer_key_hash + or response_ocsp.issuer_name_hash != request_ocsp.issuer_name_hash): + raise AssertionError('the issuer does not correspond to issuer of the certificate.') + + # In following checks, two situations can occur: + # * nextUpdate is set, and requirement is thisUpdate < now < nextUpdate + # * nextUpdate is not set, and requirement is thisUpdate < now + # NB1: We add a validity period tolerance to handle clock time inconsistencies, + # value is 5 min like for OpenSSL. + # NB2: Another check is to verify that thisUpdate is not too old, it is optional + # for OpenSSL, so we do not do it here. + # See OpenSSL implementation as a reference: + # https://github.com/openssl/openssl/blob/ef45aa14c5af024fcb8bef1c9007f3d1c115bd85/crypto/ocsp/ocsp_cl.c#L338-L391 + now = datetime.now() + if not response_ocsp.this_update: + raise AssertionError('param thisUpdate is not set.') + if response_ocsp.this_update > now + timedelta(minutes=5): + raise AssertionError('param thisUpdate is in the future.') + if response_ocsp.next_update and response_ocsp.next_update < now - timedelta(minutes=5): + raise AssertionError('param nextUpdate is in the past.') + + +def _check_ocsp_response_signature(response_ocsp, issuer_cert): + """Verify an OCSP response signature against certificate issuer""" + # Following line may raise UnsupportedAlgorithm + chosen_hash = response_ocsp.signature_hash_algorithm + crypto_util.verify_signed_payload(issuer_cert.public_key(), response_ocsp.signature, + response_ocsp.tbs_response_bytes, chosen_hash) - url = url.rstrip() - host = url.partition("://")[2].rstrip("/") - if host: - return url, host - else: - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) - return None, None def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): """Parse openssl's weird output to work out what it means.""" @@ -102,7 +226,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): warning = good.group(1) if good else None - if (not "Response verify OK" in ocsp_errors) or (good and warning) or unknown: + if ("Response verify OK" not in ocsp_errors) or (good and warning) or unknown: logger.info("Revocation status for %s is unknown", cert_path) logger.debug("Uncertain output:\n%s\nstderr:\n%s", ocsp_output, ocsp_errors) return False @@ -115,6 +239,5 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): return True else: logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", - ocsp_output, ocsp_errors) + ocsp_output, ocsp_errors) return False - diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 55cd24adb..ad3467e5a 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -1,18 +1,33 @@ """Tests for ocsp.py""" # pylint: disable=protected-access - import unittest +from datetime import datetime, timedelta +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +from cryptography import x509 +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp as ocsp_lib # pylint: disable=import-error + getattr(ocsp_lib.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp_lib = None # type: ignore import mock from certbot import errors +from certbot.tests import util as test_util out = """Missing = in header key=value ocsp: Use -help for summary. """ -class OCSPTest(unittest.TestCase): +class OCSPTestOpenSSL(unittest.TestCase): + """ + OCSP revokation tests using OpenSSL binary. + """ def setUp(self): from certbot import ocsp @@ -22,7 +37,7 @@ class OCSPTest(unittest.TestCase): mock_communicate.communicate.return_value = (None, out) mock_popen.return_value = mock_communicate mock_exists.return_value = True - self.checker = ocsp.RevocationChecker() + self.checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) def tearDown(self): pass @@ -37,23 +52,23 @@ class OCSPTest(unittest.TestCase): mock_exists.return_value = True from certbot import ocsp - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 1) self.assertEqual(checker.host_args("x"), ["Host=x"]) mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) self.assertEqual(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) self.assertEqual(checker.broken, True) - @mock.patch('certbot.ocsp.RevocationChecker.determine_ocsp_server') + @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.util.run_script') def test_ocsp_revoked(self, mock_run, mock_determine): self.checker.broken = True @@ -71,21 +86,12 @@ class OCSPTest(unittest.TestCase): self.assertEqual(self.checker.ocsp_revoked("x", "y"), False) self.assertEqual(mock_run.call_count, 2) + def test_determine_ocsp_server(self): + cert_path = test_util.vector_path('google_certificate.pem') - @mock.patch('certbot.ocsp.logger.info') - @mock.patch('certbot.util.run_script') - def test_determine_ocsp_server(self, mock_run, mock_info): - uri = "http://ocsp.stg-int-x1.letsencrypt.org/" - host = "ocsp.stg-int-x1.letsencrypt.org" - mock_run.return_value = uri, "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (uri, host)) - mock_run.return_value = "ftp:/" + host + "/", "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) - self.assertEqual(mock_info.call_count, 1) - - c = "confusion" - mock_run.side_effect = errors.SubprocessError(c) - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) + from certbot import ocsp + result = ocsp._determine_ocsp_server(cert_path) + self.assertEqual(('http://ocsp.digicert.com', 'ocsp.digicert.com'), result) @mock.patch('certbot.ocsp.logger') @mock.patch('certbot.util.run_script') @@ -112,6 +118,129 @@ class OCSPTest(unittest.TestCase): self.assertEqual(mock_log.info.call_count, 1) +@unittest.skipIf(not ocsp_lib, + reason='This class tests functionalities available only on cryptography>=2.5.0') +class OSCPTestCryptography(unittest.TestCase): + """ + OCSP revokation tests using Cryptography >= 2.4.0 + """ + + def setUp(self): + from certbot import ocsp + self.checker = ocsp.RevocationChecker() + self.cert_path = test_util.vector_path('google_certificate.pem') + self.chain_path = test_util.vector_path('google_issuer_certificate.pem') + + @mock.patch('certbot.ocsp._determine_ocsp_server') + @mock.patch('certbot.ocsp._check_ocsp_cryptography') + def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + mock_determine.return_value = ('http://example.com', 'example.com') + self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke(self, mock_ocsp_response, mock_post): + with mock.patch('certbot.ocsp.crypto_util.verify_signed_payload'): + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertTrue(revoked) + + @mock.patch('certbot.ocsp.crypto_util.verify_signed_payload') + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke_resiliency(self, mock_ocsp_response, mock_post, mock_check): + # Server return an invalid HTTP response + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=400) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response in invalid + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response is valid, but certificate status is unknown + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # The OCSP response says that the certificate is revoked, but certificate + # does not contain the OCSP extension. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + with mock.patch('cryptography.x509.Extensions.get_extension_for_class', + side_effect=x509.ExtensionNotFound( + 'Not found', x509.AuthorityInformationAccessOID.OCSP)): + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Valid response, OCSP extension is present, + # but OCSP response uses an unsupported signature. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = UnsupportedAlgorithm('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # And now, the signature itself is invalid. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = InvalidSignature('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Finally, assertion error on OCSP response validity + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = AssertionError('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + +def _construct_mock_ocsp_response(certificate_status, response_status): + cert = x509.load_pem_x509_certificate( + test_util.load_vector('google_certificate.pem'), default_backend()) + issuer = x509.load_pem_x509_certificate( + test_util.load_vector('google_issuer_certificate.pem'), default_backend()) + builder = ocsp_lib.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + + return mock.Mock( + response_status=response_status, + certificate_status=certificate_status, + serial_number=request.serial_number, + issuer_key_hash=request.issuer_key_hash, + issuer_name_hash=request.issuer_name_hash, + hash_algorithm=hashes.SHA1(), + next_update=datetime.now() + timedelta(days=1), + this_update=datetime.now() - timedelta(days=1), + signature_algorithm_oid=x509.oid.SignatureAlgorithmOID.RSA_WITH_SHA1, + ) + + # pylint: disable=line-too-long openssl_confused = ("", """ /etc/letsencrypt/live/example.org/cert.pem: good @@ -165,5 +294,6 @@ revoked """, """Response verify OK""") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/testdata/google_certificate.pem b/certbot/tests/testdata/google_certificate.pem new file mode 100644 index 000000000..c26fea0b1 --- /dev/null +++ b/certbot/tests/testdata/google_certificate.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1 +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk +IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy +MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB +BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF +Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG +A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD +VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ +w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj +/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE +sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ +xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM +fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag +re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG +A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE +AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0 +oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy +LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy +dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB +FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF +BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS +BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0 +U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA +MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY +BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT +Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg +U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE +AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51 +vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV +CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN +8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG +eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v +ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN +HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB +Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk +myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq +WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik= +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/google_issuer_certificate.pem b/certbot/tests/testdata/google_issuer_certificate.pem new file mode 100644 index 000000000..50db47bc4 --- /dev/null +++ b/certbot/tests/testdata/google_issuer_certificate.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEXDCCA0SgAwIBAgINAeOpMBz8cgY4P5pTHTANBgkqhkiG9w0BAQsFADBMMSAw +HgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFs +U2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEy +MTUwMDAwNDJaMFQxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3Qg +U2VydmljZXMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzMw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKUkvqHv/OJGuo2nIYaNVW +XQ5IWi01CXZaz6TIHLGp/lOJ+600/4hbn7vn6AAB3DVzdQOts7G5pH0rJnnOFUAK +71G4nzKMfHCGUksW/mona+Y2emJQ2N+aicwJKetPKRSIgAuPOB6Aahh8Hb2XO3h9 +RUk2T0HNouB2VzxoMXlkyW7XUR5mw6JkLHnA52XDVoRTWkNty5oCINLvGmnRsJ1z +ouAqYGVQMc/7sy+/EYhALrVJEA8KbtyX+r8snwU5C1hUrwaW6MWOARa8qBpNQcWT +kaIeoYvy/sGIJEmjR0vFEwHdp1cSaWIr6/4g72n7OqXwfinu7ZYW97EfoOSQJeAz +AgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH +AwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHfCuFCa +Z3Z2sS3ChtCDoH6mfrpLMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYu +MDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdv +b2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dz +cjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYc +aHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEA +HLeJluRT7bvs26gyAZ8so81trUISd7O45skDUmAge1cnxhG1P2cNmSxbWsoiCt2e +ux9LSD+PAj2LIYRFHW31/6xoic1k4tbWXkDCjir37xTTNqRAMPUyFRWSdvt+nlPq +wnb8Oa2I/maSJukcxDjNSfpDh/Bd1lZNgdd/8cLdsE3+wypufJ9uXO1iQpnh9zbu +FIwsIONGl1p3A8CgxkqI/UAih3JaGOqcpcdaCIzkBaR9uYQ1X4k2Vg5APRLouzVy +7a8IVk6wuy6pm+T7HT4LY8ibS5FEZlfAFLSW8NwsVz9SBK2Vqn1N0PIMn5xA6NZV +c7o835DLAFshEWfC7TIe3g== +-----END CERTIFICATE----- diff --git a/certbot/util.py b/certbot/util.py index 416075ce8..097593e9d 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -62,7 +62,7 @@ def run_script(params, log=logger.error): """Run the script with the given params. :param list params: List of parameters to pass to Popen - :param logging.Logger log: Logger to use for errors + :param callable log: Logger method to use for errors """ try: diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 02656521f..19aaa4eeb 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1034,26 +1034,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1fac78836..dff57dfd5 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -60,26 +60,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 630571148..9d1ae9ec2 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -15,9 +15,11 @@ command -v python > /dev/null || (echo "Error, python executable is not in the P . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx +CURRENT_DIR="$(pwd)" cleanup_and_exit() { EXIT_STATUS=$? + cd $CURRENT_DIR if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` then echo Kill server subprocess, left running by abnormal exit @@ -527,3 +529,56 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then fi coverage report --fail-under 64 --include 'certbot/*' --show-missing + +# Test OCSP status + +## OCSP 1: Check stale OCSP status +pushd ./tests/integration + +OUT=`common certificates --config-dir sample-config` +TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` +EXPIRED=`echo "$OUT" | grep EXPIRED | wc -l` + +if [ "$TEST_CERTS" != 2 ] ; then + echo "Did not find two test certs as expected ($TEST_CERTS)" + exit 1 +fi + +if [ "$EXPIRED" != 2 ] ; then + echo "Did not find two test certs as expected ($EXPIRED)" + exit 1 +fi + +popd + +## OSCP 2: Check live certificate OCSP status (VALID) +common --domains le-ocsp-check.wtf +OUT=`common certificates` +VALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep VALID | wc -l` +EXPIRED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep EXPIRED | wc -l` + +if [ "$VALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be VALID" + exit 1 +fi + +if [ "$EXPIRED" != 0 ] ; then + echo "Did not expect le-ocsp-check.wtf to be EXPIRED" + exit 1 +fi + +## OSCP 3: Check live certificate OCSP status (REVOKED) +common revoke --cert-name le-ocsp-check.wtf --no-delete-after-revoke +OUT=`common certificates` +INVALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep INVALID | wc -l` +REVOKED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep REVOKED | wc -l` + +if [ "$INVALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be INVALID" + exit 1 +fi + +if [ "$REVOKED" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be REVOKED" + exit 1 +fi diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem b/tests/integration/sample-config/csr/0000_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem rename to tests/integration/sample-config/csr/0000_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem b/tests/integration/sample-config/csr/0001_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem rename to tests/integration/sample-config/csr/0001_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem b/tests/integration/sample-config/csr/0002_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem rename to tests/integration/sample-config/csr/0002_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem b/tests/integration/sample-config/csr/0003_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem rename to tests/integration/sample-config/csr/0003_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem b/tests/integration/sample-config/keys/0000_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem rename to tests/integration/sample-config/keys/0000_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem b/tests/integration/sample-config/keys/0001_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem rename to tests/integration/sample-config/keys/0001_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem b/tests/integration/sample-config/keys/0002_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem rename to tests/integration/sample-config/keys/0002_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem b/tests/integration/sample-config/keys/0003_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem rename to tests/integration/sample-config/keys/0003_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/README b/tests/integration/sample-config/live/a.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/README rename to tests/integration/sample-config/live/a.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem b/tests/integration/sample-config/live/a.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/a.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem b/tests/integration/sample-config/live/a.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/a.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/a.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/README b/tests/integration/sample-config/live/b.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/README rename to tests/integration/sample-config/live/b.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem b/tests/integration/sample-config/live/b.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/b.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem b/tests/integration/sample-config/live/b.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/b.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/b.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/options-ssl-apache.conf b/tests/integration/sample-config/options-ssl-apache.conf similarity index 100% rename from tests/letstest/testdata/sample-config/options-ssl-apache.conf rename to tests/integration/sample-config/options-ssl-apache.conf diff --git a/tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf b/tests/integration/sample-config/renewal/a.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf rename to tests/integration/sample-config/renewal/a.encryption-example.com.conf diff --git a/tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf b/tests/integration/sample-config/renewal/b.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf rename to tests/integration/sample-config/renewal/b.encryption-example.com.conf diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 2cbe66a83..081ff3829 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -17,27 +17,6 @@ letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -# we have to jump through some hoops to cope with relative paths in renewal -# conf files ... -# 1. be in the right directory -cd tests/letstest/testdata/ - -# 2. refer to the config with the same level of relativity that it itself -# contains :/ -OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade` -TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` -REVOKED=`echo "$OUT" | grep REVOKED | wc -l` - -if [ "$TEST_CERTS" != 2 ] ; then - echo "Did not find two test certs as expected ($TEST_CERTS)" - exit 1 -fi - -if [ "$REVOKED" != 1 ] ; then - echo "Did not find one revoked cert as expected ($REVOKED)" - exit 1 -fi - if ! letsencrypt-auto --help --no-self-upgrade | grep -F "letsencrypt-auto [SUBCOMMAND]"; then echo "letsencrypt-auto not included in help output!" exit 1 From a809c3697d433515517ba07b85c5ba42b34ff7c1 Mon Sep 17 00:00:00 2001 From: schoen Date: Wed, 27 Feb 2019 16:32:57 -0800 Subject: [PATCH 587/639] Warn sysadmins about privilege escalation risk (#6795) --- docs/install.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/install.rst b/docs/install.rst index 35b262482..eae40c1f0 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,6 +11,8 @@ About Certbot *Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. +System administrators can use Certbot directly to request certificates; they should *not* allow unprivileged users to run arbitrary Certbot commands as ``root``, because Certbot allows its user to specify arbitrary file locations and run arbitrary scripts. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for From 5e849e03f6e8dd2aca31888f9482da63359bb961 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 20:35:58 +0100 Subject: [PATCH 588/639] [Windows] Fix pipstrap (#6775) * Fix pipstrap on windows * Pipstrap pin setuptools version, so explicit install it is not needed anymore. * Rebuild letsencrypt-auto source * Use sys.executable in pipstrap to allow straightforward execution in a venv by choosing the python interpreter from this venv. * Update letsencrypt-auto * Simulate test-everything * Revert "Simulate test-everything" This reverts commit b62c4d719a6e741cb11126c7490097a79c68cf4d. * Clean pipstrap code --- letsencrypt-auto-source/letsencrypt-auto | 22 +++++++++--------- letsencrypt-auto-source/pieces/pipstrap.py | 22 +++++++++--------- tools/_venv_common.py | 27 ++++++++-------------- 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 19aaa4eeb..c390f28ec 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1221,7 +1221,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1240,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1262,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1343,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1354,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1372,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 6a00dd9cb..f04ebe6fe 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,7 +23,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -43,7 +42,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -65,7 +64,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -146,7 +145,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -156,12 +156,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -174,4 +174,4 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 540842773..09383b4c0 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -107,13 +107,13 @@ def subprocess_with_print(cmd, env=os.environ, shell=False): subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_bin_path(venv_path): +def get_venv_python_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return os.path.abspath(os.path.dirname(python_linux)) + return os.path.abspath(python_linux) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return os.path.abspath(os.path.dirname(python_windows)) + return os.path.abspath(python_windows) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -149,17 +149,12 @@ def main(venv_name, venv_args, args): command.extend(shlex.split(venv_args)) subprocess_with_print(command) - # We execute the following commands in the context of the virtual environment, to install - # the packages in it. To do so, we append the venv binary to the PATH that will be used for - # these commands. With this trick, correct python executable will be selected. - new_environ = os.environ.copy() - new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) - subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), - env=new_environ, shell=True) - subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', - env=new_environ, shell=True) - subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), - env=new_environ, shell=True) + # Using the python executable from venv, we ensure to execute following commands in this venv. + py_venv = get_venv_python_path(venv_name) + subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + command = [py_venv, os.path.abspath('tools/pip_install.py')] + command.extend(args) + subprocess_with_print(command) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -179,6 +174,4 @@ def main(venv_name, venv_args, args): if __name__ == '__main__': - main('venv', - '', - sys.argv[1:]) + main('venv', '', sys.argv[1:]) From efc8d49806b14a31d88cfc0f1b6daca1dd373d8d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 21:49:36 +0100 Subject: [PATCH 589/639] Disable rerun feature of Travis (#6800) The rerun capability in a test campaign can be a nice feature. It can also a bad design. It is right that with high level tests, like performance or end-to-end tests, a given test runtime can depend on an external component, outside the scope of the developers, that would spuriously fail. In theses situations, having the capacity to rerun several time a test can be a great benefit. Indeed, as these tests are inherently flaky, the rerun greatly reduces their failure because of external reasons, reducing also the Pierre et le Loup effect (from a well-known french book for children): because tests constantly fail for no reason, you stop to listen to them and to not see when they fail for a real reason. However this not apply to unit tests, or operations about code quality. For theses executions, the flakiness should not exist: a unit test is supposed to have no external dependency. A rerun approach would just hide a situation that is not desirable for this kind of tests Also effectively I see that our tests are usually not flaky: so the only effect of the rerun is to give the failure state about a test three times slower. If a test becomes flaky, this should be fixed, and so be visible immediately in our CI. For these reasons, I remove the travis_retry in the script section of .travis-ci.yml, to call directly tox and let the pipeline fails on the first error. * Disable rerun feature of Travis * Update .travis.yml * Remove completely the retry logic --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d59ec9e1..4adab1e4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -197,10 +197,10 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" +install: "$(command -v pip || command -v pip3) install codecov tox" script: - - travis_retry tox - - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' + - tox + - '[ -z "${BOULDER_INTEGRATION+x}" ] || (tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' From 841f8efd0aa7dc2ba249b303e6be664d0a6647e3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 22:18:06 +0100 Subject: [PATCH 590/639] [Unix] Create a framework for certbot integration tests: PART 1 (#6578) * First part * Several optimizations about the docker env setup * Documentation * Various corrections and documentation. Add acme and certbot explicitly as dependencies of certbot-ci. * Correct a variable misinterpreted as a pytest hook * Correct strict parsing option on pebble * Refactor acme setup to be executed from pytest hooks. * Pass TRAVIS env variable to trigger specific xdist logic * Retrigger build. * Work in progress * Config operational * Propagate to xdist * Corrections on acme and misc * Correct subnet for pebble * Remove gobetween, as tls-sni challenges are not tested anymore. * Improve pebble setup. Reduce LOC. * Update acme.py * Optimize acme ca setup, with less temporary assets * Silent setup * Clean code * Remove unused workspace * Use default network driver * Remove bridge * Update package documentation * Remove rerun capability for integration tests, not needed. * Add documentation * Variable for all ports and subnets used by the stack * Update certbot-ci/certbot_integration_tests/conftest.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/misc.py Co-Authored-By: adferrand * Update tox.ini Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/misc.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/conftest.py Co-Authored-By: adferrand * Rename to acme_server * Add comment * Refactor in a unique context fixture * Remove the need of CERTBOT_ACME_XDIST environment variable * Remove nonstrict/strict options in pebble * Clean dependencies * Clean tox * Change function name * Add comment about coveragerc specificities * Change a comment. * Update setup.py * Update conftest.py * Use the production-ready docker-compose.yml file for Pebble * New style class * Tune pebble to have a stable test environment * Pin a dependency --- .../certbot_integration_tests/.coveragerc | 8 + .../certbot_integration_tests/__init__.py | 1 + .../certbot_tests/__init__.py | 0 .../certbot_tests/context.py | 16 ++ .../certbot_tests/test_main.py | 40 ++++ .../certbot_integration_tests/conftest.py | 92 +++++++++ .../nginx_tests/__init__.py | 0 .../nginx_tests/context.py | 5 + .../nginx_tests/test_main.py | 17 ++ .../utils/__init__.py | 0 .../utils/acme_server.py | 194 ++++++++++++++++++ .../certbot_integration_tests/utils/misc.py | 45 ++++ certbot-ci/setup.py | 45 ++++ tools/dev_constraints.txt | 1 + tox.ini | 12 ++ 15 files changed, 476 insertions(+) create mode 100644 certbot-ci/certbot_integration_tests/.coveragerc create mode 100644 certbot-ci/certbot_integration_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/context.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/test_main.py create mode 100644 certbot-ci/certbot_integration_tests/conftest.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/context.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/test_main.py create mode 100644 certbot-ci/certbot_integration_tests/utils/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/utils/acme_server.py create mode 100644 certbot-ci/certbot_integration_tests/utils/misc.py create mode 100644 certbot-ci/setup.py diff --git a/certbot-ci/certbot_integration_tests/.coveragerc b/certbot-ci/certbot_integration_tests/.coveragerc new file mode 100644 index 000000000..de36d4e02 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/.coveragerc @@ -0,0 +1,8 @@ +[run] +# Avoid false warnings because certbot packages are not installed in the thread that executes +# the coverage: indeed, certbot is launched as a CLI from a subprocess. +disable_warnings = module-not-imported,no-data-collected + +[report] +# Exclude unit tests in coverage during integration tests. +omit = **/*_test.py,**/tests/*,**/certbot_nginx/parser_obj.py diff --git a/certbot-ci/certbot_integration_tests/__init__.py b/certbot-ci/certbot_integration_tests/__init__.py new file mode 100644 index 000000000..434a85a23 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/__init__.py @@ -0,0 +1 @@ +"""Package certbot_integration_test is for tests that require a live acme ca server instance""" diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py new file mode 100644 index 000000000..9045cd37d --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -0,0 +1,16 @@ +class IntegrationTestsContext(object): + """General fixture describing a certbot integration tests context""" + def __init__(self, request): + self.request = request + if hasattr(request.config, 'slaveinput'): # Worker node + self.worker_id = request.config.slaveinput['slaveid'] + self.acme_xdist = request.config.slaveinput['acme_xdist'] + else: # Primary node + self.worker_id = 'primary' + self.acme_xdist = request.config.acme_xdist + self.directory_url = self.acme_xdist['directory_url'] + self.tls_alpn_01_port = self.acme_xdist['https_port'][self.worker_id] + self.http_01_port = self.acme_xdist['http_port'][self.worker_id] + + def cleanup(self): + pass diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py new file mode 100644 index 000000000..5b0981b36 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -0,0 +1,40 @@ +import requests +import urllib3 + +import pytest + +from certbot_integration_tests.certbot_tests import context as certbot_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = certbot_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello_1(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass + + +def test_hello_2(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py new file mode 100644 index 000000000..892c16266 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -0,0 +1,92 @@ +""" +General conftest for pytest execution of all integration tests lying +in the certbot_integration tests package. +As stated by pytest documentation, conftest module is used to set on +for a directory a specific configuration using built-in pytest hooks. + +See https://docs.pytest.org/en/latest/reference.html#hook-reference +""" +import contextlib +import sys +import subprocess + +from certbot_integration_tests.utils import acme_server as acme_lib + + +def pytest_addoption(parser): + """ + Standard pytest hook to add options to the pytest parser. + :param parser: current pytest parser that will be used on the CLI + """ + parser.addoption('--acme-server', default='pebble', + choices=['boulder-v1', 'boulder-v2', 'pebble'], + help='select the ACME server to use (boulder-v1, boulder-v2, ' + 'pebble), defaulting to pebble') + + +def pytest_configure(config): + """ + Standard pytest hook used to add a configuration logic for each node of a pytest run. + :param config: the current pytest configuration + """ + if not hasattr(config, 'slaveinput'): # If true, this is the primary node + with _print_on_err(): + config.acme_xdist = _setup_primary_node(config) + + +def pytest_configure_node(node): + """ + Standard pytest-xdist hook used to configure a worker node. + :param node: current worker node + """ + node.slaveinput['acme_xdist'] = node.config.acme_xdist + + +@contextlib.contextmanager +def _print_on_err(): + """ + During pytest-xdist setup, stdout is used for nodes communication, so print is useless. + However, stderr is still available. This context manager transfers stdout to stderr + for the duration of the context, allowing to display prints to the user. + """ + old_stdout = sys.stdout + sys.stdout = sys.stderr + try: + yield + finally: + sys.stdout = old_stdout + + +def _setup_primary_node(config): + """ + Setup the environment for integration tests. + Will: + - check runtime compatiblity (Docker, docker-compose, Nginx) + - create a temporary workspace and the persistent GIT repositories space + - configure and start paralleled ACME CA servers using Docker + - transfer ACME CA servers configurations to pytest nodes using env variables + :param config: Configuration of the pytest primary node + """ + # Check for runtime compatibility: some tools are required to be available in PATH + try: + subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + try: + subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + # Parameter numprocesses is added to option by pytest-xdist + workers = ['primary'] if not config.option.numprocesses\ + else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] + + # By calling setup_acme_server we ensure that all necessary acme server instances will be + # fully started. This runtime is reflected by the acme_xdist returned. + acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers) + print('ACME xdist config:\n{0}'.format(acme_xdist)) + + return acme_xdist diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py b/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py new file mode 100644 index 000000000..6d7d6012b --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -0,0 +1,5 @@ +from certbot_integration_tests.certbot_tests import context as certbot_context + + +class IntegrationTestsContext(certbot_context.IntegrationTestsContext): + """General fixture describing a certbot-nginx integration tests context""" diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py new file mode 100644 index 000000000..472e5e7b7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -0,0 +1,17 @@ +import pytest + +from certbot_integration_tests.nginx_tests import context as nginx_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = nginx_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello(context): + print(context.directory_url) diff --git a/certbot-ci/certbot_integration_tests/utils/__init__.py b/certbot-ci/certbot_integration_tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py new file mode 100644 index 000000000..33ef05194 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -0,0 +1,194 @@ +"""Module to setup an ACME CA server environment able to run multiple tests in parallel""" +from __future__ import print_function +import tempfile +import atexit +import os +import subprocess +import shutil +import sys +from os.path import join + +import requests +import json +import yaml + +from certbot_integration_tests.utils import misc + +# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble. +CHALLTESTSRV_PORT = 8055 +HTTP_01_PORT = 5002 + + +def setup_acme_server(acme_server, nodes): + """ + This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel + execution of integration tests against the unique http-01 port expected by the ACME CA server. + Instances are properly closed and cleaned when the Python process exits using atexit. + Typically all pytest integration tests will be executed in this context. + This method returns an object describing ports and directory url to use for each pytest node + with the relevant pytest xdist node. + :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) + :param str[] nodes: list of node names that will be setup by pytest xdist + :return: a dict describing the challenge ports that have been setup for the nodes + :rtype: dict + """ + acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' + acme_xdist = _construct_acme_xdist(acme_server, nodes) + workspace = _construct_workspace(acme_type) + + _prepare_traefik_proxy(workspace, acme_xdist) + _prepare_acme_server(workspace, acme_type, acme_xdist) + + return acme_xdist + + +def _construct_acme_xdist(acme_server, nodes): + """Generate and return the acme_xdist dict""" + acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} + + # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + if acme_server == 'pebble': + acme_xdist['directory_url'] = 'https://localhost:14000/dir' + else: # boulder + port = 4001 if acme_server == 'boulder-v2' else 4000 + acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port) + + acme_xdist['http_port'] = {node: port for (node, port) + in zip(nodes, range(5200, 5200 + len(nodes)))} + acme_xdist['https_port'] = {node: port for (node, port) + in zip(nodes, range(5100, 5100 + len(nodes)))} + + return acme_xdist + + +def _construct_workspace(acme_type): + """Create a temporary workspace for integration tests stack""" + workspace = tempfile.mkdtemp() + + def cleanup(): + """Cleanup function to call that will teardown relevant dockers and their configuration.""" + for instance in [acme_type, 'traefik']: + print('=> Tear down the {0} instance...'.format(instance)) + instance_path = join(workspace, instance) + try: + if os.path.isfile(join(instance_path, 'docker-compose.yml')): + _launch_command(['docker-compose', 'down'], cwd=instance_path) + except subprocess.CalledProcessError: + pass + print('=> Finished tear down of {0} instance.'.format(acme_type)) + + shutil.rmtree(workspace) + + # Here with atexit we ensure that clean function is called no matter what. + atexit.register(cleanup) + + return workspace + + +def _prepare_acme_server(workspace, acme_type, acme_xdist): + """Configure and launch the ACME server, Boulder or Pebble""" + print('=> Starting {0} instance deployment...'.format(acme_type)) + instance_path = join(workspace, acme_type) + try: + # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production. + _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type), + '--single-branch', '--depth=1', instance_path]) + if acme_type == 'boulder': + # Allow Boulder to ignore usual limit rate policies, useful for tests. + os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), + join(instance_path, 'test/rate-limit-policies.yml')) + if acme_type == 'pebble': + # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid + # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. + with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler: + config = yaml.load(file_handler.read()) + + config['services']['pebble'].setdefault('environment', [])\ + .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0']) + with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler: + file_handler.write(yaml.dump(config)) + + # Launch the ACME CA server. + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + # Wait for the ACME CA server to be up. + print('=> Waiting for {0} instance to respond...'.format(acme_type)) + misc.check_until_timeout(acme_xdist['directory_url']) + + # Configure challtestsrv to answer any A record request with ip of the docker host. + acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50' + response = requests.post('http://localhost:{0}/set-default-ipv4' + .format(acme_xdist['challtestsrv_port']), + json={'ip': '{0}.1'.format(acme_subnet)}) + response.raise_for_status() + + print('=> Finished {0} instance deployment.'.format(acme_type)) + except BaseException: + print('Error while setting up {0} instance.'.format(acme_type)) + raise + + +def _prepare_traefik_proxy(workspace, acme_xdist): + """Configure and launch Traefik, the HTTP reverse proxy""" + print('=> Starting traefik instance deployment...') + instance_path = join(workspace, 'traefik') + traefik_subnet = '10.33.33' + traefik_api_port = 8056 + try: + os.mkdir(instance_path) + + with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h: + file_h.write('''\ +version: '3' +services: + traefik: + image: traefik + command: --api --rest + ports: + - {http_01_port}:80 + - {traefik_api_port}:8080 + networks: + traefiknet: + ipv4_address: {traefik_subnet}.2 +networks: + traefiknet: + ipam: + config: + - subnet: {traefik_subnet}.0/24 +'''.format(traefik_subnet=traefik_subnet, + traefik_api_port=traefik_api_port, + http_01_port=HTTP_01_PORT)) + + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port)) + config = { + 'backends': { + node: { + 'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}} + } for node, port in acme_xdist['http_port'].items() + }, + 'frontends': { + node: { + 'backend': node, 'passHostHeader': True, + 'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}} + } for node in acme_xdist['http_port'].keys() + } + } + response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port), + data=json.dumps(config)) + response.raise_for_status() + + print('=> Finished traefik instance deployment.') + except BaseException: + print('Error while setting up traefik instance.') + raise + + +def _launch_command(command, cwd=os.getcwd()): + """Launch silently an OS command, output will be displayed in case of failure""" + try: + subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True) + except subprocess.CalledProcessError as e: + sys.stderr.write(e.output) + raise diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py new file mode 100644 index 000000000..a3b134788 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -0,0 +1,45 @@ +""" +Misc module contains stateless functions that could be used during pytest execution, +or outside during setup/teardown of the integration tests environment. +""" +import os +import time +import contextlib + +import requests + + +def check_until_timeout(url): + """ + Wait and block until given url responds with status 200, or raise an exception + after 150 attempts. + :param str url: the URL to test + :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL + """ + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + for _ in range(0, 150): + time.sleep(1) + try: + if requests.get(url, verify=False).status_code == 200: + return + except requests.exceptions.ConnectionError: + pass + + raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url)) + + +@contextlib.contextmanager +def execute_in_given_cwd(cwd): + """ + Context manager that will execute any command in the given cwd after entering context, + and restore current cwd when context is destroyed. + :param str cwd: the path to use as the temporary current workspace for python execution + """ + current_cwd = os.getcwd() + try: + os.chdir(cwd) + yield + finally: + os.chdir(current_cwd) diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py new file mode 100644 index 000000000..595bba69e --- /dev/null +++ b/certbot-ci/setup.py @@ -0,0 +1,45 @@ +from setuptools import setup +from setuptools import find_packages + + +version = '0.32.0.dev0' + +install_requires = [ + 'pytest', + 'pytest-cov', + 'pytest-xdist', + 'pytest-sugar', + 'coverage', + 'requests', + 'pyyaml', +] + +setup( + name='certbot-ci', + version=version, + description="Certbot continuous integration framework", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, +) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..f8c3f4461 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,6 +51,7 @@ pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.22.5 +pytest-sugar==0.9.2 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 diff --git a/tox.ini b/tox.ini index b386ebc86..2c5fe0644 100644 --- a/tox.ini +++ b/tox.ini @@ -250,3 +250,15 @@ commands = whitelist_externals = docker-compose passenv = DOCKER_* + +[testenv:integration] +commands = + {[base]pip_install} acme . certbot-nginx certbot-ci + pytest {toxinidir}/certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} \ + --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ + --cov-config={toxinidir}/certbot-ci/certbot_integration_tests/.coveragerc \ + -W 'ignore:Unverified HTTPS request' + coverage report --fail-under=65 --show-missing +passenv = + DOCKER_* From 7161e792e80d4b0ffa404199248334b2b34c0235 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 22:54:09 +0100 Subject: [PATCH 591/639] Fix the Nginx configuration during integration tests (#6801) If you execute `tests/lock_test.py` or `tox -e integration` on a fairly recent machine, you will get the following error during tests executing against a live Nginx instance: ``` no "ssl_certificate" is defined in server listening on SSL port while SSL handshaking, client: x.x.x.x, server: y:y:y:y:z ``` Indeed, having no defined ssl certificate for a ssl port would inevitably lead to an error during the handshake SSL process between a client and this mis-configured nginx instance. However it was not a problem one year before, because the handshake was not occurring in practice: the test just need to have a nginx started, and then immediately proceed to modify the configuration with a correct SSL setup. And nginx was able to start with a mis-configuration on SSL. But then this fix has been done: https://trac.nginx.org/nginx/ticket/178 Basically with this, validation of the configuration is done during nginx startup, that will refuse to start with invalid configuration on SSL. Consequently, all related tests are failing with a sufficiently up-to-date nginx. For now, it is not seen on Travis because Ubuntu Trusty is used, with an old Nginx. The PR fixes that, by generating on the fly self-signed certificates in the two impacted tests, and pushing the right parameters in the Nginx configuration. * Fix nginx configuration with self-signed certificates generated on the fly * Fix lint/mypy * Fix old cryptography * Unattended openssl * Update lock_test.py --- .../tests/boulder-integration.conf.sh | 29 ++++++--- certbot-nginx/tests/boulder-integration.sh | 6 +- tests/lock_test.py | 62 +++++++++++++++++-- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 470eab28e..80cca2682 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -3,16 +3,22 @@ # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf +# USAGE: ./boulder-integration.conf.sh /path/to/root cert.key cert.pem >> nginx.conf + +ROOT=$1 +CERT_KEY_PATH=$2 +CERT_PATH=$3 + cat < $nginx_conf diff --git a/tests/lock_test.py b/tests/lock_test.py index 0266cf029..aaa8ce2d9 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -2,6 +2,7 @@ from __future__ import print_function import atexit +import datetime import functools import logging import os @@ -11,6 +12,13 @@ import subprocess import sys import tempfile +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +# TODO: once mypy has cryptography types bundled, type: ignore can be removed. +# See https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography.hazmat.primitives import serialization, hashes # type: ignore +from cryptography.hazmat.primitives.asymmetric import rsa + from certbot import lock from certbot import util @@ -102,12 +110,11 @@ def set_up_nginx_dir(root_path): repo_root = check_call('git rev-parse --show-toplevel'.split()).strip() conf_script = os.path.join( repo_root, 'certbot-nginx', 'tests', 'boulder-integration.conf.sh') - # boulder-integration.conf.sh uses the root environment variable as - # the Nginx server root when writing paths - os.environ['root'] = root_path + # Prepare self-signed certificates for Nginx + key_path, cert_path = setup_certificate(root_path) + # Generate Nginx configuration with open(os.path.join(root_path, 'nginx.conf'), 'w') as f: - f.write(check_call(['/bin/sh', conf_script])) - del os.environ['root'] + f.write(check_call(['/bin/sh', conf_script, root_path, key_path, cert_path])) def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): @@ -134,6 +141,51 @@ def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): config_dir, logs_dir, work_dir, nginx_dir).split()) +def setup_certificate(workspace): + """Generate a self-signed certificate for nginx. + :param workspace: path of folder where to put the certificate + :return: tuple containing the key path and certificate path + :rtype: `tuple` + """ + # Generate key + # See comment on cryptography import about type: ignore + private_key = rsa.generate_private_key( # type: ignore + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + subject = issuer = x509.Name([ + x509.NameAttribute(x509.NameOID.COMMON_NAME, u'nginx.wtf') + ]) + certificate = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + private_key.public_key() + ).serial_number( + 1 + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=1) + ).sign(private_key, hashes.SHA256(), default_backend()) + + key_path = os.path.join(workspace, 'cert.key') + with open(key_path, 'wb') as file_handle: + file_handle.write(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + )) + + cert_path = os.path.join(workspace, 'cert.pem') + with open(cert_path, 'wb') as file_handle: + file_handle.write(certificate.public_bytes(serialization.Encoding.PEM)) + + return key_path, cert_path + + def test_command(command, directories): """Assert Certbot acquires locks in a specific order. From a468a3b255e326dbfa2b7d629eb20fa275631254 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 23:21:07 +0100 Subject: [PATCH 592/639] Disable build for commits pushed on master (#6804) Fixes #6746. Every commit on master is always the result of a merged PR, that has been tested by Travis. So retesting the merge commit on master is superfluous. This PR uses build conditions to avoid to launch a build for a commit push on master. I also added the equivalent logic for AppVeyor. Builds cannot received conditions, so it needs to be done on init using Exit-AppVeyorBuild. This command does not fail the build, it finishes it prematurely with success. * Disable build for commit pushed on master (PR are still tested of course) * Equivalent exclusion code for AppVeyor --- .travis.yml | 5 +++++ appveyor.yml | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4adab1e4e..108f853aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,11 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ +# Since master can receive only commits from PR that have already been tested, we avoid with the +# the following condition to launch again a pipeline when the merge commit is pushed to master. +# However master still needs to be set in branches section to allow PR for master to be built. +if: NOT (type = push AND branch = master) + matrix: include: # These environments are always executed diff --git a/appveyor.yml b/appveyor.yml index 2b6b82747..5dcc80ba9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,14 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +init: + # Since master can receive only commits from PR that have already been tested, we avoid with the + # the following condition to launch again a pipeline when the merge commit is pushed to master. + # However master still needs to be set in branches section to allow PR for master to be built. + - ps: | + if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master') + { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } + install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From c7f8f15e9b6c51d67f28af96c84d35b814ba2f73 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Mar 2019 16:04:26 -0800 Subject: [PATCH 593/639] Fix spurious test_relevant_values* failures (#6799) * Mock set_by_cli in _test_relevant_values_common. * Empty commit * Revert "Mock set_by_cli in _test_relevant_values_common." This reverts commit 9dfec8dfa9124d573362f282b691b66af2860fe6. * mock less * Use plugin_common code instead of reimplementing. * Revert 2nd implementation. * Simplify certbot.storage.relevant_values() tests. In addition to cleaning up the code a bit, it also removes the problems we've seen in these tests with the global state used in cli.py. --- certbot/tests/cli_test.py | 2 + certbot/tests/storage_test.py | 109 +++++++++++++--------------------- 2 files changed, 44 insertions(+), 67 deletions(-) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index e16a1bdcf..f6669e7be 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -340,6 +340,8 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( config_dir_option, cli.flag_default(config_dir_option))) + self.assertFalse(cli.option_was_set( + 'authenticator', cli.flag_default('authenticator'))) def test_encode_revocation_reason(self): for reason, code in constants.REVOCATION_REASONS.items(): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index e61ed2aca..872fb4302 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -12,7 +12,6 @@ import pytz import six import certbot -from certbot import cli from certbot import compat from certbot import errors from certbot.storage import ALL_FOUR @@ -37,6 +36,48 @@ def fill_with_sample_data(rc_object): f.write(kind) +class RelevantValuesTest(unittest.TestCase): + """Tests for certbot.storage.relevant_values.""" + + def setUp(self): + self.values = {"server": "example.org"} + + def _call(self, *args, **kwargs): + from certbot.storage import relevant_values + return relevant_values(*args, **kwargs) + + @mock.patch("certbot.cli.option_was_set") + @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") + def test_namespace(self, mock_find_all, mock_option_was_set): + mock_find_all.return_value = ["certbot-foo:bar"] + mock_option_was_set.return_value = True + + self.values["certbot_foo:bar_baz"] = 42 + self.assertEqual( + self._call(self.values.copy()), self.values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_set(self, mock_option_was_set): + mock_option_was_set.return_value = True + + self.values["allow_subset_of_names"] = True + self.values["authenticator"] = "apache" + self.values["rsa_key_size"] = 1337 + expected_relevant_values = self.values.copy() + self.values["hello"] = "there" + + self.assertEqual(self._call(self.values), expected_relevant_values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_unset(self, mock_option_was_set): + mock_option_was_set.return_value = False + + expected_relevant_values = self.values.copy() + self.values["rsa_key_size"] = 2048 + + self.assertEqual(self._call(self.values), expected_relevant_values) + + class BaseRenewableCertTest(test_util.ConfigTestCase): """Base class for setting up Renewable Cert tests. @@ -563,72 +604,6 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_chown.called) - def _test_relevant_values_common(self, values): - defaults = dict((option, cli.flag_default(option)) - for option in ("authenticator", "installer", - "rsa_key_size", "server",)) - mock_parser = mock.Mock(args=[], verb="plugins", - defaults=defaults) - - # make a copy to ensure values isn't modified - values = values.copy() - values.setdefault("server", defaults["server"]) - expected_server = values["server"] - - from certbot.storage import relevant_values - with mock.patch("certbot.cli.helpful_parser", mock_parser): - rv = relevant_values(values) - self.assertIn("server", rv) - self.assertEqual(rv.pop("server"), expected_server) - return rv - - def test_relevant_values(self): - """Test that relevant_values() can reject an irrelevant value.""" - self.assertEqual( - self._test_relevant_values_common({"hello": "there"}), {}) - - def test_relevant_values_default(self): - """Test that relevant_values() can reject a default value.""" - option = "rsa_key_size" - values = {option: cli.flag_default(option)} - self.assertEqual(self._test_relevant_values_common(values), {}) - - def test_relevant_values_nondefault(self): - """Test that relevant_values() can retain a non-default value.""" - values = {"rsa_key_size": 12} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_bool(self): - values = {"allow_subset_of_names": True} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_str(self): - values = {"authenticator": "apache"} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_plugins_none(self): - self.assertEqual( - self._test_relevant_values_common( - {"authenticator": None, "installer": None}), {}) - - @mock.patch("certbot.cli.set_by_cli") - @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") - def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): - mock_set_by_cli.return_value = True - mock_find_all.return_value = ["certbot-foo:bar"] - values = {"certbot_foo:bar_baz": 42} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_server(self): - self.assertEqual( - # _test_relevant_values_common handles testing the server - # value and removes it - self._test_relevant_values_common({"server": "example.org"}), {}) - @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" From 3ed3787bd8792d6ed5c58537a66425a503107721 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 02:03:33 +0100 Subject: [PATCH 594/639] Implement Retry-After, and refactor authorization polling (#6766) Fixes #5789 This PR is about allowing Certbot to respect the Retry-After HTTP header that an ACME CA server can return to a client POSTing to a challenge, to instruct him and retry the request later. However, this feature was not easily implementable in the current code of certbot.auth_handler, because the code became really hard to read. In fact, @bmw was thinking that the code was really deceiving, and a lot of supposed functionalities declared in the comments were in fact not implemented or not functional. So I took the time to understand what was going on, and effectively, most of the code is in fact not usable or not used. Then I did a refactoring against the bare ACME spec about what to do to prepare challenges, instruct the ACME CA server to perform them, then polling regularly the authorization resources until they are decided (valid or invalid). And of course this implementation takes care of Retry-After ^^ I added a lot of comments in the new implementation, to explain what is going on for a future developer. The workflow I used is relying on the relationships between authorizations and challenges states as described in section 7.1.6 of the ACME spec draft: https://datatracker.ietf.org/doc/draft-ietf-acme-acme/ * Clean auth_handler a bit, and implement retry-after. * Remove a debug logger * Correct tests * Fix mypy and lint. Setup max retries and default retry after accordingly. * Ease a comparison in tests * Update documentation * Add tests * Adapt windows coverage threshold to the global LOC reduction * Update certbot/auth_handler.py Co-Authored-By: adferrand * Corrections under review * Correction under review * Update certbot/auth_handler.py Co-Authored-By: adferrand * Corrections under review * Update auth_handler_test.py * Reimplementing user readable report for failed authorizations * Fixes two tests * Fix another test + lint + mypy * Update auth_handler.py * Update auth_handler_test.py * Fix tests * Update certbot/auth_handler.py Co-Authored-By: adferrand * Raise directly the exception on polling timout * Improve interface documentation * Move the wait on top of the loop, to be used initially or after a new loop iteration. Do not wait for negative values. * Always display the report about failed authorizations. * Clarify an exception. * Return, instead of break * Use setdefault * Remove useless assertion * Adapt tests * Improve a test about retry after value. * Update certbot/auth_handler.py Co-Authored-By: adferrand * Add a complete test on best_effort * Add entry to the changelog * Gather all failed authzrs to be reported in one unique report in case of best_effort * Build complete warn/report/raise process about failed authzrs --- CHANGELOG.md | 3 + certbot/auth_handler.py | 392 ++++++++++------------------- certbot/interfaces.py | 13 +- certbot/plugins/common.py | 2 +- certbot/tests/auth_handler_test.py | 313 +++++++++-------------- tox.cover.py | 2 +- 6 files changed, 252 insertions(+), 473 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea1923d..764fc0946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The running of manual plugin hooks is now always included in Certbot's log output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. +* An ACME CA server may return a "Retry-After" HTTP header on authorization polling, as + specified in the ACME protocol, to indicate when the next polling should occur. Certbot now + reads this header if set and respect its value. * The `acme` module avoids sending the `keyAuthorization` field in the JWS payload when responding to a challenge as the field is not included in the current ACME protocol. To ease the migration path for ACME CA servers, diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 3dfaaf26f..517b56351 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,29 +1,23 @@ """ACME AuthHandler.""" -import collections import logging import time +import datetime -import six import zope.component from acme import challenges from acme import messages # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +from acme.magic_typing import Dict, List # pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler from certbot import interfaces - logger = logging.getLogger(__name__) -AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) -"""Stores an authorization resource and its active annotated challenges.""" - - class AuthHandler(object): """ACME Authorization Handler for a client. @@ -47,242 +41,151 @@ class AuthHandler(object): self.account = account self.pref_challs = pref_challs - def handle_authorizations(self, orderr, best_effort=False): - """Retrieve all authorizations for challenges. - - :param acme.messages.OrderResource orderr: must have - authorizations filled in - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) - - :returns: List of authorization resources - :rtype: list - - :raises .AuthorizationError: If unable to retrieve all - authorizations - + def handle_authorizations(self, orderr, best_effort=False, max_retries=30): """ - aauthzrs = [AnnotatedAuthzr(authzr, []) - for authzr in orderr.authorizations] + Retrieve all authorizations, perform all challenges required to validate + these authorizations, then poll and wait for the authorization to be checked. + :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: if True, not all authorizations need to be validated (eg. renew) + :param int max_retries: maximum number of retries to poll authorizations + :returns: list of all validated authorizations + :rtype: List - self._choose_challenges(aauthzrs) - config = zope.component.getUtility(interfaces.IConfig) - notify = zope.component.getUtility(interfaces.IDisplay).notification + :raises .AuthorizationError: If unable to retrieve all authorizations + """ + authzrs = orderr.authorizations[:] + if not authzrs: + raise errors.AuthorizationError('No authorization to handle.') - # While there are still challenges remaining... - while self._has_challenges(aauthzrs): - with error_handler.ExitHandler(self._cleanup_challenges, aauthzrs): - resp = self._solve_challenges(aauthzrs) - logger.info("Waiting for verification...") + # Retrieve challenges that need to be performed to validate authorizations. + achalls = self._choose_challenges(authzrs) + if not achalls: + return authzrs + + # Starting now, challenges will be cleaned at the end no matter what. + with error_handler.ExitHandler(self._cleanup_challenges, achalls): + # To begin, let's ask the authenticator plugin to perform all challenges. + try: + resps = self.auth.perform(achalls) + + # If debug is on, wait for user input before starting the verification process. + logger.info('Waiting for verification...') + config = zope.component.getUtility(interfaces.IConfig) if config.debug_challenges: + notify = zope.component.getUtility(interfaces.IDisplay).notification notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) + except errors.AuthorizationError as error: + logger.critical('Failure in setting up challenges.') + logger.info('Attempting to clean up outstanding challenges...') + raise error + # All challenges should have been processed by the authenticator. + assert len(resps) == len(achalls), 'Some challenges have not been performed.' - # Send all Responses - this modifies achalls - self._respond(aauthzrs, resp, best_effort) + # Inform the ACME CA server that challenges are available for validation. + for achall, resp in zip(achalls, resps): + self.acme.answer_challenge(achall.challb, resp) - # Just make sure all decisions are complete. - self.verify_authzr_complete(aauthzrs) + # Wait for authorizations to be checked. + self._poll_authorizations(authzrs, max_retries, best_effort) - # Only return valid authorizations - ret_val = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + # Keep validated authorizations only. If there is none, no certificate can be issued. + authzrs_validated = [authzr for authzr in authzrs + if authzr.body.status == messages.STATUS_VALID] + if not authzrs_validated: + raise errors.AuthorizationError('All challenges have failed.') - if not ret_val: - raise errors.AuthorizationError( - "Challenges failed for all domains") + return authzrs_validated - return ret_val + def _poll_authorizations(self, authzrs, max_retries, best_effort): + """ + Poll the ACME CA server, to wait for confirmation that authorizations have their challenges + all verified. The poll may occur several times, until all authorizations are checked + (valid or invalid), or after a maximum of retries. + """ + authzrs_to_check = {index: (authzr, None) + for index, authzr in enumerate(authzrs)} + authzrs_failed_to_report = [] + # Give an initial second to the ACME CA server to check the authorizations + sleep_seconds = 1 + for _ in range(max_retries): + # Wait for appropriate time (from Retry-After, initial wait, or no wait) + if sleep_seconds > 0: + time.sleep(sleep_seconds) + # Poll all updated authorizations. + authzrs_to_check = {index: self.acme.poll(authzr) for index, (authzr, _) + in authzrs_to_check.items()} + # Update the original list of authzr with the updated authzrs from server. + for index, (authzr, _) in authzrs_to_check.items(): + authzrs[index] = authzr - def _choose_challenges(self, aauthzrs): + # Gather failed authorizations + authzrs_failed = [authzr for authzr, _ in authzrs_to_check.values() + if authzr.body.status == messages.STATUS_INVALID] + for authzr_failed in authzrs_failed: + logger.warning('Challenge failed for domain %s', + authzr_failed.body.identifier.value) + # Accumulating all failed authzrs to build a consolidated report + # on them at the end of the polling. + authzrs_failed_to_report.extend(authzrs_failed) + + # Extract out the authorization already checked for next poll iteration. + # Poll may stop here because there is no pending authorizations anymore. + authzrs_to_check = {index: (authzr, resp) for index, (authzr, resp) + in authzrs_to_check.items() + if authzr.body.status == messages.STATUS_PENDING} + if not authzrs_to_check: + # Polling process is finished, we can leave the loop + break + + # Be merciful with the ACME server CA, check the Retry-After header returned, + # and wait this time before polling again in next loop iteration. + # From all the pending authorizations, we take the greatest Retry-After value + # to avoid polling an authorization before its relevant Retry-After value. + retry_after = max(self.acme.retry_after(resp, 3) + for _, resp in authzrs_to_check.values()) + sleep_seconds = (retry_after - datetime.datetime.now()).total_seconds() + + # In case of failed authzrs, create a report to the user. + if authzrs_failed_to_report: + _report_failed_authzrs(authzrs_failed_to_report, self.account.key) + if not best_effort: + # Without best effort, having failed authzrs is critical and fail the process. + raise errors.AuthorizationError('Some challenges have failed.') + + if authzrs_to_check: + # Here authzrs_to_check is still not empty, meaning we exceeded the max polling attempt. + raise errors.AuthorizationError('All authorizations were not finalized by the CA.') + + def _choose_challenges(self, authzrs): """ Retrieve necessary and pending challenges to satisfy server. NB: Necessary and already validated challenges are not retrieved, as they can be reused for a certificate issuance. """ - pending_authzrs = [aauthzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status != messages.STATUS_VALID] + pending_authzrs = [authzr for authzr in authzrs + if authzr.body.status != messages.STATUS_VALID] + achalls = [] # type: List[achallenges.AnnotatedChallenge] if pending_authzrs: logger.info("Performing the following challenges:") - for aauthzr in pending_authzrs: - aauthzr_challenges = aauthzr.authzr.body.challenges + for authzr in pending_authzrs: + authzr_challenges = authzr.body.challenges if self.acme.acme_version == 1: - combinations = aauthzr.authzr.body.combinations + combinations = authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(aauthzr_challenges))) + combinations = tuple((i,) for i in range(len(authzr_challenges))) path = gen_challenge_path( - aauthzr_challenges, - self._get_chall_pref(aauthzr.authzr.body.identifier.value), + authzr_challenges, + self._get_chall_pref(authzr.body.identifier.value), combinations) - aauthzr_achalls = self._challenge_factory( - aauthzr.authzr, path) - aauthzr.achalls.extend(aauthzr_achalls) + achalls.extend(self._challenge_factory(authzr, path)) - for aauthzr in aauthzrs: - for achall in aauthzr.achalls: - if isinstance(achall.chall, challenges.TLSSNI01): - logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - return + if any(isinstance(achall.chall, challenges.TLSSNI01) for achall in achalls): + logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - def _has_challenges(self, aauthzrs): - """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in aauthzrs) - - def _solve_challenges(self, aauthzrs): - """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[challenges.ChallengeResponse] - all_achalls = self._get_all_achalls(aauthzrs) - try: - if all_achalls: - resp = self.auth.perform(all_achalls) - except errors.AuthorizationError: - logger.critical("Failure in setting up challenges.") - logger.info("Attempting to clean up outstanding challenges...") - raise - - assert len(resp) == len(all_achalls) - - return resp - - def _get_all_achalls(self, aauthzrs): - """Return all active challenges.""" - all_achalls = [] # type: Collection[challenges.ChallengeResponse] - for aauthzr in aauthzrs: - all_achalls.extend(aauthzr.achalls) - return all_achalls - - def _respond(self, aauthzrs, resp, best_effort): - """Send/Receive confirmation of all challenges. - - .. note:: This method also cleans up the auth_handler state. - - """ - # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() \ - # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] - self._send_responses(aauthzrs, resp, chall_update) - - # Check for updated status... - self._poll_challenges(aauthzrs, chall_update, best_effort) - - def _send_responses(self, aauthzrs, resps, chall_update): - """Send responses and make sure errors are handled. - - :param aauthzrs: authorizations and the selected annotated challenges - to try and perform - :type aauthzrs: `list` of `AnnotatedAuthzr` - :param resps: challenge responses from the authenticator where - each response at index i corresponds to the annotated - challenge at index i in the list returned by - :func:`_get_all_achalls` - :type resps: `collections.abc.Iterable` of - :class:`~acme.challenges.ChallengeResponse` or `False` or - `None` - :param dict chall_update: parameter that is updated to hold - aauthzr index to list of outstanding solved annotated challenges - - """ - active_achalls = [] - resps_iter = iter(resps) - for i, aauthzr in enumerate(aauthzrs): - for achall in aauthzr.achalls: - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) - - resp = next(resps_iter) - # Don't send challenges for None and False authenticator responses - if resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - chall_update.setdefault(i, []).append(achall) - - return active_achalls - - def _poll_challenges(self, aauthzrs, chall_update, - best_effort, min_sleep=3, max_rounds=30): - """Wait for all challenge results to be determined.""" - indices_to_check = set(chall_update.keys()) - comp_indices = set() - rounds = 0 - - while indices_to_check and rounds < max_rounds: - # TODO: Use retry-after... - time.sleep(min_sleep) - all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] - for index in indices_to_check: - comp_achalls, failed_achalls = self._handle_check( - aauthzrs, index, chall_update[index]) - - if len(comp_achalls) == len(chall_update[index]): - comp_indices.add(index) - elif not failed_achalls: - for achall, _ in comp_achalls: - chall_update[index].remove(achall) - # We failed some challenges... damage control - else: - if best_effort: - comp_indices.add(index) - logger.warning( - "Challenge failed for domain %s", - aauthzrs[index].authzr.body.identifier.value) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) - - if all_failed_achalls: - _report_failed_challs(all_failed_achalls) - raise errors.FailedChallenges(all_failed_achalls) - - indices_to_check -= comp_indices - comp_indices.clear() - rounds += 1 - - def _handle_check(self, aauthzrs, index, achalls): - """Returns tuple of ('completed', 'failed').""" - completed = [] - failed = [] - - original_aauthzr = aauthzrs[index] - updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) - if updated_authzr.body.status == messages.STATUS_VALID: - return achalls, [] - - # Note: if the whole authorization is invalid, the individual failed - # challenges will be determined here... - for achall in achalls: - updated_achall = achall.update(challb=self._find_updated_challb( - updated_authzr, achall)) - - # This does nothing for challenges that have yet to be decided yet. - if updated_achall.status == messages.STATUS_VALID: - completed.append((achall, updated_achall)) - elif updated_achall.status == messages.STATUS_INVALID: - failed.append((achall, updated_achall)) - - return completed, failed - - def _find_updated_challb(self, authzr, achall): # pylint: disable=no-self-use - """Find updated challenge body within Authorization Resource. - - .. warning:: This assumes only one instance of type of challenge in - each challenge resource. - - :param .AuthorizationResource authzr: Authorization Resource - :param .AnnotatedChallenge achall: Annotated challenge for which - to get status - - """ - for authzr_challb in authzr.body.challenges: - if type(authzr_challb.chall) is type(achall.challb.chall): # noqa - return authzr_challb - raise errors.AuthorizationError( - "Target challenge not found in authorization resource") + return achalls def _get_chall_pref(self, domain): """Return list of challenge preferences. @@ -306,43 +209,15 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, aauthzrs, achalls=None): + def _cleanup_challenges(self, achalls): """Cleanup challenges. - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :param achalls: annotated challenges to cleanup :type achalls: `list` of :class:`certbot.achallenges.AnnotatedChallenge` """ logger.info("Cleaning up challenges") - if achalls is None: - achalls = self._get_all_achalls(aauthzrs) - if achalls: - self.auth.cleanup(achalls) - for achall in achalls: - for aauthzr in aauthzrs: - if achall in aauthzr.achalls: - aauthzr.achalls.remove(achall) - break - - def verify_authzr_complete(self, aauthzrs): - """Verifies that all authorizations have been decided. - - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` - - :returns: Whether all authzr are complete - :rtype: bool - - """ - for aauthzr in aauthzrs: - authzr = aauthzr.authzr - if (authzr.body.status != messages.STATUS_VALID and - authzr.body.status != messages.STATUS_INVALID): - raise errors.AuthorizationError("Incomplete authorizations") + self.auth.cleanup(achalls) def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges @@ -530,22 +405,19 @@ _ERROR_HELP = { } -def _report_failed_challs(failed_achalls): - """Notifies the user about failed challenges. +def _report_failed_authzrs(failed_authzrs, account_key): + """Notifies the user about failed authorizations.""" + problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) + for authzr in failed_authzrs for challb in authzr.body.challenges + if challb.error] - :param set failed_achalls: A set of failed - :class:`certbot.achallenges.AnnotatedChallenge`. - - """ - problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: - if achall.error: - problems[achall.error.typ].append(achall) + problems.setdefault(achall.error.typ, []).append(achall) + reporter = zope.component.getUtility(interfaces.IReporter) - for achalls in six.itervalues(problems): - reporter.add_message( - _generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) + for achalls in problems.values(): + reporter.add_message(_generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) def _generate_failed_chall_msg(failed_achalls): diff --git a/certbot/interfaces.py b/certbot/interfaces.py index bb9a91b0f..bd91d2272 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -159,21 +159,14 @@ class IAuthenticator(IPlugin): :func:`get_chall_pref` only. :returns: `collections.Iterable` of ACME - :class:`~acme.challenges.ChallengeResponse` instances - or if the :class:`~acme.challenges.Challenge` cannot - be fulfilled then: - - ``None`` - Authenticator can perform challenge, but not at this time. - ``False`` - Authenticator will never be able to perform (error). - + :class:`~acme.challenges.ChallengeResponse` instances corresponding to each provided + :class:`~acme.challenges.Challenge`. :rtype: :class:`collections.Iterable` of :class:`acme.challenges.ChallengeResponse`, where responses are required to be returned in the same order as corresponding input challenges - :raises .PluginError: If challenges cannot be performed + :raises .PluginError: If some or all challenges cannot be performed """ diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index ee1af4978..c14129d87 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -351,7 +351,7 @@ class ChallengePerformer(object): def perform(self): """Perform all added challenges. - :returns: challenge respones + :returns: challenge responses :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index fe0ece12e..353c34da2 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -4,13 +4,11 @@ import logging import unittest import mock -import six import zope.component from acme import challenges from acme import client as acme_client from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -82,6 +80,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) self.mock_net.acme_version = 1 + self.mock_net.retry_after.side_effect = acme_client.Client.retry_after self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -95,23 +94,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=1, wait_value=30) + with mock.patch('certbot.auth_handler.time') as mock_time: authzr = self.handler.handle_authorizations(mock_order) - self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 2) # Because there is one retry + self.assertEqual(mock_time.sleep.call_count, 2) + # Retry-After header is 30 seconds, but at the time sleep is invoked, several + # instructions are executed, and next pool is in less than 30 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + # However, assert that we did not took the default value of 3 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) - self.assertEqual(self.mock_auth.cleanup.call_count, 1) - # Test if list first element is TLSSNI01, use typ because it is an achall - self.assertEqual( - self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + # Test if list first element is TLSSNI01, use typ because it is an achall + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - self.assertEqual(len(authzr), 1) + self.assertEqual(len(authzr), 1) def test_name1_tls_sni_01_1_acme_1(self): self._test_name1_tls_sni_01_1_common(combos=True) @@ -120,9 +122,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name1_tls_sni_01_1_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): - mock_poll.side_effect = self._validate_all + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self): + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -132,10 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 3) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is TLSSNI01, use typ because it is an achall @@ -145,10 +143,9 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p # Length of authorizations list self.assertEqual(len(authzr), 1) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self): self.mock_net.acme_version = 2 - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -158,10 +155,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] @@ -175,27 +169,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.request_domain_challenges.side_effect = functools.partial( gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - authzr = self.handler.handle_authorizations(mock_order) + + self.mock_net.poll.side_effect = _gen_mock_on_poll() + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) # Check poll call - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue(0 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[0]), 1) - self.assertTrue(1 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[1]), 1) - self.assertTrue(2 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[2]), 1) + self.assertEqual(self.mock_net.poll.call_count, 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -208,14 +193,13 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name3_tls_sni_01_3_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_debug_challenges(self, mock_poll): + def test_debug_challenges(self): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.handler.handle_authorizations(mock_order) @@ -231,6 +215,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_max_retries_exceeded(self): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + + # We will return STATUS_PENDING twice before returning STATUS_VALID. + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=2) + + with self.assertRaises(errors.AuthorizationError) as error: + # We retry only once, so retries will be exhausted before STATUS_VALID is returned. + self.handler.handle_authorizations(mock_order, False, 1) + self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -244,9 +240,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - self.handler.handle_authorizations(mock_order) + self.mock_net.poll.side_effect = _gen_mock_on_poll() + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( @@ -290,11 +285,11 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._respond") - def test_respond_error(self, mock_respond): + def test_answer_error(self): + self.mock_net.answer_challenge.side_effect = errors.AuthorizationError + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_respond.side_effect = errors.AuthorizationError self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -302,20 +297,52 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - @mock.patch("certbot.auth_handler.AuthHandler.verify_authzr_complete") - def test_incomplete_authzr_error(self, mock_verify, mock_poll): + def test_incomplete_authzr_error(self): authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_verify.side_effect = errors.AuthorizationError - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) - self.assertRaises( - errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, False) + self.assertTrue('Some challenges have failed.' in str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_best_effort(self): + def _conditional_mock_on_poll(authzr): + """This mock will invalidate one authzr, and invalidate the other one""" + valid_mock = _gen_mock_on_poll(messages.STATUS_VALID) + invalid_mock = _gen_mock_on_poll(messages.STATUS_INVALID) + + if authzr.body.identifier.value == 'will-be-invalid': + return invalid_mock(authzr) + return valid_mock(authzr) + + # Two authzrs. Only one will be valid. + authzrs = [gen_dom_authzr(domain="will-be-valid", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="will-be-invalid", challs=acme_util.CHALLENGES)] + self.mock_net.poll.side_effect = _conditional_mock_on_poll + + mock_order = mock.MagicMock(authorizations=authzrs) + + with mock.patch('certbot.auth_handler._report_failed_authzrs') as mock_report: + valid_authzr = self.handler.handle_authorizations(mock_order, True) + + # Because best_effort=True, we did not blow up. Instead ... + self.assertEqual(len(valid_authzr), 1) # ... the valid authzr has been processed + self.assertEqual(mock_report.call_count, 1) # ... the invalid authzr has been reported + + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) + + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, True) + + # Despite best_effort=True, process will fail because no authzr is valid. + self.assertTrue('All challenges have failed.' in str(error.exception)) + def test_validated_challenge_not_rerun(self): # With pending challenge, we expect the challenge to be tried, and fail. authzr = acme_util.gen_authzr( @@ -334,138 +361,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p mock_order = mock.MagicMock(authorizations=[authzr]) self.handler.handle_authorizations(mock_order) - def _validate_all(self, aauthzrs, unused_1, unused_2): - for i, aauthzr in enumerate(aauthzrs): - azr = aauthzr.authzr - updated_azr = acme_util.gen_authzr( - messages.STATUS_VALID, - azr.body.identifier.value, - [challb.chall for challb in azr.body.challenges], - [messages.STATUS_VALID] * len(azr.body.challenges), - azr.body.combinations) - aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) - @mock.patch("certbot.auth_handler.logger") def test_tls_sni_logs(self, logger): self._test_name1_tls_sni_01_1_common(combos=True) self.assertTrue("deprecated" in logger.warning.call_args[0][0]) -class PollChallengesTest(unittest.TestCase): - # pylint: disable=protected-access - """Test poll challenges.""" +def _gen_mock_on_poll(status=messages.STATUS_VALID, retry=0, wait_value=1): + state = {'count': retry} - def setUp(self): - from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler, AnnotatedAuthzr - - # Account and network are mocked... - self.mock_net = mock.MagicMock() - self.handler = AuthHandler( - None, self.mock_net, mock.Mock(key="mock_key"), []) - - self.doms = ["0", "1", "2"] - self.aauthzrs = [ - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) - ] - - self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] - for i, aauthzr in enumerate(self.aauthzrs): - self.chall_update[i] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) - for challb in aauthzr.authzr.body.challenges] - - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges_failure_best_effort(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) - - @mock.patch("certbot.auth_handler.time") - @test_util.patch_get_utility() - def test_poll_challenges_failure(self, unused_mock_time, unused_mock_zope): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - @mock.patch("certbot.auth_handler.time") - def test_unable_to_find_challenge_status(self, unused_mock_time): - from certbot.auth_handler import challb_to_achall - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[0].append( - challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - def test_verify_authzr_failure(self): - self.assertRaises(errors.AuthorizationError, - self.handler.verify_authzr_complete, self.aauthzrs) - - def _mock_poll_solve_one_valid(self, authzr): - # Pending here because my dummy script won't change the full status. - # Basically it didn't raise an error and it stopped earlier than - # Making all challenges invalid which would make mock_poll_solve_one - # change authzr to invalid - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_VALID) - - def _mock_poll_solve_one_invalid(self, authzr): - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_INVALID) - - def _mock_poll_solve_one_chall(self, authzr, desired_status): - # pylint: disable=no-self-use - """Dummy method that solves one chall at a time to desired_status. - - When all are solved.. it changes authzr.status to desired_status - - """ - new_challbs = authzr.body.challenges - for challb in authzr.body.challenges: - if challb.status != desired_status: - new_challbs = tuple( - challb_temp if challb_temp != challb - else acme_util.chall_to_challb(challb.chall, desired_status) - for challb_temp in authzr.body.challenges - ) - break - - if all(test_challb.status == desired_status - for test_challb in new_challbs): - status_ = desired_status - else: - status_ = authzr.body.status - - new_authzr = messages.AuthorizationResource( - uri=authzr.uri, - body=messages.Authorization( - identifier=authzr.body.identifier, - challenges=new_challbs, - combinations=authzr.body.combinations, - status=status_, - ), - ) - return (new_authzr, "response") + def _mock(authzr): + state['count'] = state['count'] - 1 + effective_status = status if state['count'] < 0 else messages.STATUS_PENDING + updated_azr = acme_util.gen_authzr( + effective_status, + authzr.body.identifier.value, + [challb.chall for challb in authzr.body.challenges], + [effective_status] * len(authzr.body.challenges), + authzr.body.combinations) + return updated_azr, mock.MagicMock(headers={'Retry-After': str(wait_value)}) + return _mock class ChallbToAchallTest(unittest.TestCase): @@ -527,8 +442,8 @@ class GenChallengePathTest(unittest.TestCase): errors.AuthorizationError, self._call, challbs, prefs, None) -class ReportFailedChallsTest(unittest.TestCase): - """Tests for certbot.auth_handler._report_failed_challs.""" +class ReportFailedAuthzrsTest(unittest.TestCase): + """Tests for certbot.auth_handler._report_failed_authzrs.""" # pylint: disable=protected-access def setUp(self): @@ -542,31 +457,27 @@ class ReportFailedChallsTest(unittest.TestCase): # Prevent future regressions if the error type changes self.assertTrue(kwargs["error"].description is not None) - self.http01 = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + http_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args kwargs["chall"] = acme_util.TLSSNI01 - self.tls_sni_same = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + tls_sni_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr1 = mock.MagicMock() + self.authzr1.body.identifier.value = 'example.com' + self.authzr1.body.challenges = [http_01, tls_sni_01] kwargs["error"] = messages.Error(typ="dnssec", detail="detail") - self.tls_sni_diff = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="foo.bar", - account_key="key") + tls_sni_01_diff = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr2 = mock.MagicMock() + self.authzr2.body.identifier.value = 'foo.bar' + self.authzr2.body.challenges = [tls_sni_01_diff] @test_util.patch_get_utility() def test_same_error_and_domain(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) + auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @@ -575,7 +486,7 @@ class ReportFailedChallsTest(unittest.TestCase): def test_different_errors_and_domains(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_diff]) + auth_handler._report_failed_authzrs([self.authzr1, self.authzr2], 'key') self.assertTrue(mock_zope().add_message.call_count == 2) diff --git a/tox.cover.py b/tox.cover.py index e323ba255..d0f97626a 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -12,7 +12,7 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] COVER_THRESHOLDS = { - 'certbot': {'linux': 98, 'windows': 94}, + 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, 'certbot_apache': {'linux': 100, 'windows': 100}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, From bf1f83f47b49f746f1cd54076749a08e7b04250d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 02:16:22 +0100 Subject: [PATCH 595/639] Revert "Disable build for commits pushed on master (#6804)" (#6807) By removing all the builds on push to master, done in #6804, we also removing the coveralls reports that are necessary to calculate the effect of a PR on code coverage, that is part of the quality gate process. This PR is reverting #6804, and another implementation preserving the coveralls reports will be done soon. This reverts commit a468a3b255e326dbfa2b7d629eb20fa275631254. --- .travis.yml | 5 ----- appveyor.yml | 8 -------- 2 files changed, 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 108f853aa..4adab1e4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,11 +18,6 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ -# Since master can receive only commits from PR that have already been tested, we avoid with the -# the following condition to launch again a pipeline when the merge commit is pushed to master. -# However master still needs to be set in branches section to allow PR for master to be built. -if: NOT (type = push AND branch = master) - matrix: include: # These environments are always executed diff --git a/appveyor.yml b/appveyor.yml index 5dcc80ba9..2b6b82747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,14 +11,6 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ -init: - # Since master can receive only commits from PR that have already been tested, we avoid with the - # the following condition to launch again a pipeline when the merge commit is pushed to master. - # However master still needs to be set in branches section to allow PR for master to be built. - - ps: | - if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master') - { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } - install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From 6ff101dcbbe66943487633391a4f03f27bcc6aaa Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 03:07:07 +0100 Subject: [PATCH 596/639] Cover case of OpenSUSE Leap 15+ in certbot-auto (#6794) Fixes #6228. Since OpenSUSE Leap 15, python-virtualenv became a source package, breaking certbot-auto bootstrap on this version. Then python2-virtualenv must be used to create Python 2.x virtual environments. This PR makes certbot-auto compatible to prior and after Leap 15, by testing the existence of python-virtualenv on current OpenSUSE system, and then use appropriate packages. * Cover case of OpenSUSE Leap 15+ in certbot-auto * Revert increment on bootstrap for OpenSUSE * Fix configuration for Leap15+ * Add comment about explicit installation of python2-setuptools * Update letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh Co-Authored-By: adferrand * Update letsencrypt-auto --- letsencrypt-auto-source/letsencrypt-auto | 12 +++++++++++- .../pieces/bootstrappers/suse_common.sh | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c390f28ec..ce3b35e86 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh index c531cbe99..ac66119c3 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh @@ -14,10 +14,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ From d8a3fa3904ecddd00be871847c773c9370780449 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Mar 2019 15:52:38 +0200 Subject: [PATCH 597/639] Address review comments --- tools/merge_requirements.py | 25 ++++++++++++++----------- tools/pip_install.py | 2 +- tools/strip_hashes.py | 33 +++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 521d28b8f..0d41d12c4 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -55,21 +55,24 @@ def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier - files. + files. Files are read from file paths passed from the command line arguments. - :param tuple paths: paths to requirements files + If no command line arguments are defined, data is read from stdin instead. + + :param tuple paths: paths to requirements files provided on command line """ data = {} - for path in paths: - data.update(process_entries(read_file(path))) - - # Need to check if interactive to avoid blocking if nothing is piped - if not sys.stdin.isatty(): - stdin_data = [] - for line in sys.stdin: - stdin_data.append(line) - data.update(process_entries(stdin_data)) + if paths: + for path in paths: + data.update(process_entries(read_file(path))) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) return output_requirements(data) diff --git a/tools/pip_install.py b/tools/pip_install.py index 8f0437d9c..15dc2f0c0 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -48,7 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - data = os.linesep.join(strip_hashes.process_entries(data)) + data = "\n".join(strip_hashes.process_entries(data)) fd.write(data) diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py index 7e7809458..988e72eb8 100755 --- a/tools/strip_hashes.py +++ b/tools/strip_hashes.py @@ -2,7 +2,6 @@ """Removes hash information from requirement files passed to it as file path arguments or simply piped to stdin.""" -import os import re import sys @@ -24,22 +23,28 @@ def process_entries(entries): return out_lines def main(*paths): - """Reads dependency definitions from a (list of) file(s) or stdin and - removes hashes from returned entries""" + """ + Reads dependency definitions from a (list of) file(s) provided on the + command line. If no command line arguments are present, data is read from + stdin instead. + + Hashes are removed from returned entries. + """ deps = [] - for path in paths: - with open(path) as file_h: - deps += process_entries(file_h.readlines()) + if paths: + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) - # Need to check if interactive to avoid blocking if nothing is piped - if not sys.stdin.isatty(): - stdin_data = [] - for line in sys.stdin: - stdin_data.append(line) - deps += process_entries(stdin_data) - - return os.linesep.join(deps) + return "\n".join(deps) if __name__ == '__main__': print(main(*sys.argv[1:])) # pylint: disable=star-args From 6a0f3248a8f7a55b2de33283e498458e3a7b1a81 Mon Sep 17 00:00:00 2001 From: Yohann Leon Date: Tue, 5 Mar 2019 22:27:07 +0100 Subject: [PATCH 598/639] Replace deprecated Gandi plugin link --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index e17e56b64..de17742a1 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -277,7 +277,7 @@ Plugin Auth Inst Notes plesk_ Y Y Integration with the Plesk web hosting tool haproxy_ Y Y Integration with the HAProxy load balancer s3front_ Y Y Integration with Amazon CloudFront distribution of S3 buckets -gandi_ Y Y Integration with Gandi's hosting products and API +gandi_ Y Y Integration with Gandi LiveDNS API varnish_ Y N Obtain certificates via a Varnish server external_ Y N A plugin for convenient scripting (See also ticket 2782_) icecast_ N Y Deploy certificates to Icecast 2 streaming media servers @@ -290,7 +290,7 @@ heroku_ Y Y Integration with Heroku SSL .. _plesk: https://github.com/plesk/letsencrypt-plesk .. _haproxy: https://github.com/greenhost/certbot-haproxy .. _s3front: https://github.com/dlapiduz/letsencrypt-s3front -.. _gandi: https://github.com/Gandi/letsencrypt-gandi +.. _gandi: https://github.com/obynio/certbot-plugin-gandi .. _icecast: https://github.com/e00E/lets-encrypt-icecast .. _varnish: http://git.sesse.net/?p=letsencrypt-varnish-plugin .. _2782: https://github.com/certbot/certbot/issues/2782 From 198c447e77c785842a68c2fdc9326f156f29d915 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Mar 2019 14:01:08 -0800 Subject: [PATCH 599/639] Move the OCSP change to the right section. (#6818) Looks like this got added to our changelog for the released 0.31.0 instead of the upcoming release. We want this change for the release tomorrow. --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 764fc0946..ce0dddd16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. ### Changed @@ -49,9 +51,6 @@ More details about these changes can be found on our GitHub repo. * Avoid reprocessing challenges that are already validated when a certificate is issued. -* If possible, Certbot uses built-in support for OCSP from recent cryptography - versions instead of the OpenSSL binary: as a consequence Certbot does not need - the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. From 670e9d89b70258a1a8fb963f4e208de0e22aab6d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 6 Mar 2019 03:30:33 +0100 Subject: [PATCH 600/639] Fix faulty test (#6816) --- .../tests/apache-conf-files/passing/finalize-1243.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf index 0918e5669..dbfae3765 100644 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf @@ -1,7 +1,7 @@ #LoadModule ssl_module modules/mod_ssl.so -Listen 443 - +Listen 4443 + # The ServerName directive sets the request scheme, hostname and port that # the server uses to identify itself. This is used when creating # redirection URLs. In the context of virtual hosts, the ServerName From be6df7de044988613ff14ac042cda3a7e863b076 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:16:13 -0800 Subject: [PATCH 601/639] Remove Fixed section from changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce0dddd16..cc007b46f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,10 +30,6 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). field included if a `malformed` error is received. This fallback will be removed in version 0.34.0. -### Fixed - -* - Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: From a276523c096bfa46c5948b24e37cccb0d6a2bd25 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:18:08 -0800 Subject: [PATCH 602/639] Update changelog for 0.32.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc007b46f..fdb3b5f13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.32.0 - master +## 0.32.0 - 2019-03-06 ### Added From 0492855166f104bba8a7dd0395b925d82c8cc759 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:47:27 -0800 Subject: [PATCH 603/639] Release 0.32.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 100 ++++++++++-------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 10 +- letsencrypt-auto | 100 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 16 +-- letsencrypt-auto-source/letsencrypt-auto | 26 ++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++--- 26 files changed, 168 insertions(+), 146 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 6cb5c3f92..00e9a0971 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 5d15611dd..1b3173ada 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 03c2994e3..0c82a7437 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e1c82b063..c11e32cd1 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d6a16f468..8c697d0a5 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index f0f2446e1..ed3ac8c78 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c61f05339..f9d2c430f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 7d39a10d0..2039d27c7 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a5f234b5..6df71672f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 67ae1d4eb..3aee64ba4 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 429702d75..e7ff6ac78 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 6797501b9..60f6e0d36 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 07d9decdf..4406244fd 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 314b0a16c..bdd21f45b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index a878eac47..046c05382 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 39f35e953..580f4ac2a 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e915c6b86..24e097b9d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 429f960f7..8c6e28c90 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 8984704a6..b6cd0ea97 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 767448a12..a442d1e84 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.32.0.dev0' +__version__ = '0.32.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index e95b2fcd5..ed7794c3b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.31.0 + "". (default: CertbotACMEClient/0.32.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -477,7 +477,9 @@ plugins: using Sakura Cloud for DNS). (default: False) apache: - Apache Web Server plugin + Apache Web Server plugin (Please note that the default values of the + Apache plugin options change depending on the operating system Certbot is + run on.) --apache-enmod APACHE_ENMOD Path to the Apache 'a2enmod' binary (default: None) @@ -496,7 +498,7 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you (Only Ubuntu/Debian currently) (default: False) @@ -505,7 +507,7 @@ apache: Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apachectl) + apache2ctl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index 03c2994e3..0c82a7437 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index d95f9abcb..eeb8d7d12 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxcop8ACgkQTRfJlc2X -dfIbZwf/faKu7IjLi0qFQ+kw8zaAnV47JDgfWqbR5GSdwWPqld+QyHlcRfPgwYma -fKj9+g/FvPNPSfjHRRCoFrYvpZ4lZ+f4HPN9+OjydfM77rdDhVDwzs8dbKIk02yU -0IEJhXj5Q9hF3TSDZcyXAJdBU1lz51ohtVIXelMBPmzhYPCZF47iE9/k9pApQi86 -RTji7hxPcF/n7mzXrbyTvk+kDxSdDlE0Eg9syK7XaFDBTa2lqgG8wTnMPVqhc/hm -WM/uwkzbYarjy05ffV1kM683nP0rECnHlYT38pYcT2puw2kn/QthwR5j/jB/DWSc -94Kw7BeMH651V8EaNwYIiouylnVH3A== -=U+Qh +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlyAMZ0ACgkQTRfJlc2X +dfLItQf/SNv+at1Pw5oiEbWleNPpmz9srlkf9AHU92Hh3p7+OljcaWQindtCYtlO +UvWV/CjGzObmJvO+Pgy2epNhD8cTjPamI46l5UG2nwvy8V+JemS937Ae6paivt8T +/RaFKfyNDfxBjQhHS1ypVuRrFgAQ5CG0iGuJSMgwLpcKCZyKAim3+Vb57+Esq+zG +Cp7GmJk9h1z5FbNukbaFHBlJQIefJoQclh1yUw11pLab0uxIOdc9WiEWLLAVE512 +SMQM2sNv49uh7mRnxW+6WU6dor6JI9Ff1L4D5hfglzBJRM4qLU/hGv54oVycWq4i +eFjtqDfo5XMwnbnUnVkEB73pY6lzDg== +=YaE3 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ce3b35e86..0c82a7437 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.32.0.dev0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1190,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 3ecca151022170a559f05135a0b3ec86bc998e96..17eea3eba1b349826997251971e17ac8efe7ea52 100644 GIT binary patch literal 256 zcmV+b0ssCf-^}x|Ewd+i;#v>piH?o-`-qDCn+vf z$3RtG;n%M3^4xW@~i8c0kFB$Jt-(u9-Ju}xO!+YXSFfC2&HlmJy zAhH1>F_F9cSN3V5Z%-0cd~Hf{3?0JP`3%W9Iyz30B3!F$a}q6xTC|sDM>me@1oCdpGMSAJlPA0}OyX15pk1>`;NPfsRIoA7989)8hXTrsI)?K2n+{4(K0$wW- z?Xt|aQ|bMz{LJj09XOy^>A@XGdSH*2KX{+1-dg&>BqH~3rK zY2Wn@jjzL2vZ@UiY;SJVRrZ9^Fl|&kDuWGZ&Hbdn^_ Date: Wed, 6 Mar 2019 12:47:28 -0800 Subject: [PATCH 604/639] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb3b5f13..c854c74bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.33.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.32.0 - 2019-03-06 ### Added From 8dda6cc68f7f2a0b2b05d223e27df487fb5c9b3c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:47:29 -0800 Subject: [PATCH 605/639] Bump version to 0.33.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 00e9a0971..cd4ea3ef5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 1b3173ada..dfedc3f0d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c11e32cd1..858cef831 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 8c697d0a5..84414e0c0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index ed3ac8c78..4b2fe15be 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f9d2c430f..1de51f7ca 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2039d27c7..113e55e12 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 6df71672f..6e882a4f7 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 3aee64ba4..bdbd198d1 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e7ff6ac78..c2556797d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 60f6e0d36..c494d6d4e 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4406244fd..4c57c5709 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index bdd21f45b..9c9f233cc 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 046c05382..cf6fee5d7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 580f4ac2a..dacf41101 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 24e097b9d..321ad3b5d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8c6e28c90..46d92a6ff 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index db6b261f0..2108298ae 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.29.0 --e .[dev] +certbot[dev]==0.32.0 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b6cd0ea97..7494bf6bf 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index a442d1e84..38e25f3a4 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.32.0' +__version__ = '0.33.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0c82a7437..af4b181cd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.32.0" +LE_AUTO_VERSION="0.33.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From f378536ffab6213d1e34a8811e9d8a0e6ab9674b Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 6 Mar 2019 23:49:43 +0100 Subject: [PATCH 606/639] Do not run the full CI pipeline on master (#6811) * Configure appveyor * Renaming in travis yml * Update .travis.yml Co-Authored-By: adferrand * Update .travis.yml Co-Authored-By: adferrand --- .travis.yml | 68 ++++++++++++++++++++++++++++++++++------------------ appveyor.yml | 8 +++++++ 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4adab1e4e..cabd3cf73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,141 +18,163 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ +# Jobs for the main test suite are always executed (including on PRs) except for pushes on master. +not-on-master: ¬-on-master + if: NOT (type = push AND branch = master) + +# Jobs for the extended test suite are executed for cron jobs and pushes on non-master branches. +extended-test-suite: &extended-test-suite + if: type = cron OR (type = push AND branch != master) + matrix: include: - # These environments are always executed + # Main test suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master + + # This job is always executed, including on master - python: "2.7" env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - sudo: required env: TOXENV=nginx_compat services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=lint + <<: *not-on-master - python: "3.4" env: TOXENV=mypy + <<: *not-on-master - python: "3.5" env: TOXENV=mypy + <<: *not-on-master - python: "2.7" env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest' sudo: required services: docker + <<: *not-on-master - python: "3.4" env: TOXENV=py34 sudo: required services: docker + <<: *not-on-master - python: "3.7" dist: xenial env: TOXENV=py37 sudo: required services: docker + <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker before_install: addons: + <<: *not-on-master - sudo: required env: TOXENV=le_auto_trusty services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=apacheconftest-with-pebble sudo: required services: docker + <<: *not-on-master - python: "2.7" env: TOXENV=nginxroundtrip + <<: *not-on-master - # These environments are executed on cron events and commits to tested - # branches other than master. Which branches are tested is controlled by - # the "branches" section earlier in this file. + # Extended test suite on cron jobs and pushes to tested branches other than master - python: "3.7" dist: xenial env: TOXENV=py37 CERTBOT_NO_PIN=1 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_xenial services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_jessie services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_centos6 services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=docker_dev services: docker @@ -160,7 +182,7 @@ matrix: apt: packages: # don't install nginx and apache - libaugeas0 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py27 os: osx @@ -169,7 +191,7 @@ matrix: packages: - augeas - python2 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py3 os: osx @@ -178,7 +200,7 @@ matrix: packages: - augeas - python3 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite # container-based infrastructure sudo: false diff --git a/appveyor.yml b/appveyor.yml index 2b6b82747..12d882973 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,14 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +init: + # Since master can receive only commits from PR that have already been tested, following + # condition avoid to launch all jobs except the coverage one for commits pushed to master. + - ps: | + if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' ` + -And -Not ($Env:TOXENV -Like '*-cover')) + { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } + install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From 34393f9bf4a0ca7045a4cfd1e4eafa5842b3a832 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 7 Mar 2019 19:05:20 +0100 Subject: [PATCH 607/639] Correct certbot-auto for Fedora 29+ (#6812) Fixes #6698 Fedora maintainers engaged a deprecation path for Python 2.x with Fedora 29. As a first step, python2-virtualenv does not install the virtualenv binary anymore, in favor of python3-virtualenv, and so the installation of Python 3 virtual environments by default. However, certbot-auto installs python2-virtualenv for all recent RPM distributions, and relies of the execution of virtualenv, and this is failing the process. Since the plan in the future is to remove Python 2.x from Fedora, this PR follows this logic to fix certbot-auto: started to Fedora 29, certbot-auto will install and execute certbot on Python 3. This implies to detect that we are on Fedora 29+, install python3-virtualenv that will install also Python 3 dependencies and virtualenv binary, then instruct the process to use Python 3. This is in fact similar to EOL distributions shipping with Python 2.6, and for which Python 3.4 from EPEL is installed and used. Older versions of Fedora continue to use Python 2.x, and their process is untouched. Four scenarios are covered here: fresh Fedora 28: old process is used, nothing changes fresh Fedora 29: new process is used, Python 3 is installed, certbot runs on it update Fedora 29 from 28, already installed certbot-auto without rebootstrapping required: existing venv continue to be used, certbot runs on it update Fedora 29 from 28, already installed certbot-auto with rebootstrapping required: new process is used, installing python3-virtualenv, python3-devel and python3-rpm-macros, Python 3 is installed, certbot runs on it * Add a step to handle python3 on fedora29 * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update rpm_python3.sh * Rebuild certbot-auto * Empty commit to relaunch CI pipeline * Add changelog * Update CHANGELOG.md Co-Authored-By: adferrand * Update CHANGELOG.md --- CHANGELOG.md | 3 ++- letsencrypt-auto-source/letsencrypt-auto | 14 ++++++++++++-- letsencrypt-auto-source/letsencrypt-auto.template | 5 ++++- .../pieces/bootstrappers/rpm_python3.sh | 9 ++++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c854c74bf..3b49d5275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Fedora 29+ is now supported by certbot-auto. Since Python 2.x is on a deprecation + path in Fedora, certbot-auto will install and use Python 3.x on Fedora 29+. ### Changed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index af4b181cd..df7fc2dc1 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -488,11 +488,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools @@ -741,7 +748,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2cd3f4336..215fca04e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -323,7 +323,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh index b011a7235..f33b07ca9 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh @@ -5,11 +5,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools From 45229eebdfd08baac26c9e8b78d76490b6f7cbce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Mar 2019 11:57:08 -0800 Subject: [PATCH 608/639] Drop expected Apache coverage to workaround #6813. (#6826) * Drop expected Apache coverage to workaround #6813. * add comment --- tox.cover.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.cover.py b/tox.cover.py index d0f97626a..6f5392b71 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -14,7 +14,10 @@ DEFAULT_PACKAGES = [ COVER_THRESHOLDS = { 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, - 'certbot_apache': {'linux': 100, 'windows': 100}, + # certbot_apache coverage not being at 100% is a workaround for + # https://github.com/certbot/certbot/issues/6813. We should increase + # the minimum coverage back to 100% when this issue is resolved. + 'certbot_apache': {'linux': 99, 'windows': 99}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, From cac4be7046e7c349fd4f03ae8af51b7586de4138 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 11 Mar 2019 23:27:33 +0100 Subject: [PATCH 609/639] Calculate timedelta with thisUpdate/nextUpdate in UTC (#6838) Fixes #6836. OCSP responses contains a thisUpdate and nextUpdate that allow to calculate its validity. Certbot currently uses datetime.now() to get the current time when OCSP check is done through cryptography. But datetime.now() expresses the date in the machine local time, and comparison operators on datetime do not take into account the offset between two datetime objects expressed in difference timezones. As a consequence, a given thisUpdate may be seen as a future date depending on the local time, failing the OCSP check process. The error is not critical for certbot, as it will just make some valid OCSP responses giving an EXPIRED status been ignored. This PR fixes this comparison by taking the current time in UTC using datetime.utctime(). --- certbot/ocsp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 6f4c0b0fb..bc63c40f9 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -200,7 +200,7 @@ def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): # for OpenSSL, so we do not do it here. # See OpenSSL implementation as a reference: # https://github.com/openssl/openssl/blob/ef45aa14c5af024fcb8bef1c9007f3d1c115bd85/crypto/ocsp/ocsp_cl.c#L338-L391 - now = datetime.now() + now = datetime.utcnow() # thisUpdate/nextUpdate are expressed in UTC/GMT time zone if not response_ocsp.this_update: raise AssertionError('param thisUpdate is not set.') if response_ocsp.this_update > now + timedelta(minutes=5): From 81d9b5250e1d50b376170186b661c6c4ca132956 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 11 Mar 2019 23:42:32 +0100 Subject: [PATCH 610/639] Clean stderr in case of /etc/os-release does not exist (#6835) --- letsencrypt-auto-source/letsencrypt-auto | 4 ++-- letsencrypt-auto-source/letsencrypt-auto.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index df7fc2dc1..2a44a58dc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -749,8 +749,8 @@ elif [ -f /etc/redhat-release ]; then unset LE_PYTHON DeterminePythonVersion "NOCRASH" # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` - RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 215fca04e..29c54bf2d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -324,8 +324,8 @@ elif [ -f /etc/redhat-release ]; then unset LE_PYTHON DeterminePythonVersion "NOCRASH" # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` - RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" From e20adedb94fdf6774ad3789e6f493905d56f750f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 11 Mar 2019 15:50:27 -0700 Subject: [PATCH 611/639] This is now at the top level of their site --- docs/ciphers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 1b320cdf9..b748dd87a 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -227,7 +227,7 @@ BetterCrypto.org BetterCrypto.org, a collaboration of mostly European IT security experts, has published a draft paper, "Applied Crypto Hardening" -https://bettercrypto.org/static/applied-crypto-hardening.pdf +https://bettercrypto.org/ FF-DHE Internet-Draft ~~~~~~~~~~~~~~~~~~~~~ From a7f2f24426e7d4025e5c32b684324a3fb8a81f01 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Mar 2019 16:14:20 -0700 Subject: [PATCH 612/639] Mention OCSP UTC fix in changelog. (#6845) --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b49d5275..cc34634ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,15 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Certbot uses the Python library cryptography for OCSP when cryptography>=2.5 + is installed. We fixed a bug in Certbot causing it to interpret timestamps in + the OCSP response as being in the local timezone rather than UTC. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* certbot More details about these changes can be found on our GitHub repo. From cf29e89366c7eb57dd48a99a06fe05ceaa9057fa Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 12 Mar 2019 00:16:48 +0100 Subject: [PATCH 613/639] Move coverage computation during certbot integration tests at the end of the script (#6842) Currently coverage invocation during integration tests on certbot core is misplaced, just before the OCSP statuses tests. This PR move back the coverage invocation at the end of the script. --- tests/certbot-boulder-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 9d1ae9ec2..05b144df4 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -528,8 +528,6 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi -coverage report --fail-under 64 --include 'certbot/*' --show-missing - # Test OCSP status ## OCSP 1: Check stale OCSP status @@ -582,3 +580,5 @@ if [ "$REVOKED" != 1 ] ; then echo "Expected le-ocsp-check.wtf to be REVOKED" exit 1 fi + +coverage report --fail-under 64 --include 'certbot/*' --show-missing From acc918eee7fd5c6101a88b7b034a8c05c46483c6 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 13 Mar 2019 23:42:07 +0100 Subject: [PATCH 614/639] Remove tls-sni integration tests (#6852) This PR is a part of the tls-sni-01 removal plan described in #6849. This PR removes the tls-sni-01 challenge tests during the integration tests. The approach I used here is not to remove completely the existing test code, but simply editing it to use a http-01 challenge. Indeed: * the current integration tests are strongly coupled, and would require more modifications that it is worth, because ... * the certbot-ci project, that has already no tls-sni tests, will soon replace completely the current integration tests code. --- certbot-nginx/tests/boulder-integration.sh | 4 +--- tests/boulder-fetch.sh | 6 ------ tests/certbot-boulder-integration.sh | 16 ++++++++-------- tests/integration/_common.sh | 6 +++--- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 1312cd0e6..fbb34a273 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -43,8 +43,6 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -certbot_test_nginx --domains nginx-tls.wtf run --preferred-challenges tls-sni -test_deployment_and_rollback nginx-tls.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 @@ -70,4 +68,4 @@ test_deployment_and_rollback nginx6.wtf # top nginx -c $nginx_root/nginx.conf -s stop -coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing +coverage report --fail-under 72 --include 'certbot-nginx/*' --show-missing diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index a06d37325..f34deb74e 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -12,12 +12,6 @@ fi cd ${BOULDERPATH} -# Since https://github.com/letsencrypt/boulder/commit/92e8e1708a725e9d08a5da2f4a7132320ed2158b, -# Boulder support for tls-sni-01 challenges is disabled. We still need to support it until this -# challenge is officially removed from ACME CA server on production, and also removed from Certbot. -# This sed command reactivate tls-sni-01 challenges inplace temporarily. -sed -i 's/tls-alpn-01/tls-sni-01/g' test/config/ra.json - docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 05b144df4..004423a92 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -223,20 +223,20 @@ common plugins --init --prepare | grep webroot # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & +python ./tests/run_http_server.py $tls_alpn_01_port & python_server_pid=$! - certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ +common --domains le1.wtf --preferred-challenges http-01 auth \ --cert-name $certname \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid CheckDeployHook $certname -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! +# Previous test used to be a tls-sni-01 challenge that is not supported anymore. +# Now it is a http-01 challenge and this makes it a duplicate of the following test. +# But removing it would break many tests here, as they are strongly coupled. +# See https://github.com/certbot/certbot/pull/6852 certname="le2.wtf" common --domains le2.wtf --preferred-challenges http-01 run \ --cert-name $certname \ @@ -256,7 +256,7 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ CheckRenewHook $certname certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ +common -a manual -d dns.le.wtf --preferred-challenges dns run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ @@ -398,7 +398,7 @@ CheckDirHooks 1 # with fail. common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ + --preferred-challenges dns \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 83aa91a9e..76b990ac4 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,10 +3,10 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -tls_sni_01_port=5001 +tls_alpn_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir tls_sni_01_port http_01_port sources +export root config_dir tls_alpn_01_port http_01_port sources certbot_path="$(command -v certbot)" # Flags that are added here will be added to Certbot calls within # certbot_test_no_force_renew. @@ -60,7 +60,7 @@ certbot_test_no_force_renew () { "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ - --tls-sni-01-port $tls_sni_01_port \ + --tls-sni-01-port $tls_alpn_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ $other_flags \ From 8386d08de23c2bd214da0296365c381945af98d8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Mar 2019 11:07:19 -0700 Subject: [PATCH 615/639] Use Python3 to run tools/venv3.py. (#6860) --- tools/venv3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/venv3.py b/tools/venv3.py index c2374ba5a..b837baf70 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Developer virtualenv setup for Certbot client import _venv_common From 5e64349a4a76f591ec317d11dff87bc4d005dc2f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 15 Mar 2019 00:30:17 +0100 Subject: [PATCH 616/639] Remove tls-sni challenge in standalone plugin (#6856) --- certbot/plugins/standalone.py | 53 ++++++------------------------ certbot/plugins/standalone_test.py | 32 ++++++------------ 2 files changed, 20 insertions(+), 65 deletions(-) diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 16f872a3f..207eff0b6 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -6,7 +6,7 @@ import socket # https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi from socket import errno as socket_errors # type: ignore -import OpenSSL +import OpenSSL # pylint: disable=unused-import import six import zope.interface @@ -55,25 +55,21 @@ class ServerManager(object): :param int port: Port to run the server on. :param challenge_type: Subclass of `acme.challenges.Challenge`, - either `acme.challenge.HTTP01` or `acme.challenges.TLSSNI01`. + currently only `acme.challenge.HTTP01`. :param str listenaddr: (optional) The address to listen on. Defaults to all addrs. :returns: DualNetworkedServers instance. :rtype: ACMEServerMixin """ - assert challenge_type in (challenges.TLSSNI01, challenges.HTTP01) + assert challenge_type == challenges.HTTP01 if port in self._instances: return self._instances[port] address = (listenaddr, port) try: - if challenge_type is challenges.TLSSNI01: - servers = acme_standalone.TLSSNI01DualNetworkedServers( - address, self.certs) # type: acme_standalone.BaseDualNetworkedServers - else: # challenges.HTTP01 - servers = acme_standalone.HTTP01DualNetworkedServers( - address, self.http_01_resources) + servers = acme_standalone.HTTP01DualNetworkedServers( + address, self.http_01_resources) except socket.error as error: raise errors.StandaloneBindError(error, port) @@ -96,8 +92,6 @@ class ServerManager(object): for sockname in instance.getsocknames(): logger.debug("Stopping server at %s:%d...", *sockname[:2]) - # Not calling server_close causes problems when renewing multiple - # certs with `certbot renew` using TLSSNI01 and PyOpenSSL 0.13 instance.shutdown_and_server_close() del self._instances[port] @@ -114,7 +108,7 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.HTTP01, challenges.TLSSNI01] \ +SUPPORTED_CHALLENGES = [challenges.HTTP01] \ # type: List[Type[challenges.KeyAuthorizationChallenge]] @@ -132,8 +126,6 @@ class SupportedChallengesAction(argparse.Action): def _convert_and_validate(self, data): """Validate the value of supported challenges provided by the user. - References to "dvsni" are automatically converted to "tls-sni-01". - :param str data: comma delimited list of challenge types :returns: validated and converted list of challenge types @@ -141,15 +133,6 @@ class SupportedChallengesAction(argparse.Action): """ challs = data.split(",") - - # tls-sni-01 was dvsni during private beta - if "dvsni" in challs: - logger.info( - "Updating legacy standalone_supported_challenges value") - challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall - for chall in challs] - data = ",".join(challs) - unrecognized = [name for name in challs if name not in challenges.Challenge.TYPES] @@ -177,7 +160,7 @@ class Authenticator(common.Plugin): """Standalone Authenticator. This authenticator creates its own ephemeral TCP listener on the - necessary port in order to respond to incoming tls-sni-01 and http-01 + necessary port in order to respond to incoming http-01 challenges from the certificate authority. Therefore, it does not rely on any existing server program. """ @@ -187,10 +170,6 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - # one self-signed key for all tls-sni-01 certificates - self.key = OpenSSL.crypto.PKey() - self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) - self.served = collections.defaultdict(set) # type: ServedType # Stuff below is shared across threads (i.e. servers read @@ -219,9 +198,8 @@ class Authenticator(common.Plugin): def more_info(self): # pylint: disable=missing-docstring return("This authenticator creates its own ephemeral TCP listener " "on the necessary port in order to respond to incoming " - "tls-sni-01 and http-01 challenges from the certificate " - "authority. Therefore, it does not rely on any existing " - "server program.") + "http-01 challenges from the certificate authority. Therefore, " + "it does not rely on any existing server program.") def prepare(self): # pylint: disable=missing-docstring pass @@ -241,10 +219,7 @@ class Authenticator(common.Plugin): _handle_perform_error(error) def _perform_single(self, achall): - if isinstance(achall.chall, challenges.HTTP01): - servers, response = self._perform_http_01(achall) - else: # tls-sni-01 - servers, response = self._perform_tls_sni_01(achall) + servers, response = self._perform_http_01(achall) self.served[servers].add(achall) return response @@ -258,14 +233,6 @@ class Authenticator(common.Plugin): self.http_01_resources.add(resource) return servers, response - def _perform_tls_sni_01(self, achall): - port = self.config.tls_sni_01_port - addr = self.config.tls_sni_01_address - servers = self.servers.run(port, challenges.TLSSNI01, listenaddr=addr) - response, (cert, _) = achall.response_and_validation(cert_key=self.key) - self.certs[response.z_domain] = (self.key, cert) - return servers, response - def cleanup(self, achalls): # pylint: disable=missing-docstring # reduce self.served and close servers if no challenges are served for unused_servers, server_achalls in self.served.items(): diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 9b741fc6f..db4dbeb3b 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -44,9 +44,6 @@ class ServerManagerTest(unittest.TestCase): self.mgr.stop(port=port) self.assertEqual(self.mgr.running(), {}) - def test_run_stop_tls_sni_01(self): - self._test_run_stop(challenges.TLSSNI01) - def test_run_stop_http_01(self): self._test_run_stop(challenges.HTTP01) @@ -98,10 +95,7 @@ class SupportedChallengesActionTest(unittest.TestCase): self.parser.add_argument(self.flag, action=SupportedChallengesAction) def test_correct(self): - self.assertEqual("tls-sni-01", self._call("tls-sni-01")) self.assertEqual("http-01", self._call("http-01")) - self.assertEqual("tls-sni-01,http-01", self._call("tls-sni-01,http-01")) - self.assertEqual("http-01,tls-sni-01", self._call("http-01,tls-sni-01")) def test_unrecognized(self): assert "foo" not in challenges.Challenge.TYPES @@ -110,11 +104,6 @@ class SupportedChallengesActionTest(unittest.TestCase): def test_not_subset(self): self.assertRaises(SystemExit, self._call, "dns") - def test_dvsni(self): - self.assertEqual("tls-sni-01", self._call("dvsni")) - self.assertEqual("http-01,tls-sni-01", self._call("http-01,dvsni")) - self.assertEqual("tls-sni-01,http-01", self._call("dvsni,http-01")) - def get_open_port(): """Gets an open port number from the OS.""" @@ -132,31 +121,31 @@ class AuthenticatorTest(unittest.TestCase): from certbot.plugins.standalone import Authenticator self.config = mock.MagicMock( - tls_sni_01_port=get_open_port(), http01_port=get_open_port(), - standalone_supported_challenges="tls-sni-01,http-01") + http01_port=get_open_port(), + standalone_supported_challenges="http-01") self.auth = Authenticator(self.config, name="standalone") self.auth.servers = mock.MagicMock() def test_supported_challenges(self): self.assertEqual(self.auth.supported_challenges, - [challenges.TLSSNI01, challenges.HTTP01]) + [challenges.HTTP01]) def test_supported_challenges_configured(self): - self.config.standalone_supported_challenges = "tls-sni-01" + self.config.standalone_supported_challenges = "http-01" self.assertEqual(self.auth.supported_challenges, - [challenges.TLSSNI01]) + [challenges.HTTP01]) def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), - [challenges.TLSSNI01, challenges.HTTP01]) + [challenges.HTTP01]) def test_get_chall_pref_configured(self): - self.config.standalone_supported_challenges = "tls-sni-01" + self.config.standalone_supported_challenges = "http-01" self.assertEqual(self.auth.get_chall_pref(domain=None), - [challenges.TLSSNI01]) + [challenges.HTTP01]) def test_perform(self): achalls = self._get_achalls() @@ -212,10 +201,8 @@ class AuthenticatorTest(unittest.TestCase): key = jose.JWK.load(test_util.load_vector('rsa512_key.pem')) http_01 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.HTTP01_P, domain=domain, account_key=key) - tls_sni_01 = achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.TLSSNI01_P, domain=domain, account_key=key) - return [http_01, tls_sni_01] + return [http_01] def test_cleanup(self): self.auth.servers.running.return_value = { @@ -243,5 +230,6 @@ class AuthenticatorTest(unittest.TestCase): "server1": set(), "server2": set([])}) self.auth.servers.stop.assert_called_with(2) + if __name__ == "__main__": unittest.main() # pragma: no cover From c2f2aa5ee076594ad8abc4570df4cbacf7f948b6 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 15 Mar 2019 01:07:49 +0100 Subject: [PATCH 617/639] Remove tls-sni in compatibility tests (#6854) * Reconfigure compatibility tests to use http challenge * Correct simple test * Add a fake DNS resolution for HTTP simple_verify * Debug * More subtle approach: we monkey patch urllib3 to fake a dns resolution to the target IP, allowing every host header to be preserved. * Private package * Relaxed permissions on certbot temp working dir * Move the fake DNS logic in compatibility test, to avoid degrading the acme coverage * Fix lint * Update certbot-compatibility-test/certbot_compatibility_test/configurators/common.py Co-Authored-By: adferrand --- .../configurators/common.py | 3 ++ .../certbot_compatibility_test/test_driver.py | 41 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 2a800c1c2..58ac58a15 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -23,6 +23,9 @@ class Proxy(object): def __init__(self, args): """Initializes the plugin with the given command line args""" self._temp_dir = tempfile.mkdtemp() + # tempfile.mkdtemp() creates folders with too restrictive permissions to be accessible + # to an Apache worker, leading to HTTP challenge failures. Let's fix that. + os.chmod(self._temp_dir, 0o755) self.le_config = util.create_le_config(self._temp_dir) config_dir = util.extract_configs(args.configs, self._temp_dir) self._configs = [ diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 9eea95e67..f94aee3c1 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -1,5 +1,6 @@ """Tests Certbot plugins against different server configurations.""" import argparse +import contextlib import filecmp import logging import os @@ -7,6 +8,7 @@ import shutil import tempfile import time import sys +from urllib3.util import connection import OpenSSL @@ -64,18 +66,19 @@ def test_authenticator(plugin, config, temp_dir): "Plugin failed to complete %s for %s in %s", type(achalls[i]), achalls[i].domain, config) success = False - elif isinstance(responses[i], challenges.TLSSNI01Response): - verified = responses[i].simple_verify(achalls[i].chall, - achalls[i].domain, - util.JWK.public_key(), - host="127.0.0.1", - port=plugin.https_port) + elif isinstance(responses[i], challenges.HTTP01Response): + # We fake the DNS resolution to ensure that any domain is resolved + # to the local HTTP server setup for the compatibility tests + with _fake_dns_resolution("127.0.0.1"): + verified = responses[i].simple_verify( + achalls[i].chall, achalls[i].domain, + util.JWK.public_key(), port=plugin.http_port) if verified: logger.info( - "tls-sni-01 verification for %s succeeded", achalls[i].domain) + "http-01 verification for %s succeeded", achalls[i].domain) else: logger.error( - "**** tls-sni-01 verification for %s in %s failed", + "**** http-01 verification for %s in %s failed", achalls[i].domain, config) success = False @@ -102,9 +105,9 @@ def _create_achalls(plugin): for domain in names: prefs = plugin.get_chall_pref(domain) for chall_type in prefs: - if chall_type == challenges.TLSSNI01: - chall = challenges.TLSSNI01( - token=os.urandom(challenges.TLSSNI01.TOKEN_SIZE)) + if chall_type == challenges.HTTP01: + chall = challenges.HTTP01( + token=os.urandom(challenges.HTTP01.TOKEN_SIZE)) challb = acme_util.chall_to_challb( chall, messages.STATUS_PENDING) achall = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -369,5 +372,21 @@ def main(): sys.exit(1) +@contextlib.contextmanager +def _fake_dns_resolution(resolved_ip): + """Monkey patch urllib3 to make any hostname be resolved to the provided IP""" + _original_create_connection = connection.create_connection + + def _patched_create_connection(address, *args, **kwargs): + _, port = address + return _original_create_connection((resolved_ip, port), *args, **kwargs) + + try: + connection.create_connection = _patched_create_connection + yield + finally: + connection.create_connection = _original_create_connection + + if __name__ == "__main__": main() From e909b0852c124a3121b6c5d67fa604b1a4b998cf Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 15 Mar 2019 01:56:56 +0100 Subject: [PATCH 618/639] Remove tls-sni challenge in manual plugin (#6855) * Remove tls-sni challenge in manual * Remove unused logic --- certbot/plugins/manual.py | 76 +++------------------------------- certbot/plugins/manual_test.py | 51 ++++------------------- 2 files changed, 13 insertions(+), 114 deletions(-) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 123a5bfea..b4d20478a 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -15,34 +15,6 @@ from certbot import reverter from certbot.plugins import common -class ManualTlsSni01(common.TLSSNI01): - """TLS-SNI-01 authenticator for the Manual plugin - - :ivar configurator: Authenticator object - :type configurator: :class:`~certbot.plugins.manual.Authenticator` - - :ivar list achalls: Annotated - class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - challenges - - :param list indices: Meant to hold indices of challenges in a - larger array. NginxTlsSni01 is capable of solving many challenges - at once which causes an indexing issue within NginxConfigurator - who must return all responses in order. Imagine NginxConfigurator - maintaining state about where all of the http-01 Challenges, - TLS-SNI-01 Challenges belong in the response array. This is an - optional utility. - - :param str challenge_conf: location of the challenge config file - """ - - def perform(self): - """Create the SSL certificates and private keys""" - - for achall in self.achalls: - self._setup_challenge_cert(achall) - - @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): @@ -63,14 +35,9 @@ class Authenticator(common.Plugin): 'type of challenge. $CERTBOT_DOMAIN will always contain the domain ' 'being authenticated. For HTTP-01 and DNS-01, $CERTBOT_VALIDATION ' 'is the validation string, and $CERTBOT_TOKEN is the filename of the ' - 'resource requested when performing an HTTP-01 challenge. When ' - 'performing a TLS-SNI-01 challenge, $CERTBOT_SNI_DOMAIN will contain ' - 'the SNI name for which the ACME server expects to be presented with ' - 'the self-signed certificate located at $CERTBOT_CERT_PATH. The ' - 'secret key needed to complete the TLS handshake is located at ' - '$CERTBOT_KEY_PATH. An additional cleanup script can also be ' - 'provided and can use the additional variable $CERTBOT_AUTH_OUTPUT ' - 'which contains the stdout output from the auth script.') + 'resource requested when performing an HTTP-01 challenge. An additional ' + 'cleanup script can also be provided and can use the additional variable ' + '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.') _DNS_INSTRUCTIONS = """\ Please deploy a DNS TXT record under the name {domain} with the following value: @@ -86,14 +53,6 @@ Create a file containing just this data: And make it available on your web server at this URL: {uri} -""" - _TLSSNI_INSTRUCTIONS = """\ -Configure the service listening on port {port} to present the certificate -{cert} -using the secret key -{key} -when it receives a TLS ClientHello with the SNI extension set to -{sni_domain} """ _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """ (This must be set up in addition to the previous challenges; do not remove, @@ -112,7 +71,6 @@ permitted by DNS standards.) self.reverter.recovery_routine() self.env = dict() \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] - self.tls_sni_01 = None self.subsequent_dns_challenge = False self.subsequent_any_challenge = False @@ -149,7 +107,7 @@ permitted by DNS standards.) def get_chall_pref(self, domain): # pylint: disable=missing-docstring,no-self-use,unused-argument - return [challenges.HTTP01, challenges.DNS01, challenges.TLSSNI01] + return [challenges.HTTP01, challenges.DNS01] def perform(self, achalls): # pylint: disable=missing-docstring self._verify_ip_logging_ok() @@ -160,12 +118,6 @@ permitted by DNS standards.) responses = [] for achall in achalls: - if isinstance(achall.chall, challenges.TLSSNI01): - # Make a new ManualTlsSni01 instance for each challenge - # because the manual plugin deals with one challenge at a time. - self.tls_sni_01 = ManualTlsSni01(self) - self.tls_sni_01.add_chall(achall) - self.tls_sni_01.perform() perform_achall(achall) responses.append(achall.response(achall.account_key)) return responses @@ -191,16 +143,6 @@ permitted by DNS standards.) env['CERTBOT_TOKEN'] = achall.chall.encode('token') else: os.environ.pop('CERTBOT_TOKEN', None) - if isinstance(achall.chall, challenges.TLSSNI01): - env['CERTBOT_CERT_PATH'] = self.tls_sni_01.get_cert_path(achall) - env['CERTBOT_KEY_PATH'] = self.tls_sni_01.get_key_path(achall) - env['CERTBOT_SNI_DOMAIN'] = self.tls_sni_01.get_z_domain(achall) - os.environ.pop('CERTBOT_VALIDATION', None) - env.pop('CERTBOT_VALIDATION') - else: - os.environ.pop('CERTBOT_CERT_PATH', None) - os.environ.pop('CERTBOT_KEY_PATH', None) - os.environ.pop('CERTBOT_SNI_DOMAIN', None) os.environ.update(env) _, out = self._execute_hook('auth-hook') env['CERTBOT_AUTH_OUTPUT'] = out.strip() @@ -213,17 +155,11 @@ permitted by DNS standards.) achall=achall, encoded_token=achall.chall.encode('token'), port=self.config.http01_port, uri=achall.chall.uri(achall.domain), validation=validation) - elif isinstance(achall.chall, challenges.DNS01): + else: + assert isinstance(achall.chall, challenges.DNS01) msg = self._DNS_INSTRUCTIONS.format( domain=achall.validation_domain_name(achall.domain), validation=validation) - else: - assert isinstance(achall.chall, challenges.TLSSNI01) - msg = self._TLSSNI_INSTRUCTIONS.format( - cert=self.tls_sni_01.get_cert_path(achall), - key=self.tls_sni_01.get_key_path(achall), - port=self.config.tls_sni_01_port, - sni_domain=self.tls_sni_01.get_z_domain(achall)) if isinstance(achall.chall, challenges.DNS01): if self.subsequent_dns_challenge: # 2nd or later dns-01 challenge diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index 0938e8a7d..b566f6340 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -22,8 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A self.dns_achall_2 = acme_util.DNS01_A_2 - self.tls_sni_achall = acme_util.TLSSNI01_A - self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2] + self.achalls = [self.http_achall, self.dns_achall, self.dns_achall_2] for d in ["config_dir", "work_dir", "in_progress"]: os.mkdir(os.path.join(self.tempdir, d)) # "backup_dir" and "temp_checkpoint_dir" get created in @@ -38,8 +37,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): backup_dir=os.path.join(self.tempdir, "backup_dir"), temp_checkpoint_dir=os.path.join( self.tempdir, "temp_checkpoint_dir"), - in_progress_dir=os.path.join(self.tempdir, "in_progess"), - tls_sni_01_port=5001) + in_progress_dir=os.path.join(self.tempdir, "in_progess")) from certbot.plugins.manual import Authenticator self.auth = Authenticator(self.config, name='manual') @@ -58,9 +56,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), - [challenges.HTTP01, - challenges.DNS01, - challenges.TLSSNI01]) + [challenges.HTTP01, challenges.DNS01]) @test_util.patch_get_utility() def test_ip_logging_not_ok(self, mock_get_utility): @@ -79,18 +75,13 @@ class AuthenticatorTest(test_util.TempDirTestCase): '{0} -c "from __future__ import print_function;' 'import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' - 'print(os.environ.get(\'CERTBOT_CERT_PATH\', \'nocert\'));' - 'print(os.environ.get(\'CERTBOT_KEY_PATH\', \'nokey\'));' - 'print(os.environ.get(\'CERTBOT_SNI_DOMAIN\', \'nosnidomain\'));' 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' .format(sys.executable)) - dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format( + dns_expected = '{0}\n{1}\n{2}'.format( self.dns_achall.domain, 'notoken', - 'nocert', 'nokey', 'nosnidomain', self.dns_achall.validation(self.dns_achall.account_key)) - http_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format( + http_expected = '{0}\n{1}\n{2}'.format( self.http_achall.domain, self.http_achall.chall.encode('token'), - 'nocert', 'nokey', 'nosnidomain', self.http_achall.validation(self.http_achall.account_key)) self.assertEqual( @@ -102,17 +93,6 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertEqual( self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'], http_expected) - # tls_sni_01 challenge must be perform()ed above before we can - # get the cert_path and key_path. - tls_sni_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format( - self.tls_sni_achall.domain, 'notoken', - self.auth.tls_sni_01.get_cert_path(self.tls_sni_achall), - self.auth.tls_sni_01.get_key_path(self.tls_sni_achall), - self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall), - 'novalidation') - self.assertEqual( - self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'], - tls_sni_expected) @test_util.patch_get_utility() def test_manual_perform(self, mock_get_utility): @@ -122,13 +102,8 @@ class AuthenticatorTest(test_util.TempDirTestCase): [achall.response(achall.account_key) for achall in self.achalls]) for i, (args, kwargs) in enumerate(mock_get_utility().notification.call_args_list): achall = self.achalls[i] - if isinstance(achall.chall, challenges.TLSSNI01): - self.assertTrue( - self.auth.tls_sni_01.get_cert_path( - self.tls_sni_achall) in args[0]) - else: - self.assertTrue( - achall.validation(achall.account_key) in args[0]) + self.assertTrue( + achall.validation(achall.account_key) in args[0]) self.assertFalse(kwargs['wrap']) @test_util.broken_on_windows @@ -153,18 +128,6 @@ class AuthenticatorTest(test_util.TempDirTestCase): achall.chall.encode('token')) else: self.assertFalse('CERTBOT_TOKEN' in os.environ) - if isinstance(achall.chall, challenges.TLSSNI01): - self.assertEqual( - os.environ['CERTBOT_CERT_PATH'], - self.auth.tls_sni_01.get_cert_path(achall)) - self.assertEqual( - os.environ['CERTBOT_KEY_PATH'], - self.auth.tls_sni_01.get_key_path(achall)) - self.assertFalse( - os.path.exists(os.environ['CERTBOT_CERT_PATH'])) - self.assertFalse( - os.path.exists(os.environ['CERTBOT_KEY_PATH'])) - if __name__ == '__main__': From b447b0a8e9d593e085187f30de927a201bac1006 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 16 Mar 2019 00:39:43 +0100 Subject: [PATCH 619/639] Remove tls sni in apache plugin (#6858) * Add a dedicated configuration to define what is the HTTPS port for this certbot instance. * Remove tls-sni in apache plugin * Update constants.py * Update interfaces.py * Remove option * Simplify a test --- certbot-apache/certbot_apache/configurator.py | 23 +-- .../certbot_apache/tests/configurator_test.py | 21 +-- .../certbot_apache/tests/tls_sni_01_test.py | 151 --------------- certbot-apache/certbot_apache/tls_sni_01.py | 174 ------------------ 4 files changed, 8 insertions(+), 361 deletions(-) delete mode 100644 certbot-apache/certbot_apache/tests/tls_sni_01_test.py delete mode 100644 certbot-apache/certbot_apache/tls_sni_01.py diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index f7f4ff925..a3061119b 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -32,7 +32,6 @@ from certbot_apache import display_ops from certbot_apache import http_01 from certbot_apache import obj from certbot_apache import parser -from certbot_apache import tls_sni_01 from collections import defaultdict @@ -215,15 +214,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" - return os.path.join(self.config.config_dir, - constants.MOD_SSL_CONF_DEST) + return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST) @property def updated_mod_ssl_conf_digest(self): """Full absolute path to digest of updated SSL configuration file.""" return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST) - def prepare(self): """Prepare the authenticator/installer. @@ -396,7 +393,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(name.split(".")) == len(domain.split(".")): return fnmatch.fnmatch(name, domain) - def _choose_vhosts_wildcard(self, domain, create_ssl=True): """Prompts user to choose vhosts to install a wildcard certificate for""" @@ -441,7 +437,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._wildcard_vhosts[domain] = return_vhosts return return_vhosts - def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): """ Helper function for deploy_cert() that handles the actual deployment @@ -449,8 +444,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): domain originally passed for deploy_cert(). This is especially true with wildcard certificates """ - - # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately self.prepare_server_https("443") @@ -1925,7 +1918,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir(vhost.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS) - def _verify_no_certbot_redirect(self, vhost): """Checks to see if a redirect was already installed by certbot. @@ -2193,7 +2185,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :raises .errors.MisconfigurationError: If reload fails """ - error = "" try: util.run_script(self.option("restart_cmd")) except errors.SubprocessError as err: @@ -2267,7 +2258,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.HTTP01, challenges.TLSSNI01] + return [challenges.HTTP01] def perform(self, achalls): """Perform the configuration related challenge. @@ -2280,20 +2271,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._chall_out.update(achalls) responses = [None] * len(achalls) http_doer = http_01.ApacheHttp01(self) - sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - if isinstance(achall.chall, challenges.HTTP01): - http_doer.add_chall(achall, i) - else: # tls-sni-01 - sni_doer.add_chall(achall, i) + http_doer.add_chall(achall, i) http_response = http_doer.perform() - sni_response = sni_doer.perform() - if http_response or sni_response: + if http_response: # Must reload in order to activate the challenges. # Handled here because we may be able to load up other challenge # types @@ -2304,7 +2290,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): time.sleep(3) self._update_responses(responses, http_response, http_doer) - self._update_responses(responses, sni_response, sni_doer) return responses diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7c281d707..ee4833ebb 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -812,32 +812,19 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(self.config.add_name_vhost.call_count, 2) @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform") - @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform): + def test_perform(self, mock_restart, mock_http_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded account_key, achalls = self.get_key_and_achalls() - all_expected = [] - http_expected = [] - tls_expected = [] - for achall in achalls: - response = achall.response(account_key) - if isinstance(achall.chall, challenges.HTTP01): - http_expected.append(response) - else: - tls_expected.append(response) - all_expected.append(response) - - mock_http_perform.return_value = http_expected - mock_tls_perform.return_value = tls_expected + expected = [achall.response(account_key) for achall in achalls] + mock_http_perform.return_value = expected responses = self.config.perform(achalls) self.assertEqual(mock_http_perform.call_count, 1) - self.assertEqual(mock_tls_perform.call_count, 1) - self.assertEqual(responses, all_expected) + self.assertEqual(responses, expected) self.assertEqual(mock_restart.call_count, 1) diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py deleted file mode 100644 index 8cea97f04..000000000 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Test for certbot_apache.tls_sni_01.""" -import shutil -import unittest - -import mock - -from certbot import errors -from certbot.plugins import common_test - -from certbot_apache import obj -from certbot_apache.tests import util - -from six.moves import xrange # pylint: disable=redefined-builtin, import-error - - -class TlsSniPerformTest(util.ApacheTest): - """Test the ApacheTlsSni01 challenge.""" - - auth_key = common_test.AUTH_KEY - achalls = common_test.ACHALLS - - def setUp(self): # pylint: disable=arguments-differ - super(TlsSniPerformTest, self).setUp() - - config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, - self.work_dir) - config.config.tls_sni_01_port = 443 - - from certbot_apache import tls_sni_01 - self.sni = tls_sni_01.ApacheTlsSni01(config) - - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - - def test_perform0(self): - resp = self.sni.perform() - self.assertEqual(len(resp), 0) - - @mock.patch("certbot.util.exe_exists") - @mock.patch("certbot.util.run_script") - def test_perform1(self, _, mock_exists): - self.sni.configurator.parser.modules.add("socache_shmcb_module") - self.sni.configurator.parser.modules.add("ssl_module") - - mock_exists.return_value = True - self.sni.configurator.parser.update_runtime_variables = mock.Mock() - - achall = self.achalls[0] - self.sni.add_chall(achall) - response = self.achalls[0].response(self.auth_key) - mock_setup_cert = mock.MagicMock(return_value=response) - # pylint: disable=protected-access - self.sni._setup_challenge_cert = mock_setup_cert - - responses = self.sni.perform() - mock_setup_cert.assert_called_once_with(achall) - - # Check to make sure challenge config path is included in apache config - self.assertEqual( - len(self.sni.configurator.parser.find_dir( - "Include", self.sni.challenge_conf)), 1) - self.assertEqual(len(responses), 1) - self.assertEqual(responses[0], response) - - def test_perform2(self): - # Avoid load module - self.sni.configurator.parser.modules.add("ssl_module") - self.sni.configurator.parser.modules.add("socache_shmcb_module") - acme_responses = [] - for achall in self.achalls: - self.sni.add_chall(achall) - acme_responses.append(achall.response(self.auth_key)) - - mock_setup_cert = mock.MagicMock(side_effect=acme_responses) - # pylint: disable=protected-access - self.sni._setup_challenge_cert = mock_setup_cert - - with mock.patch( - "certbot_apache.override_debian.DebianConfigurator.enable_mod"): - sni_responses = self.sni.perform() - - self.assertEqual(mock_setup_cert.call_count, 2) - - # Make sure calls made to mocked function were correct - self.assertEqual( - mock_setup_cert.call_args_list[0], mock.call(self.achalls[0])) - self.assertEqual( - mock_setup_cert.call_args_list[1], mock.call(self.achalls[1])) - - self.assertEqual( - len(self.sni.configurator.parser.find_dir( - "Include", self.sni.challenge_conf)), - 1) - self.assertEqual(len(sni_responses), 2) - for i in xrange(2): - self.assertEqual(sni_responses[i], acme_responses[i]) - - def test_mod_config(self): - z_domains = [] - for achall in self.achalls: - self.sni.add_chall(achall) - z_domain = achall.response(self.auth_key).z_domain - z_domains.append(set([z_domain.decode('ascii')])) - - self.sni._mod_config() # pylint: disable=protected-access - self.sni.configurator.save() - - self.sni.configurator.parser.find_dir( - "Include", self.sni.challenge_conf) - vh_match = self.sni.configurator.aug.match( - "/files" + self.sni.challenge_conf + "//VirtualHost") - - vhs = [] - for match in vh_match: - # pylint: disable=protected-access - vhs.append(self.sni.configurator._create_vhost(match)) - self.assertEqual(len(vhs), 2) - for vhost in vhs: - self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")])) - names = vhost.get_names() - self.assertTrue(names in z_domains) - - def test_get_addrs_default(self): - self.sni.configurator.choose_vhost = mock.Mock( - return_value=obj.VirtualHost( - "path", "aug_path", - set([obj.Addr.fromstring("_default_:443")]), - False, False) - ) - - # pylint: disable=protected-access - self.assertEqual( - set([obj.Addr.fromstring("*:443")]), - self.sni._get_addrs(self.achalls[0])) - - def test_get_addrs_no_vhost_found(self): - self.sni.configurator.choose_vhost = mock.Mock( - side_effect=errors.MissingCommandlineFlag( - "Failed to run Apache plugin non-interactively")) - - # pylint: disable=protected-access - self.assertEqual( - set([obj.Addr.fromstring("*:443")]), - self.sni._get_addrs(self.achalls[0])) - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py deleted file mode 100644 index 65230cdcb..000000000 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ /dev/null @@ -1,174 +0,0 @@ -"""A class that performs TLS-SNI-01 challenges for Apache""" - -import os -import logging - -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from certbot.plugins import common -from certbot.errors import PluginError, MissingCommandlineFlag - -from certbot_apache import obj - -logger = logging.getLogger(__name__) - - -class ApacheTlsSni01(common.TLSSNI01): - """Class that performs TLS-SNI-01 challenges within the Apache configurator - - :ivar configurator: ApacheConfigurator object - :type configurator: :class:`~apache.configurator.ApacheConfigurator` - - :ivar list achalls: Annotated TLS-SNI-01 - (`.KeyAuthorizationAnnotatedChallenge`) challenges. - - :param list indices: Meant to hold indices of challenges in a - larger array. ApacheTlsSni01 is capable of solving many challenges - at once which causes an indexing issue within ApacheConfigurator - who must return all responses in order. Imagine ApacheConfigurator - maintaining state about where all of the http-01 Challenges, - TLS-SNI-01 Challenges belong in the response array. This is an - optional utility. - - :param str challenge_conf: location of the challenge config file - - """ - - VHOST_TEMPLATE = """\ - - ServerName {server_name} - UseCanonicalName on - SSLStrictSNIVHostCheck on - - LimitRequestBody 1048576 - - Include {ssl_options_conf_path} - SSLCertificateFile {cert_path} - SSLCertificateKeyFile {key_path} - - DocumentRoot {document_root} - - -""" - - def __init__(self, *args, **kwargs): - super(ApacheTlsSni01, self).__init__(*args, **kwargs) - - self.challenge_conf = os.path.join( - self.configurator.conf("challenge-location"), - "le_tls_sni_01_cert_challenge.conf") - - def perform(self): - """Perform a TLS-SNI-01 challenge.""" - if not self.achalls: - return [] - # Save any changes to the configuration as a precaution - # About to make temporary changes to the config - self.configurator.save("Changes before challenge setup", True) - - # Prepare the server for HTTPS - self.configurator.prepare_server_https( - str(self.configurator.config.tls_sni_01_port), True) - - responses = [] - - # Create all of the challenge certs - for achall in self.achalls: - responses.append(self._setup_challenge_cert(achall)) - - # Setup the configuration - addrs = self._mod_config() - self.configurator.save("Don't lose mod_config changes", True) - self.configurator.make_addrs_sni_ready(addrs) - - # Save reversible changes - self.configurator.save("SNI Challenge", True) - - return responses - - def _mod_config(self): - """Modifies Apache config files to include challenge vhosts. - - Result: Apache config includes virtual servers for issued challs - - :returns: All TLS-SNI-01 addresses used - :rtype: set - - """ - addrs = set() # type: Set[obj.Addr] - config_text = "\n" - - for achall in self.achalls: - achall_addrs = self._get_addrs(achall) - addrs.update(achall_addrs) - - config_text += self._get_config_text(achall, achall_addrs) - - config_text += "\n" - - self.configurator.parser.add_include( - self.configurator.parser.loc["default"], self.challenge_conf) - self.configurator.reverter.register_file_creation( - True, self.challenge_conf) - - logger.debug("writing a config file with text:\n %s", config_text) - with open(self.challenge_conf, "w") as new_conf: - new_conf.write(config_text) - - return addrs - - def _get_addrs(self, achall): - """Return the Apache addresses needed for TLS-SNI-01.""" - # TODO: Checkout _default_ rules. - addrs = set() - default_addr = obj.Addr(("*", str( - self.configurator.config.tls_sni_01_port))) - - try: - vhost = self.configurator.choose_vhost(achall.domain, - create_if_no_ssl=False) - except (PluginError, MissingCommandlineFlag): - # We couldn't find the virtualhost for this domain, possibly - # because it's a new vhost that's not configured yet - # (GH #677). See also GH #2600. - logger.warning("Falling back to default vhost %s...", default_addr) - addrs.add(default_addr) - return addrs - - for addr in vhost.addrs: - if "_default_" == addr.get_addr(): - addrs.add(default_addr) - else: - addrs.add( - addr.get_sni_addr( - self.configurator.config.tls_sni_01_port)) - - return addrs - - def _get_config_text(self, achall, ip_addrs): - """Chocolate virtual server configuration text - - :param .KeyAuthorizationAnnotatedChallenge achall: Annotated - TLS-SNI-01 challenge. - - :param list ip_addrs: addresses of challenged domain - :class:`list` of type `~.obj.Addr` - - :returns: virtual host configuration text - :rtype: str - - """ - ips = " ".join(str(i) for i in ip_addrs) - document_root = os.path.join( - self.configurator.config.work_dir, "tls_sni_01_page/") - # TODO: Python docs is not clear how multiline string literal - # newlines are parsed on different platforms. At least on - # Linux (Debian sid), when source file uses CRLF, Python still - # parses it as "\n"... c.f.: - # https://docs.python.org/2.7/reference/lexical_analysis.html - return self.VHOST_TEMPLATE.format( - vhost=ips, - server_name=achall.response(achall.account_key).z_domain.decode('ascii'), - ssl_options_conf_path=self.configurator.mod_ssl_conf, - cert_path=self.get_cert_path(achall), - key_path=self.get_key_path(achall), - document_root=document_root).replace("\n", os.linesep) From d9880721b31ab4e9e9d2fdc38d83ff2bd9078378 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 18 Mar 2019 18:22:19 +0100 Subject: [PATCH 620/639] Remove tls sni in nginx plugin (#6857) * Remove tls-sni from nginx config * Add a dedicated configuration to define what is the HTTPS port for this certbot instance. * Correct some tests * Reestablish default vhost creation * Clean tls references for nginx integration tests * Associate https_port only to tests and nginx --- certbot-nginx/certbot_nginx/configurator.py | 26 +-- .../certbot_nginx/tests/configurator_test.py | 23 +-- .../certbot_nginx/tests/tls_sni_01_test.py | 158 ---------------- certbot-nginx/certbot_nginx/tests/util.py | 2 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 177 ------------------ .../tests/boulder-integration.conf.sh | 2 +- tests/certbot-boulder-integration.sh | 2 +- tests/integration/_common.sh | 6 +- 8 files changed, 21 insertions(+), 375 deletions(-) delete mode 100644 certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py delete mode 100644 certbot-nginx/certbot_nginx/tls_sni_01.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index ffe1ddac7..2357735f9 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -26,7 +26,6 @@ from certbot_nginx import constants from certbot_nginx import display_ops from certbot_nginx import nginxparser from certbot_nginx import parser -from certbot_nginx import tls_sni_01 from certbot_nginx import http_01 from certbot_nginx import obj # pylint: disable=unused-import from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module @@ -149,7 +148,6 @@ class NginxConfigurator(common.Installer): # Make sure configuration is valid self.config_test() - self.parser = parser.NginxParser(self.conf('server-root')) install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest) @@ -561,7 +559,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names = set() # type: Set[str] for vhost in self.parser.get_vhosts(): all_names.update(vhost.names) @@ -611,7 +609,8 @@ class NginxConfigurator(common.Installer): :type vhost: :class:`~certbot_nginx.obj.VirtualHost` """ - ipv6info = self.ipv6_info(self.config.tls_sni_01_port) + https_port = self.config.tls_sni_01_port + ipv6info = self.ipv6_info(https_port) ipv6_block = [''] ipv4_block = [''] @@ -625,7 +624,7 @@ class NginxConfigurator(common.Installer): ipv6_block = ['\n ', 'listen', ' ', - '[::]:{0}'.format(self.config.tls_sni_01_port), + '[::]:{0}'.format(https_port), ' ', 'ssl'] if not ipv6info[1]: @@ -637,7 +636,7 @@ class NginxConfigurator(common.Installer): ipv4_block = ['\n ', 'listen', ' ', - '{0}'.format(self.config.tls_sni_01_port), + '{0}'.format(https_port), ' ', 'ssl'] @@ -799,8 +798,6 @@ class NginxConfigurator(common.Installer): :param str domain: domain to enable redirect for :param `~obj.Vhost` vhost: vhost to enable redirect for """ - - http_vhost = None if vhost.ssl: http_vhost, _ = self._split_block(vhost, ['listen', 'server_name']) @@ -1051,19 +1048,14 @@ class NginxConfigurator(common.Installer): """ self._chall_out += len(achalls) responses = [None] * len(achalls) - sni_doer = tls_sni_01.NginxTlsSni01(self) http_doer = http_01.NginxHttp01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - if isinstance(achall.chall, challenges.HTTP01): - http_doer.add_chall(achall, i) - else: # tls-sni-01 - sni_doer.add_chall(achall, i) + http_doer.add_chall(achall, i) - sni_response = sni_doer.perform() http_response = http_doer.perform() # Must restart in order to activate the challenges. # Handled here because we may be able to load up other challenge types @@ -1072,9 +1064,8 @@ class NginxConfigurator(common.Installer): # Go through all of the challenges and assign them to the proper place # in the responses return value. All responses must be in the same order # as the original challenges. - for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)): - for i, resp in enumerate(chall_response): - responses[chall_doer.indices[i]] = resp + for i, resp in enumerate(http_response): + responses[http_doer.indices[i]] = resp return responses @@ -1152,6 +1143,7 @@ def install_ssl_options_conf(options_ssl, options_ssl_digest): return common.install_version_controlled_file(options_ssl, options_ssl_digest, constants.MOD_SSL_CONF_SRC, constants.ALL_SSL_OPTIONS_HASHES) + def _determine_default_server_root(): if os.environ.get("CERTBOT_DOCS") == "1": default_server_root = "%s or %s" % (constants.LINUX_SERVER_ROOT, diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 706e2637a..08e4a56ae 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -318,21 +318,13 @@ class NginxConfiguratorTest(util.NginxTest): ]], parsed_migration_conf[0]) - @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform") @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform") @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart") @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config") - def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform, - mock_tls_perform): + def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( - challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), - uri="https://ca.org/chall0_uri", - status=messages.Status("pending"), - ), domain="localhost", account_key=self.rsa512jwk) - achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( + achall = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", @@ -340,19 +332,16 @@ class NginxConfiguratorTest(util.NginxTest): ), domain="example.com", account_key=self.rsa512jwk) expected = [ - achall1.response(self.rsa512jwk), - achall2.response(self.rsa512jwk), + achall.response(self.rsa512jwk), ] - mock_tls_perform.return_value = expected[:1] - mock_http_perform.return_value = expected[1:] - responses = self.config.perform([achall1, achall2]) + mock_http_perform.return_value = expected[:] + responses = self.config.perform([achall]) - self.assertEqual(mock_tls_perform.call_count, 1) self.assertEqual(mock_http_perform.call_count, 1) self.assertEqual(responses, expected) - self.config.cleanup([achall1, achall2]) + self.config.cleanup([achall]) self.assertEqual(0, self.config._chall_out) # pylint: disable=protected-access self.assertEqual(mock_revert.call_count, 1) self.assertEqual(mock_restart.call_count, 2) diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py deleted file mode 100644 index 62ca085ef..000000000 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Tests for certbot_nginx.tls_sni_01""" -import unittest - -import mock -import six - -from acme import challenges - -from certbot import achallenges -from certbot import errors - -from certbot.plugins import common_test -from certbot.tests import acme_util - -from certbot_nginx import obj -from certbot_nginx.tests import util - - -class TlsSniPerformTest(util.NginxTest): - """Test the NginxTlsSni01 challenge.""" - - account_key = common_test.AUTH_KEY - achalls = [ - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), - domain="www.example.com", account_key=account_key), - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01( - token=b"\xba\xa9\xda? Date: Mon, 25 Mar 2019 18:52:59 +0100 Subject: [PATCH 621/639] Construct the sanitized, pinned and hashed requirements file for certbot-auto (#6839) * Setup an independant create_venv piece for certbot-auto * Debug * First implementation * Some corrections, disable python 3 * Continue work * Add hashin * Polish CLI * Fix logic * Add executable permissions * Assynchronous process * Correction * Add comments * More controls * Correct image name * Fix image * Test with 2 * Test timeout * Remove parallelization for now. To much bugs. * Add comments * Correct installation * Correct keys map view usage * Improve filtering * Correction * Improve filtering, again * Remove dependency on python 3 * Remove necessity to run from certbot root * Add constraints. Clean code. * Pure constraints * More involved base test * Update certbot-auto with calculated dependencies * Update header * Rebuild UI * Correction * Remove debug info * Ensure docker exit when process finish * Another try to stop docker * Clean stdout/stderr * Fix python-augeas * Catch stderr * Update dependencies with new constraints * Update certbot-auto * Corrections after review. * Clean endline * Silent execution * Filter editable installation of local certbot packages, strict check on package names --- letsencrypt-auto-source/letsencrypt-auto | 415 +++++++++--------- .../letsencrypt-auto.template | 25 +- letsencrypt-auto-source/pieces/create_venv.py | 27 ++ .../pieces/dependency-requirements.txt | 361 ++++++++------- letsencrypt-auto-source/pieces/pipstrap.py | 2 +- .../rebuild_dependencies.py | 275 ++++++++++++ 6 files changed, 703 insertions(+), 402 deletions(-) create mode 100755 letsencrypt-auto-source/pieces/create_venv.py create mode 100755 letsencrypt-auto-source/rebuild_dependencies.py diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 2a44a58dc..822266785 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -908,6 +908,41 @@ else: UNLIKELY_EOF } +# Create a new virtual environment for Certbot. It will overwrite any existing one. +# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE +CreateVenv() { + "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF" +#!/usr/bin/env python +import os +import shutil +import subprocess +import sys + + +def create_venv(venv_path, pyver, verbose): + if os.path.exists(venv_path): + shutil.rmtree(venv_path) + + stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w') + + if int(pyver) <= 27: + # Use virtualenv binary + environ = os.environ.copy() + environ['VIRTUALENV_NO_DOWNLOAD'] = '1' + command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path] + subprocess.check_call(command, stdout=stdout, env=environ) + else: + # Use embedded venv module in Python 3 + command = [sys.executable, '-m', 'venv', venv_path] + subprocess.check_call(command, stdout=stdout) + + +if __name__ == '__main__': + create_venv(*sys.argv[1:]) + +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -963,22 +998,7 @@ if [ "$1" = "--le-auto-phase2" ]; then if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then say "Creating virtual environment..." DeterminePythonVersion - rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - # Use an environment variable instead of a flag for compatibility with old versions - if [ "$VERBOSE" = 1 ]; then - VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ - > /dev/null - fi - else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi - fi + CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE" if [ -n "$BOOTSTRAP_VERSION" ]; then echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" @@ -992,202 +1012,195 @@ if [ "$1" = "--le-auto-phase2" ]; then # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -# This is the flattened list of packages certbot-auto installs. To generate -# this, do -# `pip install --no-cache-dir -e acme -e . -e certbot-apache -e certbot-nginx`, -# and then use `hashin` or a more secure method to gather the hashes. - -# Hashin example: -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# sets the new certbot-auto pinned version of cryptography to 1.5.2 - -argparse==1.4.0 \ - --hash=sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314 \ - --hash=sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4 - -# This comes before cffi because cffi will otherwise install an unchecked -# version via setup_requires. -pycparser==2.14 \ - --hash=sha256:7959b4a74abdc27b312fed1c21e6caf9309ce0b29ea86b591fd2e99ecdf27f73 \ - --no-binary pycparser - -asn1crypto==0.22.0 \ - --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ - --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.11.5 \ - --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ - --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ - --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ - --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ - --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ - --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ - --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ - --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ - --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ - --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ - --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ - --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ - --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ - --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ - --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ - --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ - --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ - --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ - --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ - --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ - --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ - --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ - --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ - --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ - --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ - --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ - --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ - --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ - --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ - --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ - --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ - --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 -ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ - --no-binary ConfigArgParse +# This is the flattened list of packages certbot-auto installs. +# To generate this, do (with docker and package hashin installed): +# ``` +# letsencrypt-auto-source/rebuild_dependencies.py \ +# letsencrypt-auto-sources/pieces/dependency-requirements.txt +# ``` +ConfigArgParse==0.14.0 \ + --hash=sha256:2e2efe2be3f90577aca9415e32cb629aa2ecd92078adbe27b53a03e53ff12e91 +asn1crypto==0.24.0 \ + --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ + --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae +cffi==1.12.2 \ + --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \ + --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \ + --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \ + --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \ + --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \ + --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \ + --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \ + --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \ + --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \ + --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \ + --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \ + --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \ + --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \ + --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \ + --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \ + --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \ + --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \ + --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \ + --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \ + --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \ + --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \ + --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \ + --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \ + --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \ + --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \ + --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \ + --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \ + --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ - --no-binary configobj -cryptography==2.5 \ - --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ - --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ - --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ - --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ - --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ - --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ - --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ - --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ - --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ - --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ - --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ - --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ - --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ - --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ - --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ - --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ - --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ - --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ - --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 -enum34==1.1.2 ; python_version < '3.4' \ - --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ - --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==2.6.1 \ + --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \ + --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \ + --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \ + --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \ + --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \ + --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \ + --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \ + --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \ + --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \ + --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \ + --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \ + --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \ + --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \ + --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \ + --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \ + --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \ + --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \ + --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \ + --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6 +enum34==1.1.6 \ + --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ + --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ + --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ + --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 funcsigs==1.0.2 \ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 -idna==2.5 \ - --hash=sha256:cc19709fd6d0cbfed39ea875d29ba6d4e22c0cebc510a76d6302a28385e8bb70 \ - --hash=sha256:3cb5ce08046c4e3a560fc02f138d0ac63e00f8ce5901a56b32ec8b7994082aab -ipaddress==1.0.16 \ - --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ - --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +future==0.17.1 \ + --hash=sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c +ipaddress==1.0.22 \ + --hash=sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794 \ + --hash=sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c josepy==1.1.0 \ --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 -linecache2==1.0.0 \ - --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ - --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 -ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ - --no-binary ordereddict -packaging==16.8 \ - --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ - --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e -parsedatetime==2.1 \ - --hash=sha256:ce9d422165cf6e963905cd5f74f274ebf7cc98c941916169178ef93f0e557838 \ - --hash=sha256:17c578775520c99131634e09cfca5a05ea9e1bd2a05cd06967ebece10df7af2d -pbr==1.8.1 \ - --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ - --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==18.0.0 \ - --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ - --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 -pyparsing==2.1.8 \ - --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ - --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ - --hash=sha256:ab09aee814c0241ff0c503cff30018219fe1fc14501d89f406f4664a0ec9fbcd \ - --hash=sha256:6e9a7f052f8e26bcf749e4033e3115b6dc7e3c85aafcb794b9a88c9d9ef13c97 \ - --hash=sha256:9f463a6bcc4eeb6c08f1ed84439b17818e2085937c0dee0d7674ac127c67c12b \ - --hash=sha256:3626b4d81cfb300dad57f52f2f791caaf7b06c09b368c0aa7b868e53a5775424 \ - --hash=sha256:367b90cc877b46af56d4580cd0ae278062903f02b8204ab631f5a2c0f50adfd0 \ - --hash=sha256:9f1ea360086cd68681e7f4ca8f1f38df47bf81942a0d76a9673c2d23eff35b13 -pyRFC3339==1.0 \ - --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ - --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb +parsedatetime==2.4 \ + --hash=sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b \ + --hash=sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094 +pbr==5.1.3 \ + --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \ + --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824 +pyOpenSSL==19.0.0 \ + --hash=sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200 \ + --hash=sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6 +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 +pyparsing==2.3.1 \ + --hash=sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a \ + --hash=sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ - --no-binary python-augeas -pytz==2015.7 \ - --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ - --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ - --hash=sha256:ead4aefa7007249e05e51b01095719d5a8dd95760089f5730aac5698b1932918 \ - --hash=sha256:3cca0df08bd0ed98432390494ce3ded003f5e661aa460be7a734bffe35983605 \ - --hash=sha256:3ede470d3d17ba3c07638dfa0d10452bc1b6e5ad326127a65ba77e6aaeb11bec \ - --hash=sha256:68c47964f7186eec306b13629627722b9079cd4447ed9e5ecaecd4eac84ca734 \ - --hash=sha256:dd5d3991950aae40a6c81de1578942e73d629808cefc51d12cd157980e6cfc18 \ - --hash=sha256:a77c52062c07eb7c7b30545dbc73e32995b7e117eea750317b5cb5c7a4618f14 \ - --hash=sha256:81af9aec4bc960a9a0127c488f18772dae4634689233f06f65443e7b11ebeb51 \ - --hash=sha256:e079b1dadc5c06246cc1bb6fe1b23a50b1d1173f2edd5104efd40bb73a28f406 \ - --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ - --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ - --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.20.0 \ - --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ - --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 -six==1.10.0 \ - --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ - --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a -traceback2==1.4.0 \ - --hash=sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23 \ - --hash=sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030 -unittest2==1.1.0 \ - --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ - --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 -zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ - --no-binary zope.component -zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ - --no-binary zope.event -zope.interface==4.1.3 \ - --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ - --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ - --hash=sha256:6788416f7ea7f5b8a97be94825377aa25e8bdc73463e07baaf9858b29e737077 \ - --hash=sha256:6f3230f7254518201e5a3708cbb2de98c848304f06e3ded8bfb39e5825cba2e1 \ - --hash=sha256:5fa575a5240f04200c3088427d0d4b7b737f6e9018818a51d8d0f927a6a2517a \ - --hash=sha256:522194ad6a545735edd75c8a83f48d65d1af064e432a7d320d64f56bafc12e99 \ - --hash=sha256:e8c7b2d40943f71c99148c97f66caa7f5134147f57423f8db5b4825099ce9a09 \ - --hash=sha256:279024f0208601c3caa907c53876e37ad88625f7eaf1cb3842dbe360b2287017 \ - --hash=sha256:2e221a9eec7ccc58889a278ea13dcfed5ef939d80b07819a9a8b3cb1c681484f \ - --hash=sha256:69118965410ec86d44dc6b9017ee3ddbd582e0c0abeef62b3a19dbf6c8ad132b \ - --hash=sha256:d04df8686ec864d0cade8cf199f7f83aecd416109a20834d568f8310ded12dea \ - --hash=sha256:e75a947e15ee97e7e71e02ea302feb2fc62d3a2bb4668bf9dfbed43a506ac7e7 \ - --hash=sha256:4e45d22fb883222a5ab9f282a116fec5ee2e8d1a568ccff6a2d75bbd0eb6bcfc \ - --hash=sha256:bce9339bb3c7a55e0803b63d21c5839e8e479bc85c4adf42ae415b72f94facb2 \ - --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ - --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ - --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -requests-toolbelt==0.8.0 \ - --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ - --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 -chardet==3.0.2 \ - --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ - --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2018.9 \ + --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \ + --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 urllib3==1.24.1 \ --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 -certifi==2017.4.17 \ - --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ - --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a +zope.component==4.5 \ + --hash=sha256:6edfd626c3b593b72895a8cfcf79bff41f4619194ce996a85bce31ac02b94e55 \ + --hash=sha256:984a06ba3def0b02b1117fa4c45b56e772e8c29c0340820fbf367e440a93a3a4 +zope.deferredimport==4.3 \ + --hash=sha256:2ddef5a7ecfff132a2dd796253366ecf9748a446e30f1a0b3a636aec9d9c05c5 \ + --hash=sha256:4aae9cbacb2146cca58e62be0a914f0cec034d3b2d41135ea212ca8a96f4b5ec +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.4 \ + --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \ + --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 +zope.hookable==4.2.0 \ + --hash=sha256:22886e421234e7e8cedc21202e1d0ab59960e40a47dd7240e9659a2d82c51370 \ + --hash=sha256:39912f446e45b4e1f1951b5ffa2d5c8b074d25727ec51855ae9eab5408f105ab \ + --hash=sha256:3adb7ea0871dbc56b78f62c4f5c024851fc74299f4f2a95f913025b076cde220 \ + --hash=sha256:3d7c4b96341c02553d8b8d71065a9366ef67e6c6feca714f269894646bb8268b \ + --hash=sha256:4e826a11a529ed0464ffcecf34b0b7bd1b4928dd5848c5c61bedd7833e8f4801 \ + --hash=sha256:700d68cc30728de1c4c62088a981c6daeaefdf20a0d81995d2c0b7f442c5f88c \ + --hash=sha256:77c82a430cedfbf508d1aa406b2f437363c24fa90c73f577ead0fb5295749b83 \ + --hash=sha256:c1df3929a3666fc5a0c80d60a0c1e6f6ef97c7f6ed2f1b7cf49f3e6f3d4dde15 \ + --hash=sha256:dba8b2dd2cd41cb5f37bfa3f3d82721b8ae10e492944e48ddd90a439227f2893 \ + --hash=sha256:f492540305b15b5591bd7195d61f28946bb071de071cee5d68b6b8414da90fd2 +zope.interface==4.6.0 \ + --hash=sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c \ + --hash=sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b \ + --hash=sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02 \ + --hash=sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f \ + --hash=sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5 \ + --hash=sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375 \ + --hash=sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487 \ + --hash=sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2 \ + --hash=sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0 \ + --hash=sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b \ + --hash=sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63 \ + --hash=sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39 \ + --hash=sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745 \ + --hash=sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc \ + --hash=sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2 \ + --hash=sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa \ + --hash=sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1 \ + --hash=sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc \ + --hash=sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98 \ + --hash=sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97 \ + --hash=sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab \ + --hash=sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127 \ + --hash=sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d \ + --hash=sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe \ + --hash=sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891 \ + --hash=sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1 \ + --hash=sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b \ + --hash=sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966 \ + --hash=sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317 +zope.proxy==4.3.1 \ + --hash=sha256:0cbcfcafaa3b5fde7ba7a7b9a2b5f09af25c9b90087ad65f9e61359fed0ca63b \ + --hash=sha256:3de631dd5054a3a20b9ebff0e375f39c0565f1fb9131200d589a6a8f379214cd \ + --hash=sha256:5429134d04d42262f4dac25f6dea907f6334e9a751ffc62cb1d40226fb52bdeb \ + --hash=sha256:563c2454b2d0f23bca54d2e0e4d781149b7b06cb5df67e253ca3620f37202dd2 \ + --hash=sha256:5bcf773345016b1461bb07f70c635b9386e5eaaa08e37d3939dcdf12d3fdbec5 \ + --hash=sha256:8d84b7aef38c693874e2f2084514522bf73fd720fde0ce2a9352a51315ffa475 \ + --hash=sha256:90de9473c05819b36816b6cb957097f809691836ed3142648bf62da84b4502fe \ + --hash=sha256:dd592a69fe872445542a6e1acbefb8e28cbe6b4007b8f5146da917e49b155cc3 \ + --hash=sha256:e7399ab865399fce322f9cefc6f2f3e4099d087ba581888a9fea1bbe1db42a08 \ + --hash=sha256:e7d1c280d86d72735a420610df592aac72332194e531a8beff43a592c3a1b8eb \ + --hash=sha256:e90243fee902adb0c39eceb3c69995c0f2004bc3fdb482fbf629efc656d124ed # Contains the requirements for the letsencrypt package. # @@ -1374,7 +1387,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade. command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] # Disable cache since it is not used and it otherwise sometimes throws permission warnings: command.extend(['--no-cache-dir'] if has_pip_cache else []) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 29c54bf2d..0d3312968 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -483,6 +483,14 @@ else: UNLIKELY_EOF } +# Create a new virtual environment for Certbot. It will overwrite any existing one. +# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE +CreateVenv() { + "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF" +{{ create_venv.py }} +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -538,22 +546,7 @@ if [ "$1" = "--le-auto-phase2" ]; then if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then say "Creating virtual environment..." DeterminePythonVersion - rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - # Use an environment variable instead of a flag for compatibility with old versions - if [ "$VERBOSE" = 1 ]; then - VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" \ - > /dev/null - fi - else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi - fi + CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE" if [ -n "$BOOTSTRAP_VERSION" ]; then echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" diff --git a/letsencrypt-auto-source/pieces/create_venv.py b/letsencrypt-auto-source/pieces/create_venv.py new file mode 100755 index 000000000..a618e228a --- /dev/null +++ b/letsencrypt-auto-source/pieces/create_venv.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +import os +import shutil +import subprocess +import sys + + +def create_venv(venv_path, pyver, verbose): + if os.path.exists(venv_path): + shutil.rmtree(venv_path) + + stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w') + + if int(pyver) <= 27: + # Use virtualenv binary + environ = os.environ.copy() + environ['VIRTUALENV_NO_DOWNLOAD'] = '1' + command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path] + subprocess.check_call(command, stdout=stdout, env=environ) + else: + # Use embedded venv module in Python 3 + command = [sys.executable, '-m', 'venv', venv_path] + subprocess.check_call(command, stdout=stdout) + + +if __name__ == '__main__': + create_venv(*sys.argv[1:]) diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index dff57dfd5..625ae45f1 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -1,196 +1,189 @@ -# This is the flattened list of packages certbot-auto installs. To generate -# this, do -# `pip install --no-cache-dir -e acme -e . -e certbot-apache -e certbot-nginx`, -# and then use `hashin` or a more secure method to gather the hashes. - -# Hashin example: -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# sets the new certbot-auto pinned version of cryptography to 1.5.2 - -argparse==1.4.0 \ - --hash=sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314 \ - --hash=sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4 - -# This comes before cffi because cffi will otherwise install an unchecked -# version via setup_requires. -pycparser==2.14 \ - --hash=sha256:7959b4a74abdc27b312fed1c21e6caf9309ce0b29ea86b591fd2e99ecdf27f73 \ - --no-binary pycparser - -asn1crypto==0.22.0 \ - --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ - --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.11.5 \ - --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ - --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ - --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ - --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ - --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ - --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ - --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ - --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ - --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ - --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ - --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ - --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ - --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ - --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ - --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ - --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ - --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ - --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ - --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ - --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ - --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ - --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ - --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ - --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ - --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ - --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ - --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ - --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ - --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ - --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ - --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ - --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 -ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ - --no-binary ConfigArgParse +# This is the flattened list of packages certbot-auto installs. +# To generate this, do (with docker and package hashin installed): +# ``` +# letsencrypt-auto-source/rebuild_dependencies.py \ +# letsencrypt-auto-sources/pieces/dependency-requirements.txt +# ``` +ConfigArgParse==0.14.0 \ + --hash=sha256:2e2efe2be3f90577aca9415e32cb629aa2ecd92078adbe27b53a03e53ff12e91 +asn1crypto==0.24.0 \ + --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ + --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae +cffi==1.12.2 \ + --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \ + --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \ + --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \ + --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \ + --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \ + --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \ + --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \ + --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \ + --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \ + --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \ + --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \ + --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \ + --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \ + --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \ + --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \ + --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \ + --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \ + --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \ + --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \ + --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \ + --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \ + --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \ + --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \ + --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \ + --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \ + --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \ + --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \ + --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ - --no-binary configobj -cryptography==2.5 \ - --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ - --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ - --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ - --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ - --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ - --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ - --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ - --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ - --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ - --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ - --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ - --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ - --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ - --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ - --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ - --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ - --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ - --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ - --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 -enum34==1.1.2 ; python_version < '3.4' \ - --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ - --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==2.6.1 \ + --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \ + --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \ + --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \ + --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \ + --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \ + --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \ + --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \ + --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \ + --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \ + --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \ + --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \ + --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \ + --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \ + --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \ + --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \ + --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \ + --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \ + --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \ + --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6 +enum34==1.1.6 \ + --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ + --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ + --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ + --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 funcsigs==1.0.2 \ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 -idna==2.5 \ - --hash=sha256:cc19709fd6d0cbfed39ea875d29ba6d4e22c0cebc510a76d6302a28385e8bb70 \ - --hash=sha256:3cb5ce08046c4e3a560fc02f138d0ac63e00f8ce5901a56b32ec8b7994082aab -ipaddress==1.0.16 \ - --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ - --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +future==0.17.1 \ + --hash=sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c +ipaddress==1.0.22 \ + --hash=sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794 \ + --hash=sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c josepy==1.1.0 \ --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \ --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086 -linecache2==1.0.0 \ - --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ - --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 -ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ - --no-binary ordereddict -packaging==16.8 \ - --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ - --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e -parsedatetime==2.1 \ - --hash=sha256:ce9d422165cf6e963905cd5f74f274ebf7cc98c941916169178ef93f0e557838 \ - --hash=sha256:17c578775520c99131634e09cfca5a05ea9e1bd2a05cd06967ebece10df7af2d -pbr==1.8.1 \ - --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ - --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==18.0.0 \ - --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ - --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 -pyparsing==2.1.8 \ - --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ - --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ - --hash=sha256:ab09aee814c0241ff0c503cff30018219fe1fc14501d89f406f4664a0ec9fbcd \ - --hash=sha256:6e9a7f052f8e26bcf749e4033e3115b6dc7e3c85aafcb794b9a88c9d9ef13c97 \ - --hash=sha256:9f463a6bcc4eeb6c08f1ed84439b17818e2085937c0dee0d7674ac127c67c12b \ - --hash=sha256:3626b4d81cfb300dad57f52f2f791caaf7b06c09b368c0aa7b868e53a5775424 \ - --hash=sha256:367b90cc877b46af56d4580cd0ae278062903f02b8204ab631f5a2c0f50adfd0 \ - --hash=sha256:9f1ea360086cd68681e7f4ca8f1f38df47bf81942a0d76a9673c2d23eff35b13 -pyRFC3339==1.0 \ - --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ - --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb +parsedatetime==2.4 \ + --hash=sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b \ + --hash=sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094 +pbr==5.1.3 \ + --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \ + --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824 +pyOpenSSL==19.0.0 \ + --hash=sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200 \ + --hash=sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6 +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 +pyparsing==2.3.1 \ + --hash=sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a \ + --hash=sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ - --no-binary python-augeas -pytz==2015.7 \ - --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ - --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ - --hash=sha256:ead4aefa7007249e05e51b01095719d5a8dd95760089f5730aac5698b1932918 \ - --hash=sha256:3cca0df08bd0ed98432390494ce3ded003f5e661aa460be7a734bffe35983605 \ - --hash=sha256:3ede470d3d17ba3c07638dfa0d10452bc1b6e5ad326127a65ba77e6aaeb11bec \ - --hash=sha256:68c47964f7186eec306b13629627722b9079cd4447ed9e5ecaecd4eac84ca734 \ - --hash=sha256:dd5d3991950aae40a6c81de1578942e73d629808cefc51d12cd157980e6cfc18 \ - --hash=sha256:a77c52062c07eb7c7b30545dbc73e32995b7e117eea750317b5cb5c7a4618f14 \ - --hash=sha256:81af9aec4bc960a9a0127c488f18772dae4634689233f06f65443e7b11ebeb51 \ - --hash=sha256:e079b1dadc5c06246cc1bb6fe1b23a50b1d1173f2edd5104efd40bb73a28f406 \ - --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ - --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ - --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 -requests==2.20.0 \ - --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \ - --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279 -six==1.10.0 \ - --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ - --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a -traceback2==1.4.0 \ - --hash=sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23 \ - --hash=sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030 -unittest2==1.1.0 \ - --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ - --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 -zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ - --no-binary zope.component -zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ - --no-binary zope.event -zope.interface==4.1.3 \ - --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ - --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ - --hash=sha256:6788416f7ea7f5b8a97be94825377aa25e8bdc73463e07baaf9858b29e737077 \ - --hash=sha256:6f3230f7254518201e5a3708cbb2de98c848304f06e3ded8bfb39e5825cba2e1 \ - --hash=sha256:5fa575a5240f04200c3088427d0d4b7b737f6e9018818a51d8d0f927a6a2517a \ - --hash=sha256:522194ad6a545735edd75c8a83f48d65d1af064e432a7d320d64f56bafc12e99 \ - --hash=sha256:e8c7b2d40943f71c99148c97f66caa7f5134147f57423f8db5b4825099ce9a09 \ - --hash=sha256:279024f0208601c3caa907c53876e37ad88625f7eaf1cb3842dbe360b2287017 \ - --hash=sha256:2e221a9eec7ccc58889a278ea13dcfed5ef939d80b07819a9a8b3cb1c681484f \ - --hash=sha256:69118965410ec86d44dc6b9017ee3ddbd582e0c0abeef62b3a19dbf6c8ad132b \ - --hash=sha256:d04df8686ec864d0cade8cf199f7f83aecd416109a20834d568f8310ded12dea \ - --hash=sha256:e75a947e15ee97e7e71e02ea302feb2fc62d3a2bb4668bf9dfbed43a506ac7e7 \ - --hash=sha256:4e45d22fb883222a5ab9f282a116fec5ee2e8d1a568ccff6a2d75bbd0eb6bcfc \ - --hash=sha256:bce9339bb3c7a55e0803b63d21c5839e8e479bc85c4adf42ae415b72f94facb2 \ - --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ - --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ - --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -requests-toolbelt==0.8.0 \ - --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ - --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 -chardet==3.0.2 \ - --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ - --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2018.9 \ + --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \ + --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 urllib3==1.24.1 \ --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 -certifi==2017.4.17 \ - --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ - --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a +zope.component==4.5 \ + --hash=sha256:6edfd626c3b593b72895a8cfcf79bff41f4619194ce996a85bce31ac02b94e55 \ + --hash=sha256:984a06ba3def0b02b1117fa4c45b56e772e8c29c0340820fbf367e440a93a3a4 +zope.deferredimport==4.3 \ + --hash=sha256:2ddef5a7ecfff132a2dd796253366ecf9748a446e30f1a0b3a636aec9d9c05c5 \ + --hash=sha256:4aae9cbacb2146cca58e62be0a914f0cec034d3b2d41135ea212ca8a96f4b5ec +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.4 \ + --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \ + --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 +zope.hookable==4.2.0 \ + --hash=sha256:22886e421234e7e8cedc21202e1d0ab59960e40a47dd7240e9659a2d82c51370 \ + --hash=sha256:39912f446e45b4e1f1951b5ffa2d5c8b074d25727ec51855ae9eab5408f105ab \ + --hash=sha256:3adb7ea0871dbc56b78f62c4f5c024851fc74299f4f2a95f913025b076cde220 \ + --hash=sha256:3d7c4b96341c02553d8b8d71065a9366ef67e6c6feca714f269894646bb8268b \ + --hash=sha256:4e826a11a529ed0464ffcecf34b0b7bd1b4928dd5848c5c61bedd7833e8f4801 \ + --hash=sha256:700d68cc30728de1c4c62088a981c6daeaefdf20a0d81995d2c0b7f442c5f88c \ + --hash=sha256:77c82a430cedfbf508d1aa406b2f437363c24fa90c73f577ead0fb5295749b83 \ + --hash=sha256:c1df3929a3666fc5a0c80d60a0c1e6f6ef97c7f6ed2f1b7cf49f3e6f3d4dde15 \ + --hash=sha256:dba8b2dd2cd41cb5f37bfa3f3d82721b8ae10e492944e48ddd90a439227f2893 \ + --hash=sha256:f492540305b15b5591bd7195d61f28946bb071de071cee5d68b6b8414da90fd2 +zope.interface==4.6.0 \ + --hash=sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c \ + --hash=sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b \ + --hash=sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02 \ + --hash=sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f \ + --hash=sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5 \ + --hash=sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375 \ + --hash=sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487 \ + --hash=sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2 \ + --hash=sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0 \ + --hash=sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b \ + --hash=sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63 \ + --hash=sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39 \ + --hash=sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745 \ + --hash=sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc \ + --hash=sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2 \ + --hash=sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa \ + --hash=sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1 \ + --hash=sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc \ + --hash=sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98 \ + --hash=sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97 \ + --hash=sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab \ + --hash=sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127 \ + --hash=sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d \ + --hash=sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe \ + --hash=sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891 \ + --hash=sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1 \ + --hash=sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b \ + --hash=sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966 \ + --hash=sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317 +zope.proxy==4.3.1 \ + --hash=sha256:0cbcfcafaa3b5fde7ba7a7b9a2b5f09af25c9b90087ad65f9e61359fed0ca63b \ + --hash=sha256:3de631dd5054a3a20b9ebff0e375f39c0565f1fb9131200d589a6a8f379214cd \ + --hash=sha256:5429134d04d42262f4dac25f6dea907f6334e9a751ffc62cb1d40226fb52bdeb \ + --hash=sha256:563c2454b2d0f23bca54d2e0e4d781149b7b06cb5df67e253ca3620f37202dd2 \ + --hash=sha256:5bcf773345016b1461bb07f70c635b9386e5eaaa08e37d3939dcdf12d3fdbec5 \ + --hash=sha256:8d84b7aef38c693874e2f2084514522bf73fd720fde0ce2a9352a51315ffa475 \ + --hash=sha256:90de9473c05819b36816b6cb957097f809691836ed3142648bf62da84b4502fe \ + --hash=sha256:dd592a69fe872445542a6e1acbefb8e28cbe6b4007b8f5146da917e49b155cc3 \ + --hash=sha256:e7399ab865399fce322f9cefc6f2f3e4099d087ba581888a9fea1bbe1db42a08 \ + --hash=sha256:e7d1c280d86d72735a420610df592aac72332194e531a8beff43a592c3a1b8eb \ + --hash=sha256:e90243fee902adb0c39eceb3c69995c0f2004bc3fdb482fbf629efc656d124ed diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index f04ebe6fe..346e23938 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -156,7 +156,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade. command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] # Disable cache since it is not used and it otherwise sometimes throws permission warnings: command.extend(['--no-cache-dir'] if has_pip_cache else []) diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/letsencrypt-auto-source/rebuild_dependencies.py new file mode 100755 index 000000000..aab7d546b --- /dev/null +++ b/letsencrypt-auto-source/rebuild_dependencies.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +""" +Gather and consolidate the up-to-date dependencies available and required to install certbot +on various Linux distributions. It generates a requirements file contained the pinned and hashed +versions, ready to be used by pip to install the certbot dependencies. + +This script is typically used to update the certbot-requirements.txt file of certbot-auto. + +To achieve its purpose, this script will start a certbot installation with unpinned dependencies, +then gather them, on various distributions started as Docker containers. + +Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt + +NB1: Docker must be installed on the machine running this script. +NB2: Python library 'hashin' must be installed on the machine running this script. +""" +from __future__ import print_function +import re +import shutil +import subprocess +import tempfile +import os +from os.path import dirname, abspath, join +import sys +import argparse + +# The list of docker distributions to test dependencies against with. +DISTRIBUTION_LIST = [ + 'ubuntu:18.04', 'ubuntu:14.04', + 'debian:stretch', 'debian:jessie', + 'centos:7', 'centos:6', + 'opensuse/leap:15', + 'fedora:29', +] + +# Theses constraints will be added while gathering dependencies on each distribution. +# It can be used because a particular version for a package is required for any reason, +# or to solve a version conflict between two distributions requirements. +AUTHORITATIVE_CONSTRAINTS = { + # Using an older version of mock here prevents regressions of #5276. + 'mock': '1.3.0', + # Too touchy to move to a new version. And will be removed soon + # in favor of pure python parser for Apache. + 'python-augeas': '0.5.0', +} + + +# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) +CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) + +# The script will be used to gather dependencies for a given distribution. +# - certbot-auto is used to install relevant OS packages, and set up an initial venv +# - then this venv is used to consistently construct an empty new venv +# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), +# without pinned dependencies, and respecting input authoritative requirements +# - `certbot plugins` is called to check we have an healthy environment +# - finally current set of dependencies is extracted out of the docker using pip freeze +SCRIPT = """\ +#!/bin/sh +set -e + +cd /tmp/certbot +letsencrypt-auto-source/letsencrypt-auto --install-only -n +PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + +/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 + +/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py +/tmp/venv/bin/pip install -e acme -e . -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt +/tmp/venv/bin/certbot plugins +/tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt +""" + + +def _read_from(file): + """Read all content of the file, and return it as a string.""" + with open(file, 'r') as file_h: + return file_h.read() + + +def _write_to(file, content): + """Write given string content to the file, overwriting its initial content.""" + with open(file, 'w') as file_h: + file_h.write(content) + + +def _requirements_from_one_distribution(distribution, verbose): + """ + Calculate the Certbot dependencies expressed for the given distribution, using the official + Docker for this distribution, and return the lines of the generated requirements file. + """ + print('===> Gathering dependencies for {0}.'.format(distribution)) + workspace = tempfile.mkdtemp() + script = join(workspace, 'script.sh') + authoritative_constraints = join(workspace, 'constraints.txt') + cid_file = join(workspace, 'cid') + + try: + _write_to(script, SCRIPT) + os.chmod(script, 0o755) + + _write_to(authoritative_constraints, '\n'.join( + ['{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items()])) + + command = ['docker', 'run', '--rm', '--cidfile', cid_file, + '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), + '-v', '{0}:/tmp/workspace'.format(workspace), + '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), + distribution, '/tmp/workspace/script.sh'] + sub_stdout = sys.stdout if verbose else subprocess.PIPE + sub_stderr = sys.stderr if verbose else subprocess.STDOUT + process = subprocess.Popen(command, stdout=sub_stdout, stderr=sub_stderr, universal_newlines=True) + stdoutdata, _ = process.communicate() + + if process.returncode: + if stdoutdata: + sys.stderr.write('Output was:\n{0}'.format(stdoutdata)) + raise RuntimeError('Error while gathering dependencies for {0}.'.format(distribution)) + + with open(join(workspace, 'requirements.txt'), 'r') as file_h: + return file_h.readlines() + finally: + if os.path.isfile(cid_file): + cid = _read_from(cid_file) + try: + subprocess.check_output(['docker', 'kill', cid], stderr=subprocess.PIPE) + except subprocess.CalledProcessError: + pass + shutil.rmtree(workspace) + + +def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution): + """ + Extract every requirement from the given requirements file, and merge it in the dependency map. + Merging here means that the map contain every encountered dependency, and the version used in + each distribution. + + Example: + # dependencies_map = { + # } + _parse_and_merge_requirements(['cryptography=='1.2','requests=='2.1.0'], dependencies_map, 'debian:stretch') + # dependencies_map = { + # 'cryptography': [('1.2', 'debian:stretch)], + # 'requests': [('2.1.0', 'debian:stretch')] + # } + _parse_and_merge_requirements(['requests=='2.4.0', 'mock==1.3'], dependencies_map, 'centos:7') + # dependencies_map = { + # 'cryptography': [('1.2', 'debian:stretch)], + # 'requests': [('2.1.0', 'debian:stretch'), ('2.4.0', 'centos:7')], + # 'mock': [('2.4.0', 'centos:7')] + # } + """ + for line in requirements_file_lines: + match = re.match(r'([^=]+)==([^=]+)', line.strip()) + if not line.startswith('-e') and match: + package, version = match.groups() + if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: + dependencies_map.setdefault(package, []).append((version, distribution)) + + +def _consolidate_and_validate_dependencies(dependency_map): + """ + Given the dependency map of all requirements found in all distributions for Certbot, + construct an array containing the unit requirements for Certbot to be used by pip, + and the version conflicts, if any, between several distributions for a package. + Return requirements and conflicts as a tuple. + """ + print('===> Consolidate and validate the dependency map.') + requirements = [] + conflicts = [] + for package, versions in dependency_map.items(): + reduced_versions = _reduce_versions(versions) + + if len(reduced_versions) > 1: + version_list = ['{0} ({1})'.format(version, ','.join(distributions)) + for version, distributions in reduced_versions.items()] + conflict = ('package {0} is declared with several versions: {1}' + .format(package, ', '.join(version_list))) + conflicts.append(conflict) + sys.stderr.write('ERROR: {0}\n'.format(conflict)) + else: + requirements.append((package, list(reduced_versions)[0])) + + requirements.sort(key=lambda x: x[0]) + return requirements, conflicts + + +def _reduce_versions(version_dist_tuples): + """ + Get an array of version/distribution tuples, + and reduce it to a map based on the version values. + + Example: [('1.2.0', 'debian:stretch'), ('1.4.0', 'ubuntu:18.04'), ('1.2.0', 'centos:6')] + => {'1.2.0': ['debiqn:stretch', 'centos:6'], '1.4.0': ['ubuntu:18.04']} + """ + version_dist_map = {} + for version, distribution in version_dist_tuples: + version_dist_map.setdefault(version, []).append(distribution) + + return version_dist_map + + +def _write_requirements(dest_file, requirements, conflicts): + """ + Given the list of requirements and conflicts, write a well-formatted requirements file, + whose requirements are hashed signed using hashin library. Conflicts are written at the end + of the generated file. + """ + print('===> Calculating hashes for the requirement file.') + + _write_to(dest_file, '''\ +# This is the flattened list of packages certbot-auto installs. +# To generate this, do (with docker and package hashin installed): +# ``` +# letsencrypt-auto-source/rebuild_dependencies.py \\ +# letsencrypt-auto-sources/pieces/dependency-requirements.txt +# ``` +''') + + for req in requirements: + subprocess.check_call(['hashin', '{0}=={1}'.format(req[0], req[1]), + '--requirements-file', dest_file]) + + if conflicts: + with open(dest_file, 'a') as file_h: + file_h.write('\n## ! SOME ERRORS OCCURRED ! ##\n') + file_h.write('\n'.join('# {0}'.format(conflict) for conflict in conflicts)) + file_h.write('\n') + + return _read_from(dest_file) + + +def _gather_dependencies(dest_file, verbose): + """ + Main method of this script. Given a destination file path, will write the file + containing the consolidated and hashed requirements for Certbot, validated + against several Linux distributions. + """ + dependencies_map = {} + + for distribution in DISTRIBUTION_LIST: + requirements_file_lines = _requirements_from_one_distribution(distribution, verbose) + _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution) + + requirements, conflicts = _consolidate_and_validate_dependencies(dependencies_map) + + return _write_requirements(dest_file, requirements, conflicts) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' + 'validated against several OS distributions using Docker.')) + parser.add_argument('requirements_path', + help='path for the generated requirements file') + parser.add_argument('--verbose', '-v', action='store_true', + help='verbose will display all output during docker execution') + + namespace = parser.parse_args() + + try: + subprocess.check_output(['hashin', '--version']) + except subprocess.CalledProcessError: + raise RuntimeError('Python library hashin is not installed in the current environment.') + + try: + subprocess.check_output(['docker', '--version'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + raise RuntimeError('Docker is not installed or accessible to current user.') + + file_content = _gather_dependencies(namespace.requirements_path, namespace.verbose) + + print(file_content) + print('===> Rebuilt requirement file is available on path {0}' + .format(abspath(namespace.requirements_path))) From 537bffbc23341e6e7973429ff26cc3cfa72c2ec4 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 25 Mar 2019 20:56:28 +0100 Subject: [PATCH 622/639] [Windows] Fix some unit tests (#6865) This PR is a part of the effort to remove the last broken unit tests in certbot codebase for Windows, as described in #6850. This PR fixes various unit tests on Windows, whose resolution was only to modify some logic in the tests, or minor changes in certbot codecase impacting Windows only (like handling correctly paths with DOS-style). * Correct several tests * Skip test definitively * Test to be reactivated with #6497 * Mock log system to avoid errors due to multiple calls to main in main_test * Simplify mock * Update cli_test.py * One test to be repaired when windows file permissions PR is merged --- certbot/plugins/manual_test.py | 4 ++-- certbot/tests/cli_test.py | 34 +++++++++++++++--------------- certbot/tests/lock_test.py | 12 +++++------ certbot/tests/main_test.py | 38 +++++++++++++++++----------------- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index b566f6340..13e79ce6e 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -106,10 +106,10 @@ class AuthenticatorTest(test_util.TempDirTestCase): achall.validation(achall.account_key) in args[0]) self.assertFalse(kwargs['wrap']) - @test_util.broken_on_windows def test_cleanup(self): self.config.manual_public_ip_logging_ok = True - self.config.manual_auth_hook = 'echo foo;' + self.config.manual_auth_hook = ('{0} -c "import sys; sys.stdout.write(\'foo\')"' + .format(sys.executable)) self.config.manual_cleanup_hook = '# cleanup' self.auth.perform(self.achalls) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index f6669e7be..60191730f 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -86,26 +86,26 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods return output.getvalue() - @test_util.broken_on_windows @mock.patch("certbot.cli.flag_default") def test_cli_ini_domains(self, mock_flag_default): - tmp_config = tempfile.NamedTemporaryFile() - # use a shim to get ConfigArgParse to pick up tmp_config - shim = ( - lambda v: copy.deepcopy(constants.CLI_DEFAULTS[v]) - if v != "config_files" - else [tmp_config.name] - ) - mock_flag_default.side_effect = shim + with tempfile.NamedTemporaryFile() as tmp_config: + tmp_config.close() # close now because of compatibility issues on Windows + # use a shim to get ConfigArgParse to pick up tmp_config + shim = ( + lambda v: copy.deepcopy(constants.CLI_DEFAULTS[v]) + if v != "config_files" + else [tmp_config.name] + ) + mock_flag_default.side_effect = shim - namespace = self.parse(["certonly"]) - self.assertEqual(namespace.domains, []) - tmp_config.write(b"domains = example.com") - tmp_config.flush() - namespace = self.parse(["certonly"]) - self.assertEqual(namespace.domains, ["example.com"]) - namespace = self.parse(["renew"]) - self.assertEqual(namespace.domains, []) + namespace = self.parse(["certonly"]) + self.assertEqual(namespace.domains, []) + with open(tmp_config.name, 'w') as file_h: + file_h.write("domains = example.com") + namespace = self.parse(["certonly"]) + self.assertEqual(namespace.domains, ["example.com"]) + namespace = self.parse(["renew"]) + self.assertEqual(namespace.domains, []) def test_no_args(self): namespace = self.parse([]) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 6379693ae..d2e61e386 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -41,7 +41,6 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') - @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -59,12 +58,14 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) - @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) - locked_repr = repr(lock_file) - self._test_repr_common(lock_file, locked_repr) - self.assertTrue('acquired' in locked_repr) + try: + locked_repr = repr(lock_file) + self._test_repr_common(lock_file, locked_repr) + self.assertTrue('acquired' in locked_repr) + finally: + lock_file.release() def test_released_repr(self): lock_file = self._call(self.lock_path) @@ -94,7 +95,6 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) - @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 786b91a94..e1be6e023 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -593,20 +593,20 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) - @test_util.broken_on_windows - def test_noninteractive(self): + @mock.patch('certbot.log.post_arg_parse_setup') + def test_noninteractive(self, _): args = ['-n', 'certonly'] self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - @test_util.broken_on_windows + @mock.patch('certbot.log.post_arg_parse_setup') @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @mock.patch('certbot.main._determine_account') @mock.patch('certbot.main.client.Client.obtain_and_enroll_certificate') @mock.patch('certbot.main._get_and_save_cert') - def test_user_agent(self, gsc, _obt, det, _client, unused_report): + def test_user_agent(self, gsc, _obt, det, _client, _, __): # Normally the client is totally mocked out, but here we need more # arguments to automate it... args = ["--standalone", "certonly", "-m", "none@none.com", @@ -655,11 +655,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) self.assertEqual(call_config.key_path, test_util.temp_join('privkey')) - @test_util.broken_on_windows + @mock.patch('certbot.log.post_arg_parse_setup') @mock.patch('certbot.main._install_cert') @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') - def test_installer_param_override(self, _inst, _rec, mock_install): + def test_installer_param_override(self, _inst, _rec, mock_install, _): mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), chain_path=test_util.temp_join('chain'), fullchain_path=test_util.temp_join('chain'), @@ -706,10 +706,10 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(mock_getcert.called) self.assertTrue(mock_inst.called) - @test_util.broken_on_windows + @mock.patch('certbot.log.post_arg_parse_setup') @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') - def test_configurator_selection(self, mock_exe_exists, unused_report): + def test_configurator_selection(self, mock_exe_exists, _, __): mock_exe_exists.return_value = True real_plugins = disco.PluginsRegistry.find_all() args = ['--apache', '--authenticator', 'standalone'] @@ -746,8 +746,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(["auth", "--standalone"]) self.assertEqual(1, mock_certonly.call_count) - @test_util.broken_on_windows - def test_rollback(self): + @mock.patch('certbot.log.post_arg_parse_setup') + def test_rollback(self, _): _, _, _, client = self._call(['rollback']) self.assertEqual(1, client.rollback.call_count) @@ -774,8 +774,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['delete']) self.assertEqual(1, mock_cert_manager.call_count) - @test_util.broken_on_windows - def test_plugins(self): + @mock.patch('certbot.log.post_arg_parse_setup') + def test_plugins(self, _): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( *(itertools.combinations(flags, r) @@ -1047,7 +1047,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met return mock_lineage, mock_get_utility, stdout @mock.patch('certbot.crypto_util.notAfter') - def test_certonly_renewal(self, unused_notafter): + def test_certonly_renewal(self, _): lineage, get_utility, _ = self._test_renewal_common(True, []) self.assertEqual(lineage.save_successor.call_count, 1) lineage.update_all_links_to.assert_called_once_with( @@ -1056,9 +1056,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('fullchain.pem' in cert_msg) self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) - @test_util.broken_on_windows + @mock.patch('certbot.log.logging.handlers.RotatingFileHandler.doRollover') @mock.patch('certbot.crypto_util.notAfter') - def test_certonly_renewal_triggers(self, unused_notafter): + def test_certonly_renewal_triggers(self, _, __): # --dry-run should force renewal _, get_utility, _ = self._test_renewal_common(False, ['--dry-run', '--keep'], log_out="simulating renewal") @@ -1125,8 +1125,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('No renewals were attempted.' in stdout.getvalue()) self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) - @test_util.broken_on_windows - def test_quiet_renew(self): + @mock.patch('certbot.log.post_arg_parse_setup') + def test_quiet_renew(self, _): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True) @@ -1381,8 +1381,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(['-c', test_util.vector_path('cli.ini')]) self.assertTrue(mocked_run.called) - @test_util.broken_on_windows - def test_register(self): + @mock.patch('certbot.log.post_arg_parse_setup') + def test_register(self, _): with mock.patch('certbot.main.client') as mocked_client: acc = mock.MagicMock() acc.id = "imaginary_account" From 4d2dfab4dda7e81e7a09246fd10d34b518001adf Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 25 Mar 2019 21:22:56 +0100 Subject: [PATCH 623/639] Simplify a branching that is not totally covered. (#6881) --- certbot-apache/certbot_apache/parser.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 148f052d0..b9f23f6cf 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -93,12 +93,7 @@ class ApacheParser(object): # Add new path to parser paths new_dir = os.path.dirname(inc_path) new_file = os.path.basename(inc_path) - if new_dir in self.existing_paths.keys(): - # Add to existing path - self.existing_paths[new_dir].append(new_file) - else: - # Create a new path - self.existing_paths[new_dir] = [new_file] + self.existing_paths.setdefault(new_dir, []).append(new_file) def add_mod(self, mod_name): """Shortcut for updating parser modules.""" From 20ed165699da7e4c082abb844cca662d0e045d85 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 25 Mar 2019 21:48:36 +0100 Subject: [PATCH 624/639] Remove unused code in apache (#6882) To fix one of the two uncovered lines in certbot-apache, given in #6880. Instead of adding a test to just increase the coverage, this fixes the uncovered line by removing the unused code. --- certbot-apache/certbot_apache/configurator.py | 17 ----------------- .../certbot_apache/tests/configurator_test.py | 9 --------- 2 files changed, 26 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index a3061119b..49591a141 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1076,23 +1076,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if "ssl_module" not in self.parser.modules: self.enable_mod("ssl", temp=temp) - def make_addrs_sni_ready(self, addrs): - """Checks to see if the server is ready for SNI challenges. - - :param addrs: Addresses to check SNI compatibility - :type addrs: :class:`~certbot_apache.obj.Addr` - - """ - # Version 2.4 and later are automatically SNI ready. - if self.version >= (2, 4): - return - - for addr in addrs: - if not self.is_name_vhost(addr): - logger.debug("Setting VirtualHost at %s to be a name " - "based virtual host", addr) - self.add_name_vhost(addr) - def make_vhost_ssl(self, nonssl_vhost): # pylint: disable=too-many-locals """Makes an ssl_vhost version of a nonssl_vhost. diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ee4833ebb..a8d5dd96e 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1336,15 +1336,6 @@ class MultipleVhostsTest(util.ApacheTest): return account_key, (achall1, achall2, achall3) - def test_make_addrs_sni_ready(self): - self.config.version = (2, 2) - self.config.make_addrs_sni_ready( - set([obj.Addr.fromstring("*:443"), obj.Addr.fromstring("*:80")])) - self.assertTrue(self.config.parser.find_dir( - "NameVirtualHost", "*:80", exclude=False)) - self.assertTrue(self.config.parser.find_dir( - "NameVirtualHost", "*:443", exclude=False)) - def test_aug_version(self): mock_match = mock.Mock(return_value=["something"]) self.config.aug.match = mock_match From 97d269ceb542168816fb6381927766dae05c7d43 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 25 Mar 2019 23:06:13 +0100 Subject: [PATCH 625/639] Raise explicitly an error (#6883) Explicit is better than implicit When calling raise without an argument, Python will raise the last error occured from the caller except block. This makes my PyCharm very sad however. So this PR makes the function handling the error raising explicitly the error received as an argument. --- certbot/plugins/standalone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 207eff0b6..804d5f8dc 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -263,4 +263,4 @@ def _handle_perform_error(error): if not should_retry: raise errors.PluginError(msg) else: - raise + raise error From 50607eb0ff4279f3395231f3c6f35586d21557f3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 25 Mar 2019 15:57:53 -0700 Subject: [PATCH 626/639] Document dropped tls-sni-01 support in plugins. (#6884) --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc34634ac..3617c0e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Support for TLS-SNI-01 has been removed from all official Certbot plugins. ### Fixed @@ -24,6 +24,8 @@ all Certbot components during releases for the time being, however, the only package with changes other than its version number was: * certbot +* certbot-apache +* certbot-nginx More details about these changes can be found on our GitHub repo. From a27bd28b39e5312ef77d5767a989fc4661e179fb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 26 Mar 2019 19:35:43 +0100 Subject: [PATCH 627/639] Configure jessie repos in LTS mode during Docker build (#6887) Currently, `tox -e le_auto_jessie` job fails. It breaks in particular the cron pipeline that test everything each night. The failure occurs while setting up the Jessie Docker container to run the tests for certbot-auto, when `apt-get update` is invoked, with this error: ``` W: Failed to fetch http://deb.debian.org/debian/dists/jessie-updates/main/binary-amd64/Packages 404 Not Found ``` Indeed, if there are `stretch-updates`, `buster-updates` and so on in the repository, there is no `jessie-updates`. I do not know exactly the logic of Debian here, but as `*-updates` folders store stable updates, a distribution moving to LTS support like Jessie has no stable updates anymore. I suppose `jessie-updates have been decommissioned recently, and the official Docker has not been updated yet to use the LTS configuration for repositories. This PR does that live in the Dockerfile, using official instructions from https://wiki.debian.org/LTS/Using, and fixes this specific job. An example of a successful job with this modification can be found here: https://travis-ci.com/certbot/certbot/jobs/187864341 --- letsencrypt-auto-source/Dockerfile.jessie | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/Dockerfile.jessie b/letsencrypt-auto-source/Dockerfile.jessie index 9ee37b763..528ab5c4a 100644 --- a/letsencrypt-auto-source/Dockerfile.jessie +++ b/letsencrypt-auto-source/Dockerfile.jessie @@ -7,7 +7,9 @@ FROM debian:jessie RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea # Install pip, sudo, and openssl: -RUN apt-get update && \ +RUN echo "deb http://deb.debian.org/debian/ jessie main" > /etc/apt/sources.list && \ + echo "deb http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list && \ + apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean # Use pipstrap to update to a stable and tested version of pip From 821bec69979ecd699c92206316b9e3514b6efd62 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 27 Mar 2019 01:46:32 +0100 Subject: [PATCH 628/639] Remove tls-sni related flags in cli. Add a deprecation warning instead. (#6853) This PR is a part of the tls-sni-01 removal plan described in #6849. This PR removes --tls-sni-01-port, --tls-sni-01-address and tls-sni-01/tls-sni options from --preferred-challenges. They are replace by deprecation warning, indicating that these options will be removed soon. This deprecation, instead of complete removal, is done to avoid certbot instances to hard fail if some automated scripts still use these flags for some users. Once this PR lands, we can remove completely theses flags in one or two release. * Remove tls-sni related flags in cli. Add a deprecation warning instead. * Adapt tests to cli and renewal towards tls-sni flags deprecation * Add https_port option. Make tls_sni_01_port show a deprecation warning, but silently modify https_port if set * Migrate last items * Fix lint * Update certbot/cli.py Co-Authored-By: adferrand * Ensure to remove all occurences of tls-sni-01 * Remove unused parameter * Revert modifications on cli-help.txt * Use logger.warning instead of sys.stderr * Update the logger warning message * Remove standalone_supported_challenges option. * Fix order of preferred-challenges * Remove supported_challenges property * Fix some tests * Fix lint * Fix tests * Add a changelog * Clean code, fix test * Update CI * Reload * No hard date for tls-sni removal * Remove useless cast to list * Update certbot/tests/renewal_test.py Co-Authored-By: adferrand * Add entry to the changelog * Add entry to the changelog --- CHANGELOG.md | 8 +++ certbot-nginx/certbot_nginx/configurator.py | 4 +- certbot-nginx/certbot_nginx/tests/util.py | 2 +- certbot/cli.py | 36 ++++++++--- certbot/configuration.py | 6 +- certbot/constants.py | 3 +- certbot/interfaces.py | 11 ++-- certbot/plugins/standalone.py | 60 +------------------ certbot/plugins/standalone_test.py | 51 +--------------- certbot/renewal.py | 6 +- certbot/tests/cli_test.py | 19 ++++-- certbot/tests/configuration_test.py | 8 +-- certbot/tests/renewal_test.py | 10 +++- .../testdata/sample-renewal-ancient.conf | 2 - certbot/tests/testdata/sample-renewal.conf | 2 - certbot/tests/util_test.py | 23 ++++--- certbot/util.py | 4 +- docs/cli-help.txt | 2 +- docs/using.rst | 3 - examples/cli.ini | 1 - tests/integration/_common.sh | 2 +- 21 files changed, 90 insertions(+), 173 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3617c0e70..9761ad3e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,18 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fedora 29+ is now supported by certbot-auto. Since Python 2.x is on a deprecation path in Fedora, certbot-auto will install and use Python 3.x on Fedora 29+. +* CLI flag `--http-port` has been added for Nginx plugin exclusively, and replaces + `--tls-sni-01-port`. It defines the HTTPS port the Nginx plugin will use while + setting up a new SSL vhost. By default the HTTPS port is 443. ### Changed * Support for TLS-SNI-01 has been removed from all official Certbot plugins. +* CLI flags `--tls-sni-01-port` and `--tls-sni-01-address` are now no-op, will + generate a deprecation warning if used, and will be removed soon. +* Options `tls-sni` and `tls-sni-01` in `--preferred-challenges` flag are now no-op, + will generate a deprecation warning if used, and will be removed soon. +* CLI flag `--standalone-supported-challenges` has been removed. ### Fixed diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 2357735f9..8b39ee664 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -295,7 +295,7 @@ class NginxConfigurator(common.Installer): if create_if_no_match: # result will not be [None] because it errors on failure vhosts = [self._vhost_from_duplicated_default(target_name, True, - str(self.config.tls_sni_01_port))] + str(self.config.https_port))] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -609,7 +609,7 @@ class NginxConfigurator(common.Installer): :type vhost: :class:`~certbot_nginx.obj.VirtualHost` """ - https_port = self.config.tls_sni_01_port + https_port = self.config.https_port ipv6info = self.ipv6_info(https_port) ipv6_block = [''] ipv4_block = [''] diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 93b0b4c7f..e33c2c16c 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -82,7 +82,7 @@ def get_nginx_configurator( in_progress_dir=os.path.join(backups, "IN_PROGRESS"), server="https://acme-server.org:443/new", http01_port=80, - tls_sni_01_port=5001, + https_port=5001, ), name="nginx", version=version) diff --git a/certbot/cli.py b/certbot/cli.py index 398124510..1472b0139 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1117,14 +1117,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "testing", "--no-verify-ssl", action="store_true", help=config_help("no_verify_ssl"), default=flag_default("no_verify_ssl")) - helpful.add( - ["testing", "standalone", "apache", "nginx"], "--tls-sni-01-port", type=int, - default=flag_default("tls_sni_01_port"), - help=config_help("tls_sni_01_port")) - helpful.add( - ["testing", "standalone"], "--tls-sni-01-address", - default=flag_default("tls_sni_01_address"), - help=config_help("tls_sni_01_address")) helpful.add( ["testing", "standalone", "manual"], "--http-01-port", type=int, dest="http01_port", @@ -1133,6 +1125,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis ["testing", "standalone"], "--http-01-address", dest="http01_address", default=flag_default("http01_address"), help=config_help("http01_address")) + helpful.add( + ["testing", "nginx"], "--https-port", type=int, + default=flag_default("https_port"), + help=config_help("https_port")) helpful.add( "testing", "--break-my-certs", action="store_true", default=flag_default("break_my_certs"), @@ -1193,7 +1189,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis action=_PrefChallAction, default=flag_default("pref_challs"), help='A sorted, comma delimited list of the preferred challenge to ' 'use during authorization with the most preferred challenge ' - 'listed first (Eg, "dns" or "tls-sni-01,http,dns"). ' + 'listed first (Eg, "dns" or "http,dns"). ' 'Not all plugins support all challenges. See ' 'https://certbot.eff.org/docs/using.html#plugins for details. ' 'ACME Challenges are versioned, but if you pick "http" rather ' @@ -1264,6 +1260,17 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) + # Deprecation of tls-sni-01 related cli flags + # TODO: remove theses flags completely in few releases + class _DeprecatedTLSSNIAction(util._ShowWarning): # pylint: disable=protected-access + def __call__(self, parser, namespace, values, option_string=None): + super(_DeprecatedTLSSNIAction, self).__call__(parser, namespace, values, option_string) + namespace.https_port = values + helpful.add( + ["testing", "standalone", "apache", "nginx"], "--tls-sni-01-port", + type=int, action=_DeprecatedTLSSNIAction, help=argparse.SUPPRESS) + helpful.add_deprecated_argument("--tls-sni-01-address", 1) + # Populate the command line parameters for new style enhancements enhancements.populate_cli(helpful.add) @@ -1556,6 +1563,15 @@ def parse_preferred_challenges(pref_challs): aliases = {"dns": "dns-01", "http": "http-01", "tls-sni": "tls-sni-01"} challs = [c.strip() for c in pref_challs] challs = [aliases.get(c, c) for c in challs] + + # Ignore tls-sni-01 from the list, and generates a deprecation warning + # TODO: remove this option completely in few releases + if "tls-sni-01" in challs: + logger.warning('TLS-SNI-01 support is deprecated. This value is being dropped from the ' + 'setting of --preferred-challenges and future versions of Certbot will ' + 'error if it is included.') + challs = [chall for chall in challs if chall != "tls-sni-01"] + unrecognized = ", ".join(name for name in challs if name not in challenges.Challenge.TYPES) if unrecognized: @@ -1563,11 +1579,13 @@ def parse_preferred_challenges(pref_challs): "Unrecognized challenges: {0}".format(unrecognized)) return challs + def _user_agent_comment_type(value): if "(" in value or ")" in value: raise argparse.ArgumentTypeError("may not contain parentheses") return value + class _DeployHookAction(argparse.Action): """Action class for parsing deploy hooks.""" diff --git a/certbot/configuration.py b/certbot/configuration.py index 15f9fa3e0..5d248f39e 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -148,10 +148,10 @@ def check_config_sanity(config): """ # Port check - if config.http01_port == config.tls_sni_01_port: + if config.http01_port == config.https_port: raise errors.ConfigurationError( - "Trying to run http-01 and tls-sni-01 " - "on the same port ({0})".format(config.tls_sni_01_port)) + "Trying to run http-01 and https-port " + "on the same port ({0})".format(config.https_port)) # Domain checks if config.namespace.domains is not None: diff --git a/certbot/constants.py b/certbot/constants.py index c6a80747e..31b243518 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -50,10 +50,9 @@ CLI_DEFAULTS = dict( debug=False, debug_challenges=False, no_verify_ssl=False, - tls_sni_01_port=challenges.TLSSNI01Response.PORT, - tls_sni_01_address="", http01_port=challenges.HTTP01Response.PORT, http01_address="", + https_port=443, break_my_certs=False, rsa_key_size=2048, must_staple=False, diff --git a/certbot/interfaces.py b/certbot/interfaces.py index bd91d2272..96155fa88 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -220,12 +220,6 @@ class IConfig(zope.interface.Interface): no_verify_ssl = zope.interface.Attribute( "Disable verification of the ACME server's certificate.") - tls_sni_01_port = zope.interface.Attribute( - "Port used during tls-sni-01 challenge. " - "This only affects the port Certbot listens on. " - "A conforming ACME server will still attempt to connect on port 443.") - tls_sni_01_address = zope.interface.Attribute( - "The address the server listens to during tls-sni-01 challenge.") http01_port = zope.interface.Attribute( "Port used in the http-01 challenge. " @@ -235,6 +229,11 @@ class IConfig(zope.interface.Interface): http01_address = zope.interface.Attribute( "The address the server listens to during http-01 challenge.") + https_port = zope.interface.Attribute( + "Port used to serve HTTPS. " + "This affects which port Nginx will listen on after a LE certificate " + "is installed.") + pref_challs = zope.interface.Attribute( "Sorted user specified preferred challenges" "type strings with the most preferred challenge listed first") diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 804d5f8dc..9723116c1 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -1,5 +1,4 @@ """Standalone Authenticator.""" -import argparse import collections import logging import socket @@ -108,52 +107,6 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.HTTP01] \ -# type: List[Type[challenges.KeyAuthorizationChallenge]] - - -class SupportedChallengesAction(argparse.Action): - """Action class for parsing standalone_supported_challenges.""" - - def __call__(self, parser, namespace, values, option_string=None): - logger.warning( - "The standalone specific supported challenges flag is " - "deprecated. Please use the --preferred-challenges flag " - "instead.") - converted_values = self._convert_and_validate(values) - namespace.standalone_supported_challenges = converted_values - - def _convert_and_validate(self, data): - """Validate the value of supported challenges provided by the user. - - :param str data: comma delimited list of challenge types - - :returns: validated and converted list of challenge types - :rtype: str - - """ - challs = data.split(",") - unrecognized = [name for name in challs - if name not in challenges.Challenge.TYPES] - - # argparse.ArgumentErrors raised out of argparse.Action objects - # are caught by argparse which prints usage information and the - # error that occurred before calling sys.exit. - if unrecognized: - raise argparse.ArgumentError( - self, - "Unrecognized challenges: {0}".format(", ".join(unrecognized))) - - choices = set(chall.typ for chall in SUPPORTED_CHALLENGES) - if not set(challs).issubset(choices): - raise argparse.ArgumentError( - self, - "Plugin does not support the following (valid) " - "challenges: {0}".format(", ".join(set(challs) - choices))) - - return data - - @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): @@ -184,16 +137,7 @@ class Authenticator(common.Plugin): @classmethod def add_parser_arguments(cls, add): - add("supported-challenges", - help=argparse.SUPPRESS, - action=SupportedChallengesAction, - default=",".join(chall.typ for chall in SUPPORTED_CHALLENGES)) - - @property - def supported_challenges(self): - """Challenges supported by this plugin.""" - return [challenges.Challenge.TYPES[name] for name in - self.conf("supported-challenges").split(",")] + pass # No additional argument for the standalone plugin parser def more_info(self): # pylint: disable=missing-docstring return("This authenticator creates its own ephemeral TCP listener " @@ -206,7 +150,7 @@ class Authenticator(common.Plugin): def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - return self.supported_challenges + return [challenges.HTTP01] def perform(self, achalls): # pylint: disable=missing-docstring return [self._try_perform_single(achall) for achall in achalls] diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index db4dbeb3b..b2a7c32d4 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -1,5 +1,4 @@ """Tests for certbot.plugins.standalone.""" -import argparse import socket import unittest # https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi @@ -73,38 +72,6 @@ class ServerManagerTest(unittest.TestCase): maybe_another_server.close() -class SupportedChallengesActionTest(unittest.TestCase): - """Tests for plugins.standalone.SupportedChallengesAction.""" - - def _call(self, value): - with mock.patch("certbot.plugins.standalone.logger") as mock_logger: - # stderr is mocked to prevent potential argparse error - # output from cluttering test output - with mock.patch("sys.stderr"): - config = self.parser.parse_args([self.flag, value]) - - self.assertTrue(mock_logger.warning.called) - return getattr(config, self.dest) - - def setUp(self): - self.flag = "--standalone-supported-challenges" - self.dest = self.flag[2:].replace("-", "_") - self.parser = argparse.ArgumentParser() - - from certbot.plugins.standalone import SupportedChallengesAction - self.parser.add_argument(self.flag, action=SupportedChallengesAction) - - def test_correct(self): - self.assertEqual("http-01", self._call("http-01")) - - def test_unrecognized(self): - assert "foo" not in challenges.Challenge.TYPES - self.assertRaises(SystemExit, self._call, "foo") - - def test_not_subset(self): - self.assertRaises(SystemExit, self._call, "dns") - - def get_open_port(): """Gets an open port number from the OS.""" open_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) @@ -120,21 +87,10 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.standalone import Authenticator - self.config = mock.MagicMock( - http01_port=get_open_port(), - standalone_supported_challenges="http-01") + self.config = mock.MagicMock(http01_port=get_open_port()) self.auth = Authenticator(self.config, name="standalone") self.auth.servers = mock.MagicMock() - def test_supported_challenges(self): - self.assertEqual(self.auth.supported_challenges, - [challenges.HTTP01]) - - def test_supported_challenges_configured(self): - self.config.standalone_supported_challenges = "http-01" - self.assertEqual(self.auth.supported_challenges, - [challenges.HTTP01]) - def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) @@ -142,11 +98,6 @@ class AuthenticatorTest(unittest.TestCase): self.assertEqual(self.auth.get_chall_pref(domain=None), [challenges.HTTP01]) - def test_get_chall_pref_configured(self): - self.config.standalone_supported_challenges = "http-01" - self.assertEqual(self.auth.get_chall_pref(domain=None), - [challenges.HTTP01]) - def test_perform(self): achalls = self._get_achalls() response = self.auth.perform(achalls) diff --git a/certbot/renewal.py b/certbot/renewal.py index 4c592b27f..9da0ec596 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -35,10 +35,8 @@ logger = logging.getLogger(__name__) # the renewal configuration process loses this information. STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "server", "account", "authenticator", "installer", - "standalone_supported_challenges", "renew_hook", - "pre_hook", "post_hook", "tls_sni_01_address", - "http01_address"] -INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] + "renew_hook", "pre_hook", "post_hook", "http01_address"] +INT_CONFIG_ITEMS = ["rsa_key_size", "http01_port"] BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key", "autorenew"] diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 60191730f..ebba0b0c0 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -235,13 +235,19 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(namespace.domains, ['example.com', 'another.net']) def test_preferred_challenges(self): - short_args = ['--preferred-challenges', 'http, tls-sni-01, dns'] + short_args = ['--preferred-challenges', 'http, dns'] namespace = self.parse(short_args) - expected = [challenges.HTTP01.typ, - challenges.TLSSNI01.typ, challenges.DNS01.typ] + expected = [challenges.HTTP01.typ, challenges.DNS01.typ] self.assertEqual(namespace.pref_challs, expected) + # TODO: to be removed once tls-sni deprecation logic is removed + with mock.patch('certbot.cli.logger.warning') as mock_warn: + self.assertEqual(self.parse(['--preferred-challenges', 'http, tls-sni']).pref_challs, + [challenges.HTTP01.typ]) + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue('deprecated' in mock_warn.call_args[0][0]) + short_args = ['--preferred-challenges', 'jumping-over-the-moon'] # argparse.ArgumentError makes argparse print more information # to stderr and call sys.exit() @@ -260,12 +266,13 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_no_gui(self): args = ['renew', '--dialog'] - stderr = six.StringIO() - with mock.patch('certbot.main.sys.stderr', new=stderr): + with mock.patch("certbot.util.logger.warning") as mock_warn: namespace = self.parse(args) self.assertTrue(namespace.noninteractive_mode) - self.assertTrue("--dialog is deprecated" in stderr.getvalue()) + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertEqual("--dialog", mock_warn.call_args[0][1]) def _check_server_conflict_message(self, parser_args, conflicting_args): try: diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 10d9059b3..ce3cded20 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -17,11 +17,11 @@ class NamespaceConfigTest(test_util.ConfigTestCase): super(NamespaceConfigTest, self).setUp() self.config.foo = 'bar' self.config.server = 'https://acme-server.org:443/new' - self.config.tls_sni_01_port = 1234 + self.config.https_port = 1234 self.config.http01_port = 4321 def test_init_same_ports(self): - self.config.namespace.tls_sni_01_port = 4321 + self.config.namespace.https_port = 4321 from certbot.configuration import NamespaceConfig self.assertRaises(errors.Error, NamespaceConfig, self.config.namespace) @@ -79,7 +79,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_namespace = mock.MagicMock(spec=['config_dir', 'work_dir', 'logs_dir', 'http01_port', - 'tls_sni_01_port', + 'https_port', 'domains', 'server']) mock_namespace.config_dir = config_base mock_namespace.work_dir = work_base @@ -126,7 +126,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_namespace = mock.MagicMock(spec=['config_dir', 'work_dir', 'logs_dir', 'http01_port', - 'tls_sni_01_port', + 'https_port', 'domains', 'server']) mock_namespace.config_dir = config_base mock_namespace.work_dir = work_base diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index d292d909a..94e321f0d 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -58,11 +58,15 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): @mock.patch('certbot.renewal.cli.set_by_cli') def test_pref_challs_list(self, mock_set_by_cli): mock_set_by_cli.return_value = False + # TODO: remove tls-sni and related assertions to logger.warning call once + # the deprecation logic has been removed renewalparams = {'pref_challs': 'tls-sni, http-01, dns'.split(',')} - self._call(self.config, renewalparams) - expected = [challenges.TLSSNI01.typ, - challenges.HTTP01.typ, challenges.DNS01.typ] + with mock.patch('certbot.renewal.cli.logger.warning') as mock_warn: + self._call(self.config, renewalparams) + expected = [challenges.HTTP01.typ, challenges.DNS01.typ] self.assertEqual(self.config.pref_challs, expected) + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue('deprecated' in mock_warn.call_args[0][0]) @mock.patch('certbot.renewal.cli.set_by_cli') def test_pref_challs_str(self, mock_set_by_cli): diff --git a/certbot/tests/testdata/sample-renewal-ancient.conf b/certbot/tests/testdata/sample-renewal-ancient.conf index 333bcaa18..9586d5492 100644 --- a/certbot/tests/testdata/sample-renewal-ancient.conf +++ b/certbot/tests/testdata/sample-renewal-ancient.conf @@ -62,14 +62,12 @@ break_my_certs = False standalone = True manual = False server = https://acme-staging.api.letsencrypt.org/directory -standalone_supported_challenges = "tls-sni-01,http-01" webroot = True os_packages_only = False apache_init_script = None user_agent = None apache_le_vhost_ext = -le-ssl.conf debug = False -tls_sni_01_port = 443 logs_dir = /var/log/letsencrypt apache_vhost_root = /etc/apache2/sites-available configurator = None diff --git a/certbot/tests/testdata/sample-renewal.conf b/certbot/tests/testdata/sample-renewal.conf index 04f9ae8ca..936c5c0e0 100644 --- a/certbot/tests/testdata/sample-renewal.conf +++ b/certbot/tests/testdata/sample-renewal.conf @@ -62,14 +62,12 @@ break_my_certs = False standalone = True manual = False server = https://acme-staging-v02.api.letsencrypt.org/directory -standalone_supported_challenges = "tls-sni-01,http-01" webroot = False os_packages_only = False apache_init_script = None user_agent = None apache_le_vhost_ext = -le-ssl.conf debug = False -tls_sni_01_port = 443 logs_dir = /var/log/letsencrypt apache_vhost_root = /etc/apache2/sites-available configurator = None diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index cac587995..b848dbb9f 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -351,29 +351,28 @@ class AddDeprecatedArgumentTest(unittest.TestCase): def _call(self, argument_name, nargs): from certbot.util import add_deprecated_argument - add_deprecated_argument(self.parser.add_argument, argument_name, nargs) def test_warning_no_arg(self): self._call("--old-option", 0) - stderr = self._get_argparse_warnings(["--old-option"]) - self.assertTrue("--old-option is deprecated" in stderr) + with mock.patch("certbot.util.logger.warning") as mock_warn: + self.parser.parse_args(["--old-option"]) + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_warning_with_arg(self): self._call("--old-option", 1) - stderr = self._get_argparse_warnings(["--old-option", "42"]) - self.assertTrue("--old-option is deprecated" in stderr) - - def _get_argparse_warnings(self, args): - stderr = six.StringIO() - with mock.patch("certbot.util.sys.stderr", new=stderr): - self.parser.parse_args(args) - return stderr.getvalue() + with mock.patch("certbot.util.logger.warning") as mock_warn: + self.parser.parse_args(["--old-option", "42"]) + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_help(self): self._call("--old-option", 2) stdout = six.StringIO() - with mock.patch("certbot.util.sys.stdout", new=stdout): + with mock.patch("sys.stdout", new=stdout): try: self.parser.parse_args(["-h"]) except SystemExit: diff --git a/certbot/util.py b/certbot/util.py index 097593e9d..6481e30d2 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -13,7 +13,6 @@ import re import six import socket import subprocess -import sys from collections import OrderedDict @@ -477,8 +476,7 @@ def safe_email(email): class _ShowWarning(argparse.Action): """Action to log a warning when an argument is used.""" def __call__(self, unused1, unused2, unused3, option_string=None): - sys.stderr.write( - "Use of {0} is deprecated.\n".format(option_string)) + logger.warning("Use of %s is deprecated.", option_string) def add_deprecated_argument(add_argument, argument_name, nargs): diff --git a/docs/cli-help.txt b/docs/cli-help.txt index ed7794c3b..d883319f4 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -1,4 +1,4 @@ -usage: +usage: certbot [SUBCOMMAND] [options] [-d DOMAIN] [-d DOMAIN] ... Certbot can obtain and install HTTPS/TLS/SSL certificates. By default, diff --git a/docs/using.rst b/docs/using.rst index de17742a1..d2799f7dc 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -169,9 +169,6 @@ the bound IPv6 port and the failure during the second bind is expected. Use ``---address`` to explicitly tell Certbot which interface (and protocol) to bind. -.. note:: The ``--standalone-supported-challenges`` option has been - deprecated since ``certbot`` version 0.9.0. - .. _dns_plugins: DNS Plugins diff --git a/examples/cli.ini b/examples/cli.ini index dbaa9c599..4215fda5b 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -15,7 +15,6 @@ rsa-key-size = 4096 # Uncomment to use the standalone authenticator on port 443 # authenticator = standalone -# standalone-supported-challenges = tls-sni-01 # Uncomment to use the webroot authenticator. Replace webroot-path with the # path to the public_html / webroot folder being served by your web server. diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index bccba0891..a0cf3d1b4 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -61,7 +61,7 @@ certbot_test_no_force_renew () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --http-01-port $http_01_port \ - --tls-sni-01-port $https_port \ + --https-port $https_port \ --manual-public-ip-logging-ok \ $other_flags \ --non-interactive \ From a03e7b95d3adf029f91aa48b7903f765b6cb936f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 27 Mar 2019 02:26:38 +0100 Subject: [PATCH 629/639] Deprecate all tls-sni related objects in acme module (#6859) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is a part of the tls-sni-01 removal plan described in #6849. As `acme` is a library, we need to put some efforts to make a decent deprecation path before totally removing tls-sni in it. While initialization of `acme.challenges.TLSSNI01` was already creating deprecation warning, not all cases were covered. For instance, and innocent call like this ... ```python if not isinstance(challenge, acme.challenges.TLSSNI01): print('I am not using this TLS-SNI deprecated stuff, what could possibly go wrong?') ``` ... would break if we suddenly remove all objects related to this challenge. So, I use the _Deprecator Warning Machine, Let's Pacify this Technical Debt_ (Guido ®), to make `acme.challenges` and `acme.standalone` patch themselves, and display a deprecation warning on stderr for any access to the tls-sni challenge objects. No dev should be able to avoid the deprecation warning. I set the deprecation warning in the idea to remove the code on `0.34.0`, but the exact deprecation window is open to discussion of course. * Modules challenges and standalone patch themselves to generated deprecation warning when tls-sni related objects are accessed. * Correct unit tests * Correct lint * Update challenges_test.py * Correct lint * Fix an error during tests * Update coverage * Use multiprocessing for coverage * Add coverage * Update test_util.py * Factor the logic about global deprecation warning when accessing TLS-SNI-01 attributes * Fix coverage * Add comment for cryptography example. * Use warnings. * Add a changelog * Fix deprecation during tests * Reload * Update acme/acme/__init__.py Co-Authored-By: adferrand * Update CHANGELOG.md * Pick a random free port. --- CHANGELOG.md | 2 ++ acme/acme/__init__.py | 28 ++++++++++++++++++++ acme/acme/challenges.py | 9 ++++--- acme/acme/challenges_test.py | 28 ++++++++++---------- acme/acme/crypto_util.py | 13 ++++------ acme/acme/messages.py | 2 +- acme/acme/standalone.py | 7 ++++- acme/acme/standalone_test.py | 50 +++++++++++++++++++++--------------- acme/acme/test_util.py | 9 ------- pytest.ini | 2 +- 10 files changed, 91 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9761ad3e9..2cc823d1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * Support for TLS-SNI-01 has been removed from all official Certbot plugins. +* Attributes related to the TLS-SNI-01 challenge in `acme.challenges` and `acme.standalone` + modules are deprecated and will be removed soon. * CLI flags `--tls-sni-01-port` and `--tls-sni-01-address` are now no-op, will generate a deprecation warning if used, and will be removed soon. * Options `tls-sni` and `tls-sni-01` in `--preferred-challenges` flag are now no-op, diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index d91072a3b..20c008d64 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -6,6 +6,7 @@ This module is an implementation of the `ACME protocol`_. """ import sys +import warnings # This code exists to keep backwards compatibility with people using acme.jose # before it became the standalone josepy package. @@ -20,3 +21,30 @@ for mod in list(sys.modules): # preserved (acme.jose.* is josepy.*) if mod == 'josepy' or mod.startswith('josepy.'): sys.modules['acme.' + mod.replace('josepy', 'jose', 1)] = sys.modules[mod] + + +# This class takes a similar approach to the cryptography project to deprecate attributes +# in public modules. See the _ModuleWithDeprecation class here: +# https://github.com/pyca/cryptography/blob/91105952739442a74582d3e62b3d2111365b0dc7/src/cryptography/utils.py#L129 +class _TLSSNI01DeprecationModule(object): + """ + Internal class delegating to a module, and displaying warnings when + attributes related to TLS-SNI-01 are accessed. + """ + def __init__(self, module): + self.__dict__['_module'] = module + + def __getattr__(self, attr): + if 'TLSSNI01' in attr: + warnings.warn('{0} attribute is deprecated, and will be removed soon.'.format(attr), + DeprecationWarning, stacklevel=2) + return getattr(self._module, attr) + + def __setattr__(self, attr, value): # pragma: no cover + setattr(self._module, attr, value) + + def __delattr__(self, attr): # pragma: no cover + delattr(self._module, attr) + + def __dir__(self): # pragma: no cover + return ['_module'] + dir(self._module) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 6f2b3757b..36e7ab41c 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -4,7 +4,7 @@ import functools import hashlib import logging import socket -import warnings +import sys from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose @@ -15,6 +15,7 @@ import six from acme import errors from acme import crypto_util from acme import fields +from acme import _TLSSNI01DeprecationModule logger = logging.getLogger(__name__) @@ -515,8 +516,6 @@ class TLSSNI01(KeyAuthorizationChallenge): #n = jose.Field("n", encoder=int, decoder=int) def __init__(self, *args, **kwargs): - warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.", - DeprecationWarning, stacklevel=2) super(TLSSNI01, self).__init__(*args, **kwargs) def validation(self, account_key, **kwargs): @@ -641,3 +640,7 @@ class DNSResponse(ChallengeResponse): """ return chall.check_validation(self.validation, account_public_key) + + +# Patching ourselves to warn about TLS-SNI challenge deprecation and removal. +sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__]) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 4b905c1e5..edfaa3423 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,6 +1,5 @@ """Tests for acme.challenges.""" import unittest -import warnings import josepy as jose import mock @@ -374,25 +373,16 @@ class TLSSNI01Test(unittest.TestCase): 'type': 'tls-sni-01', 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', } - - def _msg(self): from acme.challenges import TLSSNI01 - with warnings.catch_warnings(record=True) as warn: - warnings.simplefilter("always") - msg = TLSSNI01( - token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) - assert warn is not None # using a raw assert for mypy - self.assertTrue(len(warn) == 1) - self.assertTrue(issubclass(warn[-1].category, DeprecationWarning)) - self.assertTrue('deprecated' in str(warn[-1].message)) - return msg + self.msg = TLSSNI01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) def test_to_partial_json(self): - self.assertEqual(self.jmsg, self._msg().to_partial_json()) + self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): from acme.challenges import TLSSNI01 - self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg)) + self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSSNI01 @@ -407,10 +397,18 @@ class TLSSNI01Test(unittest.TestCase): @mock.patch('acme.challenges.TLSSNI01Response.gen_cert') def test_validation(self, mock_gen_cert): mock_gen_cert.return_value = ('cert', 'key') - self.assertEqual(('cert', 'key'), self._msg().validation( + self.assertEqual(('cert', 'key'), self.msg.validation( KEY, cert_key=mock.sentinel.cert_key)) mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + def test_deprecation_message(self): + with mock.patch('acme.warnings.warn') as mock_warn: + from acme.challenges import TLSSNI01 + assert TLSSNI01 + self.assertEqual(mock_warn.call_count, 1) + self.assertTrue('deprecated' in mock_warn.call_args[0][0]) + + class TLSALPN01ResponseTest(unittest.TestCase): # pylint: disable=too-many-instance-attributes diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index c88cab943..c47e88e73 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -18,17 +18,14 @@ from acme.magic_typing import Callable, Union, Tuple, Optional logger = logging.getLogger(__name__) -# TLSSNI01 certificate serving and probing is not affected by SSL -# vulnerabilities: prober needs to check certificate for expected -# contents anyway. Working SNI is the only thing that's necessary for -# the challenge and thus scoping down SSL/TLS method (version) would -# cause interoperability issues: TLSv1_METHOD is only compatible with +# Default SSL method selected here is the most compatible, while secure +# SSL method: TLSv1_METHOD is only compatible with # TLSv1_METHOD, while SSLv23_METHOD is compatible with all other # methods, including TLSv2_METHOD (read more at # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni # should be changed to use "set_options" to disable SSLv2 and SSLv3, # in case it's used for things other than probing/serving! -_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore +_DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore class SSLSocket(object): # pylint: disable=too-few-public-methods @@ -40,7 +37,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar method: See `OpenSSL.SSL.Context` for allowed values. """ - def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): + def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD): self.sock = sock self.certs = certs self.method = method @@ -112,7 +109,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): + method=_DEFAULT_SSL_METHOD, source_address=('', 0)): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 7c82c8507..d8684603c 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -3,7 +3,7 @@ import six import json try: from collections.abc import Hashable # pylint: disable=no-name-in-module -except ImportError: +except ImportError: # pragma: no cover from collections import Hashable import josepy as jose diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ff9159933..c82967897 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -17,6 +17,7 @@ import OpenSSL from acme import challenges from acme import crypto_util from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme import _TLSSNI01DeprecationModule logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ class TLSServer(socketserver.TCPServer): self.certs = kwargs.pop("certs", {}) self.method = kwargs.pop( # pylint: disable=protected-access - "method", crypto_util._DEFAULT_TLSSNI01_SSL_METHOD) + "method", crypto_util._DEFAULT_SSL_METHOD) self.allow_reuse_address = kwargs.pop("allow_reuse_address", True) socketserver.TCPServer.__init__(self, *args, **kwargs) @@ -296,5 +297,9 @@ def simple_tls_sni_01_server(cli_args, forever=True): server.handle_request() +# Patching ourselves to warn about TLS-SNI challenge deprecation and removal. +sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__]) + + if __name__ == "__main__": sys.exit(simple_tls_sni_01_server(sys.argv)) # pragma: no cover diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index ee527782a..953df40d4 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -1,13 +1,15 @@ """Tests for acme.standalone.""" +import multiprocessing import os import shutil import socket import threading import tempfile import unittest +import time +from contextlib import closing from six.moves import http_client # pylint: disable=import-error -from six.moves import queue # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error import josepy as jose @@ -16,6 +18,7 @@ import requests from acme import challenges from acme import crypto_util +from acme import errors from acme import test_util from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module @@ -248,7 +251,6 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) -@test_util.broken_on_windows class TestSimpleTLSSNI01Server(unittest.TestCase): """Tests for acme.standalone.simple_tls_sni_01_server.""" @@ -263,35 +265,41 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): shutil.copy(test_util.vector_path('rsa2048_key.pem'), os.path.join(localhost_dir, 'key.pem')) + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.bind(('', 0)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.port = sock.getsockname()[1] + from acme.standalone import simple_tls_sni_01_server - self.thread = threading.Thread( - target=simple_tls_sni_01_server, kwargs={ - 'cli_args': ('filename',), - 'forever': False, - }, - ) + self.process = multiprocessing.Process(target=simple_tls_sni_01_server, + args=(['path', '-p', str(self.port)],)) self.old_cwd = os.getcwd() os.chdir(self.test_cwd) def tearDown(self): os.chdir(self.old_cwd) - self.thread.join() + if self.process.is_alive(): + self.process.terminate() shutil.rmtree(self.test_cwd) - @mock.patch('acme.standalone.logger') - def test_it(self, mock_logger): - # Use a Queue because mock objects aren't thread safe. - q = queue.Queue() # type: queue.Queue[int] - # Add port number to the queue. - mock_logger.info.side_effect = lambda *args: q.put(args[-1]) - self.thread.start() + @mock.patch('acme.standalone.TLSSNI01Server.handle_request') + def test_mock(self, handle): + from acme.standalone import simple_tls_sni_01_server + simple_tls_sni_01_server(cli_args=['path', '-p', str(self.port)], forever=False) + self.assertEqual(handle.call_count, 1) - # After the timeout, an exception is raised if the queue is empty. - port = q.get(timeout=5) - cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port) + def test_live(self): + self.process.start() + cert = None + for _ in range(50): + time.sleep(0.1) + try: + cert = crypto_util.probe_sni(b'localhost', b'127.0.0.1', self.port) + break + except errors.Error: # pragma: no cover + pass self.assertEqual(jose.ComparableX509(cert), - test_util.load_comparable_cert( - 'rsa2048_cert.pem')) + test_util.load_comparable_cert('rsa2048_cert.pem')) if __name__ == "__main__": diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index f97614700..1a0b67056 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -4,7 +4,6 @@ """ import os -import sys import pkg_resources import unittest @@ -95,11 +94,3 @@ def skip_unless(condition, reason): # pragma: no cover return lambda cls: cls else: return lambda cls: None - -def broken_on_windows(function): - """Decorator to skip temporarily a broken test on Windows.""" - reason = 'Test is broken and ignored on windows but should be fixed.' - return unittest.skipIf( - sys.platform == 'win32' - and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', - reason)(function) diff --git a/pytest.ini b/pytest.ini index 49db7da09..2531e50d2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,6 @@ filterwarnings = error ignore:decodestring:DeprecationWarning - ignore:TLS-SNI-01:DeprecationWarning + ignore:(TLSSNI01|TLS-SNI-01):DeprecationWarning ignore:.*collections\.abc:DeprecationWarning ignore:The `color_scheme` argument is deprecated:DeprecationWarning:IPython.* From 491d6c8f4538c28f7c05d780db9a700982460242 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 26 Mar 2019 23:27:06 -0700 Subject: [PATCH 630/639] Revert "Configure jessie repos in LTS mode during Docker build (#6887)" (#6889) This reverts commit a27bd28b39e5312ef77d5767a989fc4661e179fb. --- letsencrypt-auto-source/Dockerfile.jessie | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/letsencrypt-auto-source/Dockerfile.jessie b/letsencrypt-auto-source/Dockerfile.jessie index 528ab5c4a..9ee37b763 100644 --- a/letsencrypt-auto-source/Dockerfile.jessie +++ b/letsencrypt-auto-source/Dockerfile.jessie @@ -7,9 +7,7 @@ FROM debian:jessie RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea # Install pip, sudo, and openssl: -RUN echo "deb http://deb.debian.org/debian/ jessie main" > /etc/apt/sources.list && \ - echo "deb http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list && \ - apt-get update && \ +RUN apt-get update && \ apt-get -q -y install python-pip sudo openssl && \ apt-get clean # Use pipstrap to update to a stable and tested version of pip From b30a5e5b738f812c23937ea63eedb6443f7b98fd Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 27 Mar 2019 19:10:52 +0200 Subject: [PATCH 631/639] Add a test to ensure test coverage regardless of the vhost order (#6873) Add a new test to make sure that we are covering all the branches of get_virtual_hosts() regardless of the order that Augeas returns the found VirtualHost paths. Fixes: #6813 * Add a test to ensure test coverage regardless of the order of returned vhosts * Use deepcopy instead, and increase coverage requirement back to 100% --- .../certbot_apache/tests/configurator_test.py | 24 +++++++++++++++++++ tox.cover.py | 5 +--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index a8d5dd96e..884e82cb3 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1,5 +1,6 @@ # pylint: disable=too-many-public-methods,too-many-lines """Test for certbot_apache.configurator.""" +import copy import os import shutil import socket @@ -1512,6 +1513,29 @@ class MultipleVhostsTest(util.ApacheTest): second_id = self.config.add_vhost_id(self.vh_truth[0]) self.assertEqual(first_id, second_id) + def test_realpath_replaces_symlink(self): + orig_match = self.config.aug.match + mock_vhost = copy.deepcopy(self.vh_truth[0]) + mock_vhost.filep = mock_vhost.filep.replace('sites-enabled', u'sites-available') + mock_vhost.path = mock_vhost.path.replace('sites-enabled', 'sites-available') + mock_vhost.enabled = False + self.config.parser.parse_file(mock_vhost.filep) + + def mock_match(aug_expr): + """Return a mocked match list of VirtualHosts""" + if "/mocked/path" in aug_expr: + return [self.vh_truth[1].path, self.vh_truth[0].path, mock_vhost.path] + return orig_match(aug_expr) + + self.config.parser.parser_paths = ["/mocked/path"] + self.config.aug.match = mock_match + vhs = self.config.get_virtual_hosts() + self.assertEqual(len(vhs), 2) + self.assertTrue(vhs[0] == self.vh_truth[1]) + # mock_vhost should have replaced the vh_truth[0], because its filepath + # isn't a symlink + self.assertTrue(vhs[1] == mock_vhost) + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" diff --git a/tox.cover.py b/tox.cover.py index 6f5392b71..d0f97626a 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -14,10 +14,7 @@ DEFAULT_PACKAGES = [ COVER_THRESHOLDS = { 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, - # certbot_apache coverage not being at 100% is a workaround for - # https://github.com/certbot/certbot/issues/6813. We should increase - # the minimum coverage back to 100% when this issue is resolved. - 'certbot_apache': {'linux': 99, 'windows': 99}, + 'certbot_apache': {'linux': 100, 'windows': 100}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, From 414c70aa6ce3d4773259f5d4c077252c8d01ef79 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Mar 2019 13:07:42 -0700 Subject: [PATCH 632/639] Bump the min Certbot version for nginx plugin. (#6890) * Bump the min Certbot version for nginx plugin. * s/certbot/./g --- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 2108298ae..db6b261f0 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.29.0 -certbot[dev]==0.32.0 +-e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 7494bf6bf..49432a00d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -10,7 +10,7 @@ version = '0.33.0.dev0' # acme/certbot version. install_requires = [ 'acme>=0.26.0', - 'certbot>=0.22.0', + 'certbot>=0.33.0.dev0', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? From 8b8fc5ae543e3ce81db2f65e445ec2b67f223c50 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Mar 2019 13:27:38 -0700 Subject: [PATCH 633/639] Fix acme race condition (#6892) * Fix acme race condition. * Assert process has executed. --- acme/acme/standalone_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 953df40d4..90e1af37f 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -280,6 +280,10 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): os.chdir(self.old_cwd) if self.process.is_alive(): self.process.terminate() + self.process.join(timeout=5) + # Check that we didn't timeout waiting for the process to + # terminate. + self.assertNotEqual(self.process.exitcode, None) shutil.rmtree(self.test_cwd) @mock.patch('acme.standalone.TLSSNI01Server.handle_request') From b0fb570c1c9b5391babd35806cfa0caf8f8b3bf3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Mar 2019 14:38:28 -0700 Subject: [PATCH 634/639] Bump min nginx requirements to tested versions. (#6891) --- certbot-nginx/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 49432a00d..f768ed5d9 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -9,7 +9,7 @@ version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.26.0', + 'acme>=0.29.0', 'certbot>=0.33.0.dev0', 'mock', 'PyOpenSSL', From 6ce6c679320c53373b1da3180d664a34b943d883 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Mar 2019 23:51:48 +0100 Subject: [PATCH 635/639] [Windows] Security model for files permissions - STEP 1 (#6893) This PR is the first part of #6497 to ease the integration, following the new plan propose by @bmw here: #6497 (comment) This step 1 refactor existing certbot.compat module into certbot.compat.misc, without any logic changed. Package certbot.compat will host the new modules that constitute the security model for Windows. * Create the certbot.compat package. Move logic in certbot.compat.misc * Add doc * Fix lint * Correct mypy * Update client.py --- certbot-nginx/certbot_nginx/configurator.py | 14 ++++++------- certbot/account.py | 9 ++++---- certbot/cert_manager.py | 11 +++++----- certbot/client.py | 16 ++++++-------- certbot/compat/__init__.py | 6 ++++++ certbot/{compat.py => compat/misc.py} | 6 ++---- certbot/configuration.py | 6 +++--- certbot/constants.py | 12 ++++++----- certbot/crypto_util.py | 18 ++++++++-------- certbot/display/ops.py | 5 ++--- certbot/display/util.py | 6 +++--- certbot/log.py | 5 +++-- certbot/main.py | 15 +++++++------- certbot/plugins/webroot_test.py | 8 +++---- certbot/reverter.py | 11 +++++----- certbot/storage.py | 9 ++++---- certbot/tests/account_test.py | 8 +++---- certbot/tests/compat_test.py | 5 +++-- certbot/tests/configuration_test.py | 6 +++--- certbot/tests/display/util_test.py | 5 ++--- certbot/tests/log_test.py | 5 ++--- certbot/tests/main_test.py | 23 ++++++++++----------- certbot/tests/reverter_test.py | 3 +-- certbot/tests/storage_test.py | 18 +++++++--------- certbot/tests/util_test.py | 20 +++++++++--------- certbot/util.py | 9 ++++---- 26 files changed, 123 insertions(+), 136 deletions(-) create mode 100644 certbot/compat/__init__.py rename certbot/{compat.py => compat/misc.py} (96%) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8b39ee664..dbcd07a03 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -12,24 +12,22 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util +from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module -from certbot import compat from certbot import constants as core_constants from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util - +from certbot.compat import misc from certbot.plugins import common from certbot_nginx import constants from certbot_nginx import display_ops -from certbot_nginx import nginxparser -from certbot_nginx import parser from certbot_nginx import http_01 -from certbot_nginx import obj # pylint: disable=unused-import -from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module - +from certbot_nginx import nginxparser +from certbot_nginx import obj # pylint: disable=unused-import +from certbot_nginx import parser NAME_RANK = 0 START_WILDCARD_RANK = 1 @@ -895,7 +893,7 @@ class NginxConfigurator(common.Installer): have permissions of root. """ - uid = compat.os_geteuid() + uid = misc.os_geteuid() util.make_or_verify_dir( self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) util.make_or_verify_dir( diff --git a/certbot/account.py b/certbot/account.py index 0c653f6dd..a37b49eb4 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,22 +7,21 @@ import os import shutil import socket -from cryptography.hazmat.primitives import serialization import josepy as jose import pyrfc3339 import pytz import six import zope.component +from cryptography.hazmat.primitives import serialization from acme import fields as acme_fields from acme import messages -from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces from certbot import util - +from certbot.compat import misc logger = logging.getLogger(__name__) @@ -141,7 +140,7 @@ class AccountFileStorage(interfaces.AccountStorage): """ def __init__(self, config): self.config = config - util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(), + util.make_or_verify_dir(config.accounts_dir, 0o700, misc.os_geteuid(), self.config.strict_permissions) def _account_dir_path(self, account_id): @@ -324,7 +323,7 @@ class AccountFileStorage(interfaces.AccountStorage): def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) - util.make_or_verify_dir(account_dir_path, 0o700, compat.os_geteuid(), + util.make_or_verify_dir(account_dir_path, 0o700, misc.os_geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 2a67f8765..8d6f04b71 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -2,20 +2,21 @@ import datetime import logging import os -import pytz import re import traceback + +import pytz import zope.component from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from certbot import compat + from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import ocsp from certbot import storage from certbot import util - +from certbot.compat import misc from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -105,7 +106,7 @@ def lineage_for_certname(cli_config, certname): """Find a lineage object with name certname.""" configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=misc.os_geteuid()) try: renewal_file = storage.renewal_file_for_certname(cli_config, certname) except errors.CertStorageError: @@ -375,7 +376,7 @@ def _search_lineages(cli_config, func, initial_rv, *args): """ configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=misc.os_geteuid()) rv = initial_rv for renewal_file in storage.renewal_conf_files(cli_config): diff --git a/certbot/client.py b/certbot/client.py index 38b77a772..a6ca17945 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -4,14 +4,13 @@ import logging import os import platform - +import OpenSSL +import josepy as jose +import zope.component from cryptography.hazmat.backends import default_backend # https://github.com/python/typeshed/blob/master/third_party/ # 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key # type: ignore -import josepy as jose -import OpenSSL -import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util @@ -20,11 +19,9 @@ from acme import messages from acme.magic_typing import Optional # pylint: disable=unused-import,no-name-in-module import certbot - from certbot import account from certbot import auth_handler from certbot import cli -from certbot import compat from certbot import constants from certbot import crypto_util from certbot import eff @@ -34,12 +31,11 @@ from certbot import interfaces from certbot import reverter from certbot import storage from certbot import util - -from certbot.display import ops as display_ops +from certbot.compat import misc from certbot.display import enhancements +from certbot.display import ops as display_ops from certbot.plugins import selection as plugin_selection - logger = logging.getLogger(__name__) @@ -466,7 +462,7 @@ class Client(object): """ for path in cert_path, chain_path, fullchain_path: util.make_or_verify_dir( - os.path.dirname(path), 0o755, compat.os_geteuid(), + os.path.dirname(path), 0o755, misc.os_geteuid(), self.config.strict_permissions) diff --git a/certbot/compat/__init__.py b/certbot/compat/__init__.py new file mode 100644 index 000000000..74451131a --- /dev/null +++ b/certbot/compat/__init__.py @@ -0,0 +1,6 @@ +""" +Compatibility layer to run certbot both on Linux and Windows. + +This package contains all logic that needs to be implemented specifically for Linux and for Windows. +Then the rest of certbot code relies on this module to be platform agnostic. +""" diff --git a/certbot/compat.py b/certbot/compat/misc.py similarity index 96% rename from certbot/compat.py rename to certbot/compat/misc.py index f533f5954..3ea4a7908 100644 --- a/certbot/compat.py +++ b/certbot/compat/misc.py @@ -1,8 +1,6 @@ """ -Compatibility layer to run certbot both on Linux and Windows. - -This module contains all required platform specific code, -allowing the rest of Certbot codebase to be platform agnostic. +This compat module handles various platform specific calls that do not fall into one +particular category. """ import os import select diff --git a/certbot/configuration.py b/certbot/configuration.py index 5d248f39e..2e7e39e28 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -2,14 +2,14 @@ import copy import os -from six.moves.urllib import parse # pylint: disable=import-error import zope.interface +from six.moves.urllib import parse # pylint: disable=import-error -from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces from certbot import util +from certbot.compat import misc @zope.interface.implementer(interfaces.IConfig) @@ -70,7 +70,7 @@ class NamespaceConfig(object): def accounts_dir_for_server_path(self, server_path): """Path to accounts directory based on server_path""" - server_path = compat.underscores_for_unsupported_characters_in_path(server_path) + server_path = misc.underscores_for_unsupported_characters_in_path(server_path) return os.path.join( self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) diff --git a/certbot/constants.py b/certbot/constants.py index 31b243518..3b2f7a2d9 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -1,10 +1,12 @@ """Certbot constants.""" import logging import os + import pkg_resources from acme import challenges -from certbot import compat + +from certbot.compat import misc SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins" """Setuptools entry point group name for plugins.""" @@ -14,7 +16,7 @@ OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT = "letsencrypt.plugins" CLI_DEFAULTS = dict( config_files=[ - os.path.join(compat.get_default_folder('config'), 'cli.ini'), + os.path.join(misc.get_default_folder('config'), 'cli.ini'), # http://freedesktop.org/wiki/Software/xdg-user-dirs/ os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), @@ -87,9 +89,9 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", key_path=None, - config_dir=compat.get_default_folder('config'), - work_dir=compat.get_default_folder('work'), - logs_dir=compat.get_default_folder('logs'), + config_dir=misc.get_default_folder('config'), + work_dir=misc.get_default_folder('work'), + logs_dir=misc.get_default_folder('logs'), server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 66c68eb38..f976372c5 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,24 +12,24 @@ import warnings import pyrfc3339 import six import zope.component +from OpenSSL import SSL # type: ignore +from OpenSSL import crypto +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.ec import ECDSA from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore -from OpenSSL import crypto -from OpenSSL import SSL # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module -from certbot import compat + from certbot import errors from certbot import interfaces from certbot import util - +from certbot.compat import misc logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): config = zope.component.getUtility(interfaces.IConfig) # Save file - util.make_or_verify_dir(key_dir, 0o700, compat.os_geteuid(), + util.make_or_verify_dir(key_dir, 0o700, misc.os_geteuid(), config.strict_permissions) key_f, key_path = util.unique_file( os.path.join(key_dir, keyname), 0o600, "wb") @@ -92,8 +92,8 @@ def init_save_csr(privkey, names, path): privkey.pem, names, must_staple=config.must_staple) # Save CSR - util.make_or_verify_dir(path, 0o755, compat.os_geteuid(), - config.strict_permissions) + util.make_or_verify_dir(path, 0o755, misc.os_geteuid(), + config.strict_permissions) csr_f, csr_filename = util.unique_file( os.path.join(path, "csr-certbot.pem"), 0o644, "wb") with csr_f: diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 3dae1070b..db69f6c9f 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -4,11 +4,10 @@ import os import zope.component -from certbot import compat from certbot import errors from certbot import interfaces from certbot import util - +from certbot.compat import misc from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -36,7 +35,7 @@ def get_email(invalid=False, optional=True): "the client with --register-unsafely-without-email " "but make sure you then backup your account key from " "{0}\n\n".format(os.path.join( - compat.get_default_folder('config'), 'accounts'))) + misc.get_default_folder('config'), 'accounts'))) if optional: if invalid: msg += unsafe_suggestion diff --git a/certbot/display/util.py b/certbot/display/util.py index 772b67d74..72ee892a7 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -6,10 +6,10 @@ import textwrap import zope.interface -from certbot import compat from certbot import constants -from certbot import interfaces from certbot import errors +from certbot import interfaces +from certbot.compat import misc from certbot.display import completer logger = logging.getLogger(__name__) @@ -79,7 +79,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): sys.stdout.write(prompt) sys.stdout.flush() - line = compat.readline_with_timeout(timeout, prompt) + line = misc.readline_with_timeout(timeout, prompt) if not line: raise EOFError diff --git a/certbot/log.py b/certbot/log.py index b883936f3..7c86dbdf0 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -13,6 +13,7 @@ and properly flushed before program exit. """ from __future__ import print_function + import functools import logging import logging.handlers @@ -23,10 +24,10 @@ import traceback from acme import messages -from certbot import compat from certbot import constants from certbot import errors from certbot import util +from certbot.compat import misc # Logging format CLI_FMT = "%(message)s" @@ -134,7 +135,7 @@ def setup_log_file_handler(config, logfile, fmt): # TODO: logs might contain sensitive data such as contents of the # private key! #525 util.set_up_core_dir( - config.logs_dir, 0o700, compat.os_geteuid(), config.strict_permissions) + config.logs_dir, 0o700, misc.os_geteuid(), config.strict_permissions) log_file_path = os.path.join(config.logs_dir, logfile) try: handler = logging.handlers.RotatingFileHandler( diff --git a/certbot/main.py b/certbot/main.py index a0c0ab64d..484ac52ea 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,6 +1,7 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines from __future__ import print_function + import functools import logging.handlers import os @@ -14,12 +15,10 @@ from acme import errors as acme_errors from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module import certbot - from certbot import account from certbot import cert_manager from certbot import cli from certbot import client -from certbot import compat from certbot import configuration from certbot import constants from certbot import crypto_util @@ -33,11 +32,11 @@ from certbot import reporter from certbot import storage from certbot import updater from certbot import util - +from certbot.compat import misc from certbot.display import util as display_util, ops as display_ops from certbot.plugins import disco as plugins_disco -from certbot.plugins import selection as plug_sel from certbot.plugins import enhancements +from certbot.plugins import selection as plug_sel USER_CANCELLED = ("User chose to cancel the operation and may " "reinvoke the client.") @@ -1285,16 +1284,16 @@ def make_or_verify_needed_dirs(config): """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, - compat.os_geteuid(), config.strict_permissions) + misc.os_geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - compat.os_geteuid(), config.strict_permissions) + misc.os_geteuid(), config.strict_permissions) hook_dirs = (config.renewal_pre_hooks_dir, config.renewal_deploy_hooks_dir, config.renewal_post_hooks_dir,) for hook_dir in hook_dirs: util.make_or_verify_dir(hook_dir, - uid=compat.os_geteuid(), + uid=misc.os_geteuid(), strict=config.strict_permissions) @@ -1345,7 +1344,7 @@ def main(cli_args=sys.argv[1:]): # On windows, shell without administrative right cannot create symlinks required by certbot. # So we check the rights before continuing. - compat.raise_for_non_administrative_windows_rights(config.verb) + misc.raise_for_non_administrative_windows_rights(config.verb) try: log.post_arg_parse_setup(config) diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 5303fe4da..a67ddbb83 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -17,14 +17,12 @@ import six from acme import challenges from certbot import achallenges -from certbot import compat from certbot import errors +from certbot.compat import misc from certbot.display import util as display_util - from certbot.tests import acme_util from certbot.tests import util as test_util - KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) @@ -171,14 +169,14 @@ class AuthenticatorTest(unittest.TestCase): # Remove exec bit from permission check, so that it # matches the file self.auth.perform([self.achall]) - self.assertTrue(compat.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644)) + self.assertTrue(misc.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644)) # Check permissions of the directories for dirpath, dirnames, _ in os.walk(self.path): for directory in dirnames: full_path = os.path.join(dirpath, directory) - self.assertTrue(compat.compare_file_modes(os.stat(full_path).st_mode, 0o755)) + self.assertTrue(misc.compare_file_modes(os.stat(full_path).st_mode, 0o755)) parent_gid = os.stat(self.path).st_gid parent_uid = os.stat(self.path).st_uid diff --git a/certbot/reverter.py b/certbot/reverter.py index 919037358..e08b468ac 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -10,12 +10,11 @@ import traceback import six import zope.component -from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces from certbot import util - +from certbot.compat import misc logger = logging.getLogger(__name__) @@ -66,7 +65,7 @@ class Reverter(object): self.config = config util.make_or_verify_dir( - config.backup_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), + config.backup_dir, constants.CONFIG_DIRS_MODE, misc.os_geteuid(), self.config.strict_permissions) def revert_temporary_config(self): @@ -220,7 +219,7 @@ class Reverter(object): """ util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, misc.os_geteuid(), self.config.strict_permissions) op_fd, existing_filepaths = self._read_and_append( @@ -434,7 +433,7 @@ class Reverter(object): cp_dir = self.config.in_progress_dir util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, misc.os_geteuid(), self.config.strict_permissions) return cp_dir @@ -576,7 +575,7 @@ class Reverter(object): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: - compat.os_rename(self.config.in_progress_dir, final_dir) + misc.os_rename(self.config.in_progress_dir, final_dir) return except OSError: logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp) diff --git a/certbot/storage.py b/certbot/storage.py index d17a0f29d..d0bc36c08 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -4,23 +4,22 @@ import glob import logging import os import re +import shutil import stat import configobj import parsedatetime import pytz -import shutil import six import certbot from certbot import cli -from certbot import compat from certbot import constants from certbot import crypto_util -from certbot import errors from certbot import error_handler +from certbot import errors from certbot import util - +from certbot.compat import misc from certbot.plugins import common as plugins_common from certbot.plugins import disco as plugins_disco @@ -192,7 +191,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): # Save only the config items that are relevant to renewal values = relevant_values(vars(cli_config.namespace)) write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - compat.os_rename(temp_filename, config_filename) + misc.os_rename(temp_filename, config_filename) return configobj.ConfigObj(config_filename) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index b062f437b..fb0205f9c 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -12,11 +12,9 @@ import pytz from acme import messages -from certbot import compat -from certbot import errors - import certbot.tests.util as test_util - +from certbot import errors +from certbot.compat import misc KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) @@ -116,7 +114,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_init_creates_dir(self): self.assertTrue(os.path.isdir( - compat.underscores_for_unsupported_characters_in_path(self.config.accounts_dir))) + misc.underscores_for_unsupported_characters_in_path(self.config.accounts_dir))) @test_util.broken_on_windows def test_save_and_restore(self): diff --git a/certbot/tests/compat_test.py b/certbot/tests/compat_test.py index 552aa5645..ffbba24fd 100644 --- a/certbot/tests/compat_test.py +++ b/certbot/tests/compat_test.py @@ -1,8 +1,9 @@ """Tests for certbot.compat.""" import os -from certbot import compat import certbot.tests.util as test_util +from certbot.compat import misc + class OsReplaceTest(test_util.TempDirTestCase): """Test to ensure consistent behavior of os_rename method""" @@ -15,7 +16,7 @@ class OsReplaceTest(test_util.TempDirTestCase): open(dst, 'w').close() # On Windows, a direct call to os.rename will fail because dst already exists. - compat.os_rename(src, dst) + misc.os_rename(src, dst) self.assertFalse(os.path.exists(src)) self.assertTrue(os.path.exists(dst)) diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index ce3cded20..a4d32a57f 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -4,12 +4,12 @@ import unittest import mock -from certbot import compat from certbot import constants from certbot import errors - +from certbot.compat import misc from certbot.tests import util as test_util + class NamespaceConfigTest(test_util.ConfigTestCase): """Tests for certbot.configuration.NamespaceConfig.""" @@ -48,7 +48,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_constants.KEY_DIR = 'keys' mock_constants.TEMP_CHECKPOINT_DIR = 't' - ref_path = compat.underscores_for_unsupported_characters_in_path( + ref_path = misc.underscores_for_unsupported_characters_in_path( 'acc/acme-server.org:443/new') self.assertEqual( os.path.normpath(self.config.accounts_dir), diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 726eb0b0f..8ee0501ef 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,14 +4,13 @@ import socket import tempfile import unittest -import six import mock +import six from certbot import errors from certbot import interfaces from certbot.display import util as display_util - CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] @@ -32,7 +31,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" stdin = six.StringIO(expected + "\n") - with mock.patch("certbot.compat.select.select") as mock_select: + with mock.patch("certbot.compat.misc.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index b82cc6ca1..c929853dd 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -8,14 +8,13 @@ import unittest import mock import six - from acme import messages from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module -from certbot import compat from certbot import constants from certbot import errors from certbot import util +from certbot.compat import misc from certbot.tests import util as test_util @@ -261,7 +260,7 @@ class TempHandlerTest(unittest.TestCase): def test_permissions(self): self.assertTrue( - util.check_permissions(self.handler.path, 0o600, compat.os_geteuid())) + util.check_permissions(self.handler.path, 0o600, misc.os_geteuid())) def test_delete(self): self.handler.close() diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e1be6e023..ba1799f32 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -3,42 +3,41 @@ # pylint: disable=too-many-lines from __future__ import print_function +import datetime import itertools import json -import mock import os import shutil +import sys +import tempfile import traceback import unittest -import datetime -import pytz -import tempfile -import sys import josepy as jose +import mock +import pytz import six from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + +import certbot.tests.util as test_util from certbot import account from certbot import cli -from certbot import compat -from certbot import constants from certbot import configuration +from certbot import constants from certbot import crypto_util from certbot import errors from certbot import interfaces # pylint: disable=unused-import from certbot import main from certbot import updater from certbot import util - +from certbot.compat import misc from certbot.plugins import disco from certbot.plugins import enhancements from certbot.plugins import manual from certbot.plugins import null -import certbot.tests.util as test_util - CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.vector_path('cert_512.pem') CSR = test_util.vector_path('csr_512.der') @@ -1587,7 +1586,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for core_dir in (self.config.config_dir, self.config.work_dir,): mock_util.set_up_core_dir.assert_any_call( core_dir, constants.CONFIG_DIRS_MODE, - compat.os_geteuid(), self.config.strict_permissions + misc.os_geteuid(), self.config.strict_permissions ) hook_dirs = (self.config.renewal_pre_hooks_dir, @@ -1596,7 +1595,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for hook_dir in hook_dirs: # default mode of 755 is used mock_util.make_or_verify_dir.assert_any_call( - hook_dir, uid=compat.os_geteuid(), + hook_dir, uid=misc.os_geteuid(), strict=self.config.strict_permissions) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d04e3c641..8e05d4f1c 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -10,7 +10,6 @@ import mock import six from certbot import errors - from certbot.tests import util as test_util @@ -356,7 +355,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @mock.patch("certbot.reverter.compat.os_rename") + @mock.patch("certbot.reverter.misc.os_rename") def test_finalize_checkpoint_no_rename_directory(self, mock_rename): self.reverter.add_to_checkpoint(self.sets[0], "perm save") diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 872fb4302..a6577deb3 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -12,12 +12,10 @@ import pytz import six import certbot -from certbot import compat -from certbot import errors -from certbot.storage import ALL_FOUR - import certbot.tests.util as test_util - +from certbot import errors +from certbot.compat import misc +from certbot.storage import ALL_FOUR CERT = test_util.load_cert('cert_512.pem') @@ -572,21 +570,21 @@ class RenewableCertTests(BaseRenewableCertTest): for kind in ALL_FOUR: self._write_out_kind(kind, 1) self.test_rc.update_all_links_to(1) - self.assertTrue(compat.compare_file_modes( + self.assertTrue(misc.compare_file_modes( os.stat(self.test_rc.version("privkey", 1)).st_mode, 0o600)) os.chmod(self.test_rc.version("privkey", 1), 0o444) # If no new key, permissions should be the same (we didn't write any keys) self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) - self.assertTrue(compat.compare_file_modes( + self.assertTrue(misc.compare_file_modes( os.stat(self.test_rc.version("privkey", 2)).st_mode, 0o444)) # If new key, permissions should be kept as 644 self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) - self.assertTrue(compat.compare_file_modes( + self.assertTrue(misc.compare_file_modes( os.stat(self.test_rc.version("privkey", 3)).st_mode, 0o644)) # If permissions reverted, next renewal will also revert permissions of new key os.chmod(self.test_rc.version("privkey", 3), 0o400) self.test_rc.save_successor(3, b"newcert", b"new_privkey", b"new chain", self.config) - self.assertTrue(compat.compare_file_modes( + self.assertTrue(misc.compare_file_modes( os.stat(self.test_rc.version("privkey", 4)).st_mode, 0o600)) @test_util.broken_on_windows @@ -624,7 +622,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.config.live_dir, "README"))) self.assertTrue(os.path.exists(os.path.join( self.config.live_dir, "the-lineage.com", "README"))) - self.assertTrue(compat.compare_file_modes(os.stat(result.key_path).st_mode, 0o600)) + self.assertTrue(misc.compare_file_modes(os.stat(result.key_path).st_mode, 0o600)) with open(result.fullchain, "rb") as f: self.assertEqual(f.read(), b"cert" + b"chain") # Let's do it again and make sure it makes a different lineage diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index b848dbb9f..2974f8f77 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -8,9 +8,9 @@ import mock import six from six.moves import reload_module # pylint: disable=import-error -from certbot import compat -from certbot import errors import certbot.tests.util as test_util +from certbot import errors +from certbot.compat import misc class RunScriptTest(unittest.TestCase): @@ -119,7 +119,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): @mock.patch('certbot.util.lock_dir_until_exit') def test_success(self, mock_lock): new_dir = os.path.join(self.tempdir, 'new') - self._call(new_dir, 0o700, compat.os_geteuid(), False) + self._call(new_dir, 0o700, misc.os_geteuid(), False) self.assertTrue(os.path.exists(new_dir)) self.assertEqual(mock_lock.call_count, 1) @@ -127,7 +127,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, - self.tempdir, 0o700, compat.os_geteuid(), False) + self.tempdir, 0o700, misc.os_geteuid(), False) class MakeOrVerifyDirTest(test_util.TempDirTestCase): @@ -144,7 +144,7 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): self.path = os.path.join(self.tempdir, "foo") os.mkdir(self.path, 0o600) - self.uid = compat.os_geteuid() + self.uid = misc.os_geteuid() def _call(self, directory, mode): from certbot.util import make_or_verify_dir @@ -154,11 +154,11 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): path = os.path.join(self.tempdir, "bar") self._call(path, 0o650) self.assertTrue(os.path.isdir(path)) - self.assertTrue(compat.compare_file_modes(os.stat(path).st_mode, 0o650)) + self.assertTrue(misc.compare_file_modes(os.stat(path).st_mode, 0o650)) def test_existing_correct_mode_does_not_fail(self): self._call(self.path, 0o600) - self.assertTrue(compat.compare_file_modes(os.stat(self.path).st_mode, 0o600)) + self.assertTrue(misc.compare_file_modes(os.stat(self.path).st_mode, 0o600)) @test_util.skip_on_windows('Umask modes are mostly ignored on Windows.') def test_existing_wrong_mode_fails(self): @@ -181,7 +181,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def setUp(self): super(CheckPermissionsTest, self).setUp() - self.uid = compat.os_geteuid() + self.uid = misc.os_geteuid() def _call(self, mode): from certbot.util import check_permissions @@ -223,8 +223,8 @@ class UniqueFileTest(test_util.TempDirTestCase): def test_right_mode(self): fd1, name1 = self._call(0o700) fd2, name2 = self._call(0o600) - self.assertTrue(compat.compare_file_modes(0o700, os.stat(name1).st_mode)) - self.assertTrue(compat.compare_file_modes(0o600, os.stat(name2).st_mode)) + self.assertTrue(misc.compare_file_modes(0o700, os.stat(name1).st_mode)) + self.assertTrue(misc.compare_file_modes(0o600, os.stat(name2).st_mode)) fd1.close() fd2.close() diff --git a/certbot/util.py b/certbot/util.py index 6481e30d2..fbdfd685a 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -10,20 +10,19 @@ import logging import os import platform import re -import six import socket import subprocess - from collections import OrderedDict import configargparse +import six from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module -from certbot import compat + from certbot import constants from certbot import errors from certbot import lock - +from certbot.compat import misc logger = logging.getLogger(__name__) @@ -204,7 +203,7 @@ def check_permissions(filepath, mode, uid=0): """ file_stat = os.stat(filepath) - return compat.compare_file_modes(file_stat.st_mode, mode) and file_stat.st_uid == uid + return misc.compare_file_modes(file_stat.st_mode, mode) and file_stat.st_uid == uid def safe_open(path, mode="w", chmod=None, buffering=None): From ea568d4dc2411214850f04943ffba4e799197f64 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 29 Mar 2019 00:50:42 +0100 Subject: [PATCH 636/639] [Windows] Fix ErrorHandler tests, by disabling signal error handling (#6868) This PR is a part of the effort to remove the last broken unit tests in certbot codebase for Windows, as described in #6850. It solves the problems associated to ErrorHandler in Windows (enlighted by tests errors) by ... wipping out the problem: no signal is handled by ErrorHandler on Windows. See the relevant inline comment in certbot.error_handler for explanation and sources. --- certbot/error_handler.py | 18 +++++++++++++++++- certbot/tests/error_handler_test.py | 12 +++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/certbot/error_handler.py b/certbot/error_handler.py index 5e72f8153..64e9a1488 100644 --- a/certbot/error_handler.py +++ b/certbot/error_handler.py @@ -19,14 +19,30 @@ logger = logging.getLogger(__name__) # potentially occur from inside Python. Signals such as SIGILL were not # included as they could be a sign of something devious and we should terminate # immediately. -_SIGNALS = [signal.SIGTERM] if os.name != "nt": + _SIGNALS = [signal.SIGTERM] for signal_code in [signal.SIGHUP, signal.SIGQUIT, signal.SIGXCPU, signal.SIGXFSZ]: # Adding only those signals that their default action is not Ignore. # This is platform-dependent, so we check it dynamically. if signal.getsignal(signal_code) != signal.SIG_IGN: _SIGNALS.append(signal_code) +else: + # POSIX signals are not implemented natively in Windows, but emulated from the C runtime. + # As consumed by CPython, most of handlers on theses signals are useless, in particular + # SIGTERM: for instance, os.kill(pid, signal.SIGTERM) will call TerminateProcess, that stops + # immediately the process without calling the attached handler. Besides, non-POSIX signals + # (CTRL_C_EVENT and CTRL_BREAK_EVENT) are implemented in a console context to handle the + # CTRL+C event to a process launched from the console. Only CTRL_C_EVENT has a reliable + # behavior in fact, and maps to the handler to SIGINT. However in this case, a + # KeyboardInterrupt is raised, that will be handled by ErrorHandler through the context manager + # protocol. Finally, no signal on Windows is electable to be handled using ErrorHandler. + # + # Refs: https://stackoverflow.com/a/35792192, https://maruel.ca/post/python_windows_signal, + # https://docs.python.org/2/library/os.html#os.kill, + # https://www.reddit.com/r/Python/comments/1dsblt/windows_command_line_automation_ctrlc_question + _SIGNALS = [] + class ErrorHandler(object): """Context manager for running code that must be cleaned up on failure. diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 8508a3df5..bc6d4fe3f 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -10,7 +10,6 @@ import mock from acme.magic_typing import Callable, Dict, Union # pylint: enable=unused-import, no-name-in-module -import certbot.tests.util as test_util def get_signals(signums): """Get the handlers for an iterable of signums.""" @@ -66,9 +65,9 @@ class ErrorHandlerTest(unittest.TestCase): self.init_func.assert_called_once_with(*self.init_args, **self.init_kwargs) - # On Windows, this test kills pytest itself ! - @test_util.broken_on_windows def test_context_manager_with_signal(self): + if not self.signals: + self.skipTest(reason='Signals cannot be handled on Windows.') init_signals = get_signals(self.signals) with signal_receiver(self.signals) as signals_received: with self.handler: @@ -98,9 +97,9 @@ class ErrorHandlerTest(unittest.TestCase): **self.init_kwargs) bad_func.assert_called_once_with() - # On Windows, this test kills pytest itself ! - @test_util.broken_on_windows def test_bad_recovery_with_signal(self): + if not self.signals: + self.skipTest(reason='Signals cannot be handled on Windows.') sig1 = self.signals[0] sig2 = self.signals[-1] bad_func = mock.MagicMock(side_effect=lambda: send_signal(sig1)) @@ -149,10 +148,9 @@ class ExitHandlerTest(ErrorHandlerTest): **self.init_kwargs) func.assert_called_once_with() - # On Windows, this test kills pytest itself ! - @test_util.broken_on_windows def test_bad_recovery_with_signal(self): super(ExitHandlerTest, self).test_bad_recovery_with_signal() + if __name__ == "__main__": unittest.main() # pragma: no cover From 63c8f2e34d622ba71f7d15349ea09384bdd65a27 Mon Sep 17 00:00:00 2001 From: aditj Date: Sun, 31 Mar 2019 00:01:11 +0530 Subject: [PATCH 637/639] Changed the text of -h to add details regarding unregister and all --- certbot/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 1472b0139..320e5e96a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -101,6 +101,7 @@ manage certificates: manage your account with Let's Encrypt: register Create a Let's Encrypt ACME account + unregister Deactivate a Let's Encrypt ACME account update_account Update a Let's Encrypt ACME account --agree-tos Agree to the ACME server's Subscriber Agreement -m EMAIL Email address for important account notifications @@ -117,7 +118,7 @@ More detailed help: all, automation, commands, paths, security, testing, or any of the subcommands or plugins (certonly, renew, install, register, nginx, apache, standalone, webroot, etc.) - + -h all print a detailed help page including all topics --version print the version number """ From 232e0ea50fa7472803510260b8368aac33478ff8 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 1 Apr 2019 18:50:08 +0200 Subject: [PATCH 638/639] Rely on universal newline mode on python 3 for windows (#6866) --- certbot/reverter.py | 19 +++++++++++++------ certbot/tests/reverter_test.py | 10 ---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/certbot/reverter.py b/certbot/reverter.py index e08b468ac..f05e24b01 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -4,17 +4,18 @@ import glob import logging import os import shutil +import sys import time import traceback import six import zope.component +from certbot.compat import misc from certbot import constants from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import misc logger = logging.getLogger(__name__) @@ -237,7 +238,7 @@ class Reverter(object): try: shutil.copy2(filename, os.path.join( cp_dir, os.path.basename(filename) + "_" + str(idx))) - op_fd.write(filename + os.linesep) + op_fd.write('{0}\n'.format(filename)) # http://stackoverflow.com/questions/4726260/effective-use-of-python-shutil-copy2 except IOError: op_fd.close() @@ -312,7 +313,10 @@ class Reverter(object): """Run all commands in a file.""" # NOTE: csv module uses native strings. That is, bytes on Python 2 and # unicode on Python 3 - with open(filepath, 'r') as csvfile: + # It is strongly advised to set newline = '' on Python 3 with CSV, + # and it fixes problems on Windows. + kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + with open(filepath, 'r', **kwargs) as csvfile: # type: ignore # pylint: disable=star-args csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): try: @@ -381,7 +385,7 @@ class Reverter(object): for path in files: if path not in ex_files: - new_fd.write("{0}{1}".format(path, os.linesep)) + new_fd.write("{0}\n".format(path)) except (IOError, OSError): logger.error("Unable to register file creation(s) - %s", files) raise errors.ReverterError( @@ -408,11 +412,14 @@ class Reverter(object): """ commands_fp = os.path.join(self._get_cp_dir(temporary), "COMMANDS") command_file = None + # It is strongly advised to set newline = '' on Python 3 with CSV, + # and it fixes problems on Windows. + kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} try: if os.path.isfile(commands_fp): - command_file = open(commands_fp, "a") + command_file = open(commands_fp, "a", **kwargs) # type: ignore # pylint: disable=star-args else: - command_file = open(commands_fp, "w") + command_file = open(commands_fp, "w", **kwargs) # type: ignore # pylint: disable=star-args csvwriter = csv.writer(command_file) csvwriter.writerow(command) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 8e05d4f1c..18e698444 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -49,7 +49,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): x = f.read() self.assertTrue("No changes" in x) - @test_util.broken_on_windows def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -91,7 +90,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, set([config3]), "invalid save") - @test_util.broken_on_windows def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") update_file(self.config1, "updated-directive") @@ -121,7 +119,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertFalse(os.path.isfile(config3)) self.assertFalse(os.path.isfile(config4)) - @test_util.broken_on_windows def test_multiple_registration_same_file(self): self.reverter.register_file_creation(True, self.config1) self.reverter.register_file_creation(True, self.config1) @@ -146,7 +143,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_file_creation, "filepath") - @test_util.broken_on_windows def test_register_undo_command(self): coms = [ ["a2dismod", "ssl"], @@ -169,7 +165,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_undo_command, True, ["command"]) - @test_util.broken_on_windows @mock.patch("certbot.util.run_script") def test_run_undo_commands(self, mock_run): mock_run.side_effect = ["", errors.SubprocessError] @@ -233,7 +228,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) - @test_util.broken_on_windows @mock.patch("certbot.reverter.logger.warning") def test_recover_checkpoint_missing_new_files(self, mock_warn): self.reverter.register_file_creation( @@ -248,7 +242,6 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) - @test_util.broken_on_windows def test_recovery_routine_temp_and_perm(self): # Register a new perm checkpoint file config3 = os.path.join(self.dir1, "config3.txt") @@ -312,7 +305,6 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.rollback_checkpoints, "one") - @test_util.broken_on_windows def test_rollback_finalize_checkpoint_valid_inputs(self): config3 = self._setup_three_checkpoints() @@ -364,7 +356,6 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @test_util.broken_on_windows @mock.patch("certbot.reverter.logger") def test_rollback_too_many(self, mock_logger): # Test no exist warning... @@ -377,7 +368,6 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.reverter.rollback_checkpoints(4) self.assertEqual(mock_logger.warning.call_count, 1) - @test_util.broken_on_windows def test_multi_rollback(self): config3 = self._setup_three_checkpoints() self.reverter.rollback_checkpoints(3) From fd6702b86951c96192336e4e77fca7d98e1425f3 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 2 Apr 2019 19:26:58 +0300 Subject: [PATCH 639/639] Fix CentOS 6 installer issue (#6784) In CentOS 6 default httpd configuration, the `LoadModule ssl_module ...` is handled in `conf.d/ssl.conf`. As the `VirtualHost` configuration files in `conf.d/` are loaded in alphabetical order, this means that all files that have `` and are loaded before `ssl.conf` are effectively ignored. This PR moves the `LoadModule ssl_module` to the main `httpd.conf` while leaving a conditional `LoadModule` directive in `ssl.conf`. Features - Reads the module configuration from `ssl.conf` in case some modifications to paths have been made by the user. - Falls back to default paths if the directive doesn't exist. - Moves the `LoadModule` directive in `ssl.conf` inside `` to avoid printing warning messages of duplicate module loads. - Adds `LoadModule ssl_module` inside of `` to the top of the main `httpd.conf`. - Ensures that these modifications are not made multiple times. Fixes: #6606 * Fix CentOS6 installer issue * Changelog entry * Address review comments * Do not enable mod_ssl if multiple different values were found * Add test comment * Address rest of the review comments * Address review comments * Better ifmodule argument checking * Test fixes * Make linter happy * Raise an exception when differing LoadModule ssl_module statements are found * If IfModule !mod_ssl.c with LoadModule ssl_module already exists in Augeas path, do not create new LoadModule directive * Do not use deprecated assertion functions * Address review comments * Kick tests * Revert "Kick tests" This reverts commit 967bb574c2d7d6175133931826cc2cdb4b997dda. * Address review comments * Add pydoc return value to create_ifmod --- CHANGELOG.md | 3 + .../certbot_apache/override_centos.py | 116 ++ certbot-apache/certbot_apache/parser.py | 62 +- .../certbot_apache/tests/centos6_test.py | 224 ++++ .../centos6_apache/apache/httpd/conf.d/README | 9 + .../apache/httpd/conf.d/ssl.conf | 222 ++++ .../apache/httpd/conf.d/test.example.com.conf | 7 + .../apache/httpd/conf.d/welcome.conf | 11 + .../apache/httpd/conf/httpd.conf | 1009 +++++++++++++++++ 9 files changed, 1655 insertions(+), 8 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/centos6_test.py create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/README create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/ssl.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/test.example.com.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/welcome.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf/httpd.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc823d1c..e74779ac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Certbot uses the Python library cryptography for OCSP when cryptography>=2.5 is installed. We fixed a bug in Certbot causing it to interpret timestamps in the OCSP response as being in the local timezone rather than UTC. +* Issue causing the default CentOS 6 TLS configuration to ignore some of the + HTTPS VirtualHosts created by Certbot. mod_ssl loading is now moved to main + http.conf for this environment where possible. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index a4f1b84ec..1995fd2a2 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -1,13 +1,21 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ +import logging import pkg_resources +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + import zope.interface from certbot import interfaces + from certbot_apache import apache_util from certbot_apache import configurator from certbot_apache import parser +from certbot.errors import MisconfigurationError + +logger = logging.getLogger(__name__) + @zope.interface.provider(interfaces.IPluginFactory) class CentOSConfigurator(configurator.ApacheConfigurator): @@ -47,6 +55,84 @@ class CentOSConfigurator(configurator.ApacheConfigurator): self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) + def _deploy_cert(self, *args, **kwargs): + """ + Override _deploy_cert in order to ensure that the Apache configuration + has "LoadModule ssl_module..." before parsing the VirtualHost configuration + that was created by Certbot + """ + super(CentOSConfigurator, self)._deploy_cert(*args, **kwargs) + if self.version < (2, 4, 0): + self._deploy_loadmodule_ssl_if_needed() + + + def _deploy_loadmodule_ssl_if_needed(self): + """ + Add "LoadModule ssl_module " to main httpd.conf if + it doesn't exist there already. + """ + + loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False) + + correct_ifmods = [] # type: List[str] + loadmod_args = [] # type: List[str] + loadmod_paths = [] # type: List[str] + for m in loadmods: + noarg_path = m.rpartition("/")[0] + path_args = self.parser.get_all_args(noarg_path) + if loadmod_args: + if loadmod_args != path_args: + msg = ("Certbot encountered multiple LoadModule directives " + "for LoadModule ssl_module with differing library paths. " + "Please remove or comment out the one(s) that are not in " + "use, and run Certbot again.") + raise MisconfigurationError(msg) + else: + loadmod_args = path_args + + if self.parser.not_modssl_ifmodule(noarg_path): # pylint: disable=no-member + if self.parser.loc["default"] in noarg_path: + # LoadModule already in the main configuration file + if ("ifmodule/" in noarg_path.lower() or + "ifmodule[1]" in noarg_path.lower()): + # It's the first or only IfModule in the file + return + # Populate the list of known !mod_ssl.c IfModules + nodir_path = noarg_path.rpartition("/directive")[0] + correct_ifmods.append(nodir_path) + else: + loadmod_paths.append(noarg_path) + + if not loadmod_args: + # Do not try to enable mod_ssl + return + + # Force creation as the directive wasn't found from the beginning of + # httpd.conf + rootconf_ifmod = self.parser.create_ifmod( + parser.get_aug_path(self.parser.loc["default"]), + "!mod_ssl.c", beginning=True) + # parser.get_ifmod returns a path postfixed with "/", remove that + self.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", loadmod_args) + correct_ifmods.append(rootconf_ifmod[:-1]) + self.save_notes += "Added LoadModule ssl_module to main configuration.\n" + + # Wrap LoadModule mod_ssl inside of if it's not + # configured like this already. + for loadmod_path in loadmod_paths: + nodir_path = loadmod_path.split("/directive")[0] + # Remove the old LoadModule directive + self.aug.remove(loadmod_path) + + # Create a new IfModule !mod_ssl.c if not already found on path + ssl_ifmod = self.parser.get_ifmod(nodir_path, "!mod_ssl.c", + beginning=True)[:-1] + if ssl_ifmod not in correct_ifmods: + self.parser.add_dir(ssl_ifmod, "LoadModule", loadmod_args) + correct_ifmods.append(ssl_ifmod) + self.save_notes += ("Wrapped pre-existing LoadModule ssl_module " + "inside of block.\n") + class CentOSParser(parser.ApacheParser): """CentOS specific ApacheParser override class""" @@ -66,3 +152,33 @@ class CentOSParser(parser.ApacheParser): defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS") for k in defines.keys(): self.variables[k] = defines[k] + + def not_modssl_ifmodule(self, path): + """Checks if the provided Augeas path has argument !mod_ssl""" + + if "ifmodule" not in path.lower(): + return False + + # Trim the path to the last ifmodule + workpath = path.lower() + while workpath: + # Get path to the last IfModule (ignore the tail) + parts = workpath.rpartition("ifmodule") + + if not parts[0]: + # IfModule not found + break + ifmod_path = parts[0] + parts[1] + # Check if ifmodule had an index + if parts[2].startswith("["): + # Append the index from tail + ifmod_path += parts[2].partition("/")[0] + # Get the original path trimmed to correct length + # This is required to preserve cases + ifmod_real_path = path[0:len(ifmod_path)] + if "!mod_ssl.c" in self.get_all_args(ifmod_real_path): + return True + # Set the workpath to the heading part + workpath = parts[0] + + return False diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index b9f23f6cf..b025396ad 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -281,7 +281,7 @@ class ApacheParser(object): """ # TODO: Add error checking code... does the path given even exist? # Does it throw exceptions? - if_mod_path = self._get_ifmod(aug_conf_path, "mod_ssl.c") + if_mod_path = self.get_ifmod(aug_conf_path, "mod_ssl.c") # IfModule can have only one valid argument, so append after self.aug.insert(if_mod_path + "arg", "directive", False) nvh_path = if_mod_path + "directive[1]" @@ -292,22 +292,54 @@ class ApacheParser(object): for i, arg in enumerate(args): self.aug.set("%s/arg[%d]" % (nvh_path, i + 1), arg) - def _get_ifmod(self, aug_conf_path, mod): + def get_ifmod(self, aug_conf_path, mod, beginning=False): """Returns the path to and creates one if it doesn't exist. :param str aug_conf_path: Augeas configuration path :param str mod: module ie. mod_ssl.c + :param bool beginning: If the IfModule should be created to the beginning + of augeas path DOM tree. + + :returns: Augeas path of the requested IfModule directive that pre-existed + or was created during the process. The path may be dynamic, + i.e. .../IfModule[last()] + :rtype: str """ if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" % (aug_conf_path, mod))) - if len(if_mods) == 0: - self.aug.set("%s/IfModule[last() + 1]" % aug_conf_path, "") - self.aug.set("%s/IfModule[last()]/arg" % aug_conf_path, mod) - if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" % - (aug_conf_path, mod))) + if not if_mods: + return self.create_ifmod(aug_conf_path, mod, beginning) + # Strip off "arg" at end of first ifmod path - return if_mods[0][:len(if_mods[0]) - 3] + return if_mods[0].rpartition("arg")[0] + + def create_ifmod(self, aug_conf_path, mod, beginning=False): + """Creates a new and returns its path. + + :param str aug_conf_path: Augeas configuration path + :param str mod: module ie. mod_ssl.c + :param bool beginning: If the IfModule should be created to the beginning + of augeas path DOM tree. + + :returns: Augeas path of the newly created IfModule directive. + The path may be dynamic, i.e. .../IfModule[last()] + :rtype: str + + """ + if beginning: + c_path_arg = "{}/IfModule[1]/arg".format(aug_conf_path) + # Insert IfModule before the first directive + self.aug.insert("{}/directive[1]".format(aug_conf_path), + "IfModule", True) + retpath = "{}/IfModule[1]/".format(aug_conf_path) + else: + c_path = "{}/IfModule[last() + 1]".format(aug_conf_path) + c_path_arg = "{}/IfModule[last()]/arg".format(aug_conf_path) + self.aug.set(c_path, "") + retpath = "{}/IfModule[last()]/".format(aug_conf_path) + self.aug.set(c_path_arg, mod) + return retpath def add_dir(self, aug_conf_path, directive, args): """Appends directive to the end fo the file given by aug_conf_path. @@ -453,6 +485,20 @@ class ApacheParser(object): return ordered_matches + def get_all_args(self, match): + """ + Tries to fetch all arguments for a directive. See get_arg. + + Note that if match is an ancestor node, it returns all names of + child directives as well as the list of arguments. + + """ + + if match[-1] != "/": + match = match+"/" + allargs = self.aug.match(match + '*') + return [self.get_arg(arg) for arg in allargs] + def get_arg(self, match): """Uses augeas.get to get argument value and interprets result. diff --git a/certbot-apache/certbot_apache/tests/centos6_test.py b/certbot-apache/certbot_apache/tests/centos6_test.py new file mode 100644 index 000000000..ea8a85ed7 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/centos6_test.py @@ -0,0 +1,224 @@ +"""Test for certbot_apache.configurator for CentOS 6 overrides""" +import os +import unittest + +from certbot_apache import obj +from certbot_apache import override_centos +from certbot_apache import parser +from certbot_apache.tests import util +from certbot.errors import MisconfigurationError + +def get_vh_truth(temp_dir, config_name): + """Return the ground truth for the specified directory.""" + prefix = os.path.join( + temp_dir, config_name, "httpd/conf.d") + + aug_pre = "/files" + prefix + vh_truth = [ + obj.VirtualHost( + os.path.join(prefix, "test.example.com.conf"), + os.path.join(aug_pre, "test.example.com.conf/VirtualHost"), + set([obj.Addr.fromstring("*:80")]), + False, True, "test.example.com"), + obj.VirtualHost( + os.path.join(prefix, "ssl.conf"), + os.path.join(aug_pre, "ssl.conf/VirtualHost"), + set([obj.Addr.fromstring("_default_:443")]), + True, True, None) + ] + return vh_truth + +class CentOS6Tests(util.ApacheTest): + """Tests for CentOS 6""" + + def setUp(self): # pylint: disable=arguments-differ + test_dir = "centos6_apache/apache" + config_root = "centos6_apache/apache/httpd" + vhost_root = "centos6_apache/apache/httpd/conf.d" + super(CentOS6Tests, self).setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + version=(2, 2, 15), os_info="centos") + self.vh_truth = get_vh_truth( + self.temp_dir, "centos6_apache/apache") + + def test_get_parser(self): + self.assertTrue(isinstance(self.config.parser, + override_centos.CentOSParser)) + + def test_get_virtual_hosts(self): + """Make sure all vhosts are being properly found.""" + vhs = self.config.get_virtual_hosts() + self.assertEqual(len(vhs), 2) + found = 0 + + for vhost in vhs: + for centos_truth in self.vh_truth: + if vhost == centos_truth: + found += 1 + break + else: + raise Exception("Missed: %s" % vhost) # pragma: no cover + self.assertEqual(found, 2) + + def test_loadmod_default(self): + ssl_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", exclude=False) + self.assertEqual(len(ssl_loadmods), 1) + # Make sure the LoadModule ssl_module is in ssl.conf (default) + self.assertTrue("ssl.conf" in ssl_loadmods[0]) + # ...and that it's not inside of + self.assertFalse("IfModule" in ssl_loadmods[0]) + + # Get the example vhost + self.config.assoc["test.example.com"] = self.vh_truth[0] + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.config.save() + + post_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", exclude=False) + + # We should now have LoadModule ssl_module in root conf and ssl.conf + self.assertEqual(len(post_loadmods), 2) + for lm in post_loadmods: + # lm[:-7] removes "/arg[#]" from the path + arguments = self.config.parser.get_all_args(lm[:-7]) + self.assertEqual(arguments, ["ssl_module", "modules/mod_ssl.so"]) + # ...and both of them should be wrapped in + # lm[:-17] strips off /directive/arg[1] from the path. + ifmod_args = self.config.parser.get_all_args(lm[:-17]) + self.assertTrue("!mod_ssl.c" in ifmod_args) + + def test_loadmod_multiple(self): + sslmod_args = ["ssl_module", "modules/mod_ssl.so"] + # Adds another LoadModule to main httpd.conf in addtition to ssl.conf + self.config.parser.add_dir(self.config.parser.loc["default"], "LoadModule", + sslmod_args) + self.config.save() + pre_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", exclude=False) + # LoadModules are not within IfModule blocks + self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods])) + self.config.assoc["test.example.com"] = self.vh_truth[0] + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + post_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", exclude=False) + + for mod in post_loadmods: + self.assertTrue(self.config.parser.not_modssl_ifmodule(mod)) #pylint: disable=no-member + + def test_loadmod_rootconf_exists(self): + sslmod_args = ["ssl_module", "modules/mod_ssl.so"] + rootconf_ifmod = self.config.parser.get_ifmod( + parser.get_aug_path(self.config.parser.loc["default"]), + "!mod_ssl.c", beginning=True) + self.config.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", sslmod_args) + self.config.save() + # Get the example vhost + self.config.assoc["test.example.com"] = self.vh_truth[0] + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.config.save() + + root_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", + start=parser.get_aug_path(self.config.parser.loc["default"]), + exclude=False) + + mods = [lm for lm in root_loadmods if self.config.parser.loc["default"] in lm] + + self.assertEqual(len(mods), 1) + # [:-7] removes "/arg[#]" from the path + self.assertEqual( + self.config.parser.get_all_args(mods[0][:-7]), + sslmod_args) + + def test_neg_loadmod_already_on_path(self): + loadmod_args = ["ssl_module", "modules/mod_ssl.so"] + ifmod = self.config.parser.get_ifmod( + self.vh_truth[1].path, "!mod_ssl.c", beginning=True) + self.config.parser.add_dir(ifmod[:-1], "LoadModule", loadmod_args) + self.config.parser.add_dir(self.vh_truth[1].path, "LoadModule", loadmod_args) + self.config.save() + pre_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", start=self.vh_truth[1].path, exclude=False) + self.assertEqual(len(pre_loadmods), 2) + # The ssl.conf now has two LoadModule directives, one inside of + # !mod_ssl.c IfModule + self.config.assoc["test.example.com"] = self.vh_truth[0] + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + self.config.save() + # Ensure that the additional LoadModule wasn't written into the IfModule + post_loadmods = self.config.parser.find_dir( + "LoadModule", "ssl_module", start=self.vh_truth[1].path, exclude=False) + self.assertEqual(len(post_loadmods), 1) + + + + + + def test_loadmod_non_duplicate(self): + # the modules/mod_ssl.so exists in ssl.conf + sslmod_args = ["ssl_module", "modules/mod_somethingelse.so"] + rootconf_ifmod = self.config.parser.get_ifmod( + parser.get_aug_path(self.config.parser.loc["default"]), + "!mod_ssl.c", beginning=True) + self.config.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", sslmod_args) + self.config.save() + self.config.assoc["test.example.com"] = self.vh_truth[0] + pre_matches = self.config.parser.find_dir("LoadModule", + "ssl_module", exclude=False) + + self.assertRaises(MisconfigurationError, self.config.deploy_cert, + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + + post_matches = self.config.parser.find_dir("LoadModule", + "ssl_module", exclude=False) + # Make sure that none was changed + self.assertEqual(pre_matches, post_matches) + + def test_loadmod_not_found(self): + # Remove all existing LoadModule ssl_module... directives + orig_loadmods = self.config.parser.find_dir("LoadModule", + "ssl_module", + exclude=False) + for mod in orig_loadmods: + noarg_path = mod.rpartition("/")[0] + self.config.aug.remove(noarg_path) + self.config.save() + self.config.deploy_cert( + "random.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + + post_loadmods = self.config.parser.find_dir("LoadModule", + "ssl_module", + exclude=False) + self.assertFalse(post_loadmods) + + def test_no_ifmod_search_false(self): + #pylint: disable=no-member + + self.assertFalse(self.config.parser.not_modssl_ifmodule( + "/path/does/not/include/ifmod" + )) + self.assertFalse(self.config.parser.not_modssl_ifmodule( + "" + )) + self.assertFalse(self.config.parser.not_modssl_ifmodule( + "/path/includes/IfModule/but/no/arguments" + )) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/README b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/README new file mode 100644 index 000000000..c12e149f2 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/README @@ -0,0 +1,9 @@ + +This directory holds Apache 2.0 module-specific configuration files; +any files in this directory which have the ".conf" extension will be +processed as Apache configuration files. + +Files are processed in alphabetical order, so if using configuration +directives which depend on, say, mod_perl being loaded, ensure that +these are placed in a filename later in the sort order than "perl.conf". + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/ssl.conf b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/ssl.conf new file mode 100644 index 000000000..fb2174af1 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/ssl.conf @@ -0,0 +1,222 @@ +# +# This is the Apache server configuration file providing SSL support. +# It contains the configuration directives to instruct the server how to +# serve pages over an https connection. For detailing information about these +# directives see +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# + +LoadModule ssl_module modules/mod_ssl.so + +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +Listen 443 + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog builtin + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000) +SSLSessionCacheTimeout 300 + +# Semaphore: +# Configure the path to the mutual exclusion semaphore the +# SSL engine uses internally for inter-process synchronization. +SSLMutex default + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + + + +# General setup for the virtual host, inherited from global configuration +#DocumentRoot "/var/www/html" +#ServerName www.example.com:443 + +# Use separate log files for the SSL virtual host; note that LogLevel +# is not inherited from httpd.conf. +ErrorLog logs/ssl_error_log +TransferLog logs/ssl_access_log +LogLevel warn + +# SSL Engine Switch: +# Enable/Disable SSL for this virtual host. +SSLEngine on + +# SSL Protocol support: +# List the enable protocol levels with which clients will be able to +# connect. Disable SSLv2 access by default: +SSLProtocol all -SSLv2 + +# SSL Cipher Suite: +# List the ciphers that the client is permitted to negotiate. +# See the mod_ssl documentation for a complete list. +SSLCipherSuite DEFAULT:!EXP:!SSLv2:!DES:!IDEA:!SEED:+3DES + +# Server Certificate: +# Point SSLCertificateFile at a PEM encoded certificate. If +# the certificate is encrypted, then you will be prompted for a +# pass phrase. Note that a kill -HUP will prompt again. A new +# certificate can be generated using the genkey(1) command. +SSLCertificateFile /etc/pki/tls/certs/localhost.crt + +# Server Private Key: +# If the key is not combined with the certificate, use this +# directive to point at the key file. Keep in mind that if +# you've both a RSA and a DSA private key you can configure +# both in parallel (to also allow the use of DSA ciphers, etc.) +SSLCertificateKeyFile /etc/pki/tls/private/localhost.key + +# Server Certificate Chain: +# Point SSLCertificateChainFile at a file containing the +# concatenation of PEM encoded CA certificates which form the +# certificate chain for the server certificate. Alternatively +# the referenced file can be the same as SSLCertificateFile +# when the CA certificates are directly appended to the server +# certificate for convinience. +#SSLCertificateChainFile /etc/pki/tls/certs/server-chain.crt + +# Certificate Authority (CA): +# Set the CA certificate verification path where to find CA +# certificates for client authentication or alternatively one +# huge file containing all of them (file must be PEM encoded) +#SSLCACertificateFile /etc/pki/tls/certs/ca-bundle.crt + +# Client Authentication (Type): +# Client certificate verification type and depth. Types are +# none, optional, require and optional_no_ca. Depth is a +# number which specifies how deeply to verify the certificate +# issuer chain before deciding the certificate is not valid. +#SSLVerifyClient require +#SSLVerifyDepth 10 + +# Access Control: +# With SSLRequire you can do per-directory access control based +# on arbitrary complex boolean expressions containing server +# variable checks and other lookup directives. The syntax is a +# mixture between C and Perl. See the mod_ssl documentation +# for more details. +# +#SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ +# and %{SSL_CLIENT_S_DN_O} eq "Snake Oil, Ltd." \ +# and %{SSL_CLIENT_S_DN_OU} in {"Staff", "CA", "Dev"} \ +# and %{TIME_WDAY} >= 1 and %{TIME_WDAY} <= 5 \ +# and %{TIME_HOUR} >= 8 and %{TIME_HOUR} <= 20 ) \ +# or %{REMOTE_ADDR} =~ m/^192\.76\.162\.[0-9]+$/ +# + +# SSL Engine Options: +# Set various options for the SSL engine. +# o FakeBasicAuth: +# Translate the client X.509 into a Basic Authorisation. This means that +# the standard Auth/DBMAuth methods can be used for access control. The +# user name is the `one line' version of the client's X.509 certificate. +# Note that no password is obtained from the user. Every entry in the user +# file needs this password: `xxj31ZMTZzkVA'. +# o ExportCertData: +# This exports two additional environment variables: SSL_CLIENT_CERT and +# SSL_SERVER_CERT. These contain the PEM-encoded certificates of the +# server (always existing) and the client (only existing when client +# authentication is used). This can be used to import the certificates +# into CGI scripts. +# o StdEnvVars: +# This exports the standard SSL/TLS related `SSL_*' environment variables. +# Per default this exportation is switched off for performance reasons, +# because the extraction step is an expensive operation and is usually +# useless for serving static content. So one usually enables the +# exportation for CGI and SSI requests only. +# o StrictRequire: +# This denies access when "SSLRequireSSL" or "SSLRequire" applied even +# under a "Satisfy any" situation, i.e. when it applies access is denied +# and no other module can change it. +# o OptRenegotiate: +# This enables optimized SSL connection renegotiation handling when SSL +# directives are used in per-directory context. +#SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire + + SSLOptions +StdEnvVars + + + SSLOptions +StdEnvVars + + +# SSL Protocol Adjustments: +# The safe and default but still SSL/TLS standard compliant shutdown +# approach is that mod_ssl sends the close notify alert but doesn't wait for +# the close notify alert from client. When you need a different shutdown +# approach you can use one of the following variables: +# o ssl-unclean-shutdown: +# This forces an unclean shutdown when the connection is closed, i.e. no +# SSL close notify alert is send or allowed to received. This violates +# the SSL/TLS standard but is needed for some brain-dead browsers. Use +# this when you receive I/O errors because of the standard approach where +# mod_ssl sends the close notify alert. +# o ssl-accurate-shutdown: +# This forces an accurate shutdown when the connection is closed, i.e. a +# SSL close notify alert is send and mod_ssl waits for the close notify +# alert of the client. This is 100% SSL/TLS standard compliant, but in +# practice often causes hanging connections with brain-dead browsers. Use +# this only for browsers where you know that their SSL implementation +# works correctly. +# Notice: Most problems of broken clients are also related to the HTTP +# keep-alive facility, so you usually additionally want to disable +# keep-alive for those clients, too. Use variable "nokeepalive" for this. +# Similarly, one has to force some clients to use HTTP/1.0 to workaround +# their broken HTTP/1.1 implementation. Use variables "downgrade-1.0" and +# "force-response-1.0" for this. +SetEnvIf User-Agent ".*MSIE.*" \ + nokeepalive ssl-unclean-shutdown \ + downgrade-1.0 force-response-1.0 + +# Per-Server Logging: +# The home of a custom SSL log file. Use this when you want a +# compact non-error SSL logfile on a virtual host basis. +CustomLog logs/ssl_request_log \ + "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + + + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/test.example.com.conf b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/test.example.com.conf new file mode 100644 index 000000000..3dd7b18f1 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/test.example.com.conf @@ -0,0 +1,7 @@ + + ServerName test.example.com + ServerAdmin webmaster@dummy-host.example.com + DocumentRoot /var/www/htdocs + ErrorLog logs/dummy-host.example.com-error_log + CustomLog logs/dummy-host.example.com-access_log common + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/welcome.conf b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/welcome.conf new file mode 100644 index 000000000..c1d23c512 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/welcome.conf @@ -0,0 +1,11 @@ +# +# This configuration file enables the default "Welcome" +# page if there is no default index page present for +# the root URL. To disable the Welcome page, comment +# out all the lines below. +# + + Options -Indexes + ErrorDocument 403 /error/noindex.html + + diff --git a/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf/httpd.conf b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf/httpd.conf new file mode 100644 index 000000000..579d194ce --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf/httpd.conf @@ -0,0 +1,1009 @@ +# +# This is the main Apache server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# The configuration directives are grouped into three basic sections: +# 1. Directives that control the operation of the Apache server process as a +# whole (the 'global environment'). +# 2. Directives that define the parameters of the 'main' or 'default' server, +# which responds to requests that aren't handled by a virtual host. +# These directives also provide default values for the settings +# of all virtual hosts. +# 3. Settings for virtual hosts, which allow Web requests to be sent to +# different IP addresses or hostnames and have them handled by the +# same Apache server process. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/foo.log" +# with ServerRoot set to "/etc/httpd" will be interpreted by the +# server as "/etc/httpd/logs/foo.log". +# + +### Section 1: Global Environment +# +# The directives in this section affect the overall operation of Apache, +# such as the number of concurrent requests it can handle or where it +# can find its configuration files. +# + +# +# Don't give away too much information about all the subcomponents +# we are running. Comment out this line if you don't mind remote sites +# finding out what major optional modules you are running +ServerTokens OS + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# NOTE! If you intend to place this on an NFS (or otherwise network) +# mounted filesystem then please read the LockFile documentation +# (available at ); +# you will save yourself a lot of trouble. +# +# Do NOT add a slash at the end of the directory path. +# +ServerRoot "/etc/httpd" + +# +# PidFile: The file in which the server should record its process +# identification number when it starts. Note the PIDFILE variable in +# /etc/sysconfig/httpd must be set appropriately if this location is +# changed. +# +PidFile run/httpd.pid + +# +# Timeout: The number of seconds before receives and sends time out. +# +Timeout 60 + +# +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +# +KeepAlive Off + +# +# MaxKeepAliveRequests: The maximum number of requests to allow +# during a persistent connection. Set to 0 to allow an unlimited amount. +# We recommend you leave this number high, for maximum performance. +# +MaxKeepAliveRequests 100 + +# +# KeepAliveTimeout: Number of seconds to wait for the next request from the +# same client on the same connection. +# +KeepAliveTimeout 15 + +## +## Server-Pool Size Regulation (MPM specific) +## + +# prefork MPM +# StartServers: number of server processes to start +# MinSpareServers: minimum number of server processes which are kept spare +# MaxSpareServers: maximum number of server processes which are kept spare +# ServerLimit: maximum value for MaxClients for the lifetime of the server +# MaxClients: maximum number of server processes allowed to start +# MaxRequestsPerChild: maximum number of requests a server process serves + +StartServers 8 +MinSpareServers 5 +MaxSpareServers 20 +ServerLimit 256 +MaxClients 256 +MaxRequestsPerChild 4000 + + +# worker MPM +# StartServers: initial number of server processes to start +# MaxClients: maximum number of simultaneous client connections +# MinSpareThreads: minimum number of worker threads which are kept spare +# MaxSpareThreads: maximum number of worker threads which are kept spare +# ThreadsPerChild: constant number of worker threads in each server process +# MaxRequestsPerChild: maximum number of requests a server process serves + +StartServers 4 +MaxClients 300 +MinSpareThreads 25 +MaxSpareThreads 75 +ThreadsPerChild 25 +MaxRequestsPerChild 0 + + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, in addition to the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses (0.0.0.0) +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule auth_digest_module modules/mod_auth_digest.so +LoadModule authn_file_module modules/mod_authn_file.so +LoadModule authn_alias_module modules/mod_authn_alias.so +LoadModule authn_anon_module modules/mod_authn_anon.so +LoadModule authn_dbm_module modules/mod_authn_dbm.so +LoadModule authn_default_module modules/mod_authn_default.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_user_module modules/mod_authz_user.so +LoadModule authz_owner_module modules/mod_authz_owner.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_dbm_module modules/mod_authz_dbm.so +LoadModule authz_default_module modules/mod_authz_default.so +LoadModule ldap_module modules/mod_ldap.so +LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +LoadModule include_module modules/mod_include.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule logio_module modules/mod_logio.so +LoadModule env_module modules/mod_env.so +LoadModule ext_filter_module modules/mod_ext_filter.so +LoadModule mime_magic_module modules/mod_mime_magic.so +LoadModule expires_module modules/mod_expires.so +LoadModule deflate_module modules/mod_deflate.so +LoadModule headers_module modules/mod_headers.so +LoadModule usertrack_module modules/mod_usertrack.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule mime_module modules/mod_mime.so +LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +LoadModule info_module modules/mod_info.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule vhost_alias_module modules/mod_vhost_alias.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +LoadModule actions_module modules/mod_actions.so +LoadModule speling_module modules/mod_speling.so +LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +LoadModule substitute_module modules/mod_substitute.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule proxy_module modules/mod_proxy.so +LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +LoadModule proxy_http_module modules/mod_proxy_http.so +LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +LoadModule proxy_connect_module modules/mod_proxy_connect.so +LoadModule cache_module modules/mod_cache.so +LoadModule suexec_module modules/mod_suexec.so +LoadModule disk_cache_module modules/mod_disk_cache.so +LoadModule cgi_module modules/mod_cgi.so +LoadModule version_module modules/mod_version.so + +# +# The following modules are not loaded by default: +# +#LoadModule asis_module modules/mod_asis.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule cern_meta_module modules/mod_cern_meta.so +#LoadModule cgid_module modules/mod_cgid.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule filter_module modules/mod_filter.so +#LoadModule ident_module modules/mod_ident.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +#LoadModule unique_id_module modules/mod_unique_id.so +# + +# +# Load config files from the config directory "/etc/httpd/conf.d". +# +Include conf.d/*.conf + +# +# ExtendedStatus controls whether Apache will generate "full" status +# information (ExtendedStatus On) or just basic information (ExtendedStatus +# Off) when the "server-status" handler is called. The default is Off. +# +#ExtendedStatus On + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# . On SCO (ODT 3) use "User nouser" and "Group nogroup". +# . On HPUX you may not be able to use shared memory as nobody, and the +# suggested workaround is to create a user www and use that user. +# NOTE that some kernels refuse to setgid(Group) or semctl(IPC_SET) +# when the value of (unsigned)Group is above 60000; +# don't use Group #-1 on these systems! +# +User apache +Group apache + +### Section 2: 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin root@localhost + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If this is not set to valid DNS name for your host, server-generated +# redirections will not work. See also the UseCanonicalName directive. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# You will have to access it by its address anyway, and this will make +# redirections work in a sensible way. +# +#ServerName www.example.com:80 + +# +# UseCanonicalName: Determines how Apache constructs self-referencing +# URLs and the SERVER_NAME and SERVER_PORT variables. +# When set "Off", Apache will use the Hostname and Port supplied +# by the client. When set "On", Apache will use the value of the +# ServerName directive. +# +UseCanonicalName Off + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/var/www/html" + +# +# Each directory to which Apache has access can be configured with respect +# to which services and features are allowed and/or disabled in that +# directory (and its subdirectories). +# +# First, we configure the "default" to be a very restrictive set of +# features. +# + + Options FollowSymLinks + AllowOverride None + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# This should be changed to whatever you set DocumentRoot to. +# + + +# +# Possible values for the Options directive are "None", "All", +# or any combination of: +# Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews +# +# Note that "MultiViews" must be named *explicitly* --- "Options All" +# doesn't give it to you. +# +# The Options directive is both complicated and important. Please see +# http://httpd.apache.org/docs/2.2/mod/core.html#options +# for more information. +# + Options Indexes FollowSymLinks + +# +# AllowOverride controls what directives may be placed in .htaccess files. +# It can be "All", "None", or any combination of the keywords: +# Options FileInfo AuthConfig Limit +# + AllowOverride None + +# +# Controls who can get stuff from this server. +# + Order allow,deny + Allow from all + + + +# +# UserDir: The name of the directory that is appended onto a user's home +# directory if a ~user request is received. +# +# The path to the end user account 'public_html' directory must be +# accessible to the webserver userid. This usually means that ~userid +# must have permissions of 711, ~userid/public_html must have permissions +# of 755, and documents contained therein must be world-readable. +# Otherwise, the client will only receive a "403 Forbidden" message. +# +# See also: http://httpd.apache.org/docs/misc/FAQ.html#forbidden +# + + # + # UserDir is disabled by default since it can confirm the presence + # of a username on the system (depending on home directory + # permissions). + # + UserDir disabled + + # + # To enable requests to /~user/ to serve the user's public_html + # directory, remove the "UserDir disabled" line above, and uncomment + # the following line instead: + # + #UserDir public_html + + + +# +# Control access to UserDir directories. The following is an example +# for a site where these directories are restricted to read-only. +# +# +# AllowOverride FileInfo AuthConfig Limit +# Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec +# +# Order allow,deny +# Allow from all +# +# +# Order deny,allow +# Deny from all +# +# + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# +# The index.html.var file (a type-map) is used to deliver content- +# negotiated documents. The MultiViews Option can be used for the +# same purpose, but it is much slower. +# +DirectoryIndex index.html index.html.var + +# +# AccessFileName: The name of the file to look for in each directory +# for additional configuration directives. See also the AllowOverride +# directive. +# +AccessFileName .htaccess + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Order allow,deny + Deny from all + Satisfy All + + +# +# TypesConfig describes where the mime.types file (or equivalent) is +# to be found. +# +TypesConfig /etc/mime.types + +# +# DefaultType is the default MIME type the server will use for a document +# if it cannot otherwise determine one, such as from filename extensions. +# If your server contains mostly text or HTML documents, "text/plain" is +# a good value. If most of your content is binary, such as applications +# or images, you may want to use "application/octet-stream" instead to +# keep browsers from trying to display binary files as though they are +# text. +# +DefaultType text/plain + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# + +# MIMEMagicFile /usr/share/magic.mime + MIMEMagicFile conf/magic + + +# +# HostnameLookups: Log the names of clients or just their IP addresses +# e.g., www.apache.org (on) or 204.62.129.132 (off). +# The default is off because it'd be overall better for the net if people +# had to knowingly turn this feature on, since enabling it means that +# each client request will result in AT LEAST one lookup request to the +# nameserver. +# +HostnameLookups Off + +# +# EnableMMAP: Control whether memory-mapping is used to deliver +# files (assuming that the underlying OS supports it). +# The default is on; turn this off if you serve from NFS-mounted +# filesystems. On some systems, turning it off (regardless of +# filesystem) can improve performance; for details, please see +# http://httpd.apache.org/docs/2.2/mod/core.html#enablemmap +# +#EnableMMAP off + +# +# EnableSendfile: Control whether the sendfile kernel support is +# used to deliver files (assuming that the OS supports it). +# The default is on; turn this off if you serve from NFS-mounted +# filesystems. Please see +# http://httpd.apache.org/docs/2.2/mod/core.html#enablesendfile +# +#EnableSendfile off + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog logs/error_log + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + +# +# The following directives define some format nicknames for use with +# a CustomLog directive (see below). +# +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + +# "combinedio" includes actual counts of actual bytes received (%I) and sent (%O); this +# requires the mod_logio module to be loaded. +#LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + +# +# The location and format of the access logfile (Common Logfile Format). +# If you do not define any access logfiles within a +# container, they will be logged here. Contrariwise, if you *do* +# define per- access logfiles, transactions will be +# logged therein and *not* in this file. +# +#CustomLog logs/access_log common + +# +# If you would like to have separate agent and referer logfiles, uncomment +# the following directives. +# +#CustomLog logs/referer_log referer +#CustomLog logs/agent_log agent + +# +# For a single logfile with access, agent, and referer information +# (Combined Logfile Format), use the following directive: +# +CustomLog logs/access_log combined + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (internal error documents, FTP directory +# listings, mod_status and mod_info output etc., but not CGI generated +# documents or custom error documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +# +ServerSignature On + +# +# Aliases: Add here as many aliases as you need (with no limit). The format is +# Alias fakename realname +# +# Note that if you include a trailing / on fakename then the server will +# require it to be present in the URL. So "/icons" isn't aliased in this +# example, only "/icons/". If the fakename is slash-terminated, then the +# realname must also be slash terminated, and if the fakename omits the +# trailing slash, the realname must also omit it. +# +# We include the /icons/ alias for FancyIndexed directory listings. If you +# do not use FancyIndexing, you may comment this out. +# +Alias /icons/ "/var/www/icons/" + + + Options Indexes MultiViews FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + + +# +# WebDAV module configuration section. +# + + # Location of the WebDAV lock database. + DAVLockDB /var/lib/dav/lockdb + + +# +# ScriptAlias: This controls which directories contain server scripts. +# ScriptAliases are essentially the same as Aliases, except that +# documents in the realname directory are treated as applications and +# run by the server when requested rather than as documents sent to the client. +# The same rules about trailing "/" apply to ScriptAlias directives as to +# Alias. +# +ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" + +# +# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Order allow,deny + Allow from all + + +# +# Redirect allows you to tell clients about documents which used to exist in +# your server's namespace, but do not anymore. This allows you to tell the +# clients where to look for the relocated document. +# Example: +# Redirect permanent /foo http://www.example.com/bar + +# +# Directives controlling the display of server-generated directory listings. +# + +# +# IndexOptions: Controls the appearance of server-generated directory +# listings. +# +IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8 + +# +# AddIcon* directives tell the server which icon to show for different +# files or filename extensions. These are only displayed for +# FancyIndexed directories. +# +AddIconByEncoding (CMP,/icons/compressed.gif) x-compress x-gzip + +AddIconByType (TXT,/icons/text.gif) text/* +AddIconByType (IMG,/icons/image2.gif) image/* +AddIconByType (SND,/icons/sound2.gif) audio/* +AddIconByType (VID,/icons/movie.gif) video/* + +AddIcon /icons/binary.gif .bin .exe +AddIcon /icons/binhex.gif .hqx +AddIcon /icons/tar.gif .tar +AddIcon /icons/world2.gif .wrl .wrl.gz .vrml .vrm .iv +AddIcon /icons/compressed.gif .Z .z .tgz .gz .zip +AddIcon /icons/a.gif .ps .ai .eps +AddIcon /icons/layout.gif .html .shtml .htm .pdf +AddIcon /icons/text.gif .txt +AddIcon /icons/c.gif .c +AddIcon /icons/p.gif .pl .py +AddIcon /icons/f.gif .for +AddIcon /icons/dvi.gif .dvi +AddIcon /icons/uuencoded.gif .uu +AddIcon /icons/script.gif .conf .sh .shar .csh .ksh .tcl +AddIcon /icons/tex.gif .tex +AddIcon /icons/bomb.gif /core + +AddIcon /icons/back.gif .. +AddIcon /icons/hand.right.gif README +AddIcon /icons/folder.gif ^^DIRECTORY^^ +AddIcon /icons/blank.gif ^^BLANKICON^^ + +# +# DefaultIcon is which icon to show for files which do not have an icon +# explicitly set. +# +DefaultIcon /icons/unknown.gif + +# +# AddDescription allows you to place a short description after a file in +# server-generated indexes. These are only displayed for FancyIndexed +# directories. +# Format: AddDescription "description" filename +# +#AddDescription "GZIP compressed document" .gz +#AddDescription "tar archive" .tar +#AddDescription "GZIP compressed tar archive" .tgz + +# +# ReadmeName is the name of the README file the server will look for by +# default, and append to directory listings. +# +# HeaderName is the name of a file which should be prepended to +# directory indexes. +ReadmeName README.html +HeaderName HEADER.html + +# +# IndexIgnore is a set of filenames which directory indexing should ignore +# and not include in the listing. Shell-style wildcarding is permitted. +# +IndexIgnore .??* *~ *# HEADER* README* RCS CVS *,v *,t + +# +# DefaultLanguage and AddLanguage allows you to specify the language of +# a document. You can then use content negotiation to give a browser a +# file in a language the user can understand. +# +# Specify a default language. This means that all data +# going out without a specific language tag (see below) will +# be marked with this one. You probably do NOT want to set +# this unless you are sure it is correct for all cases. +# +# * It is generally better to not mark a page as +# * being a certain language than marking it with the wrong +# * language! +# +# DefaultLanguage nl +# +# Note 1: The suffix does not have to be the same as the language +# keyword --- those with documents in Polish (whose net-standard +# language code is pl) may wish to use "AddLanguage pl .po" to +# avoid the ambiguity with the common suffix for perl scripts. +# +# Note 2: The example entries below illustrate that in some cases +# the two character 'Language' abbreviation is not identical to +# the two character 'Country' code for its country, +# E.g. 'Danmark/dk' versus 'Danish/da'. +# +# Note 3: In the case of 'ltz' we violate the RFC by using a three char +# specifier. There is 'work in progress' to fix this and get +# the reference data for rfc1766 cleaned up. +# +# Catalan (ca) - Croatian (hr) - Czech (cs) - Danish (da) - Dutch (nl) +# English (en) - Esperanto (eo) - Estonian (et) - French (fr) - German (de) +# Greek-Modern (el) - Hebrew (he) - Italian (it) - Japanese (ja) +# Korean (ko) - Luxembourgeois* (ltz) - Norwegian Nynorsk (nn) +# Norwegian (no) - Polish (pl) - Portugese (pt) +# Brazilian Portuguese (pt-BR) - Russian (ru) - Swedish (sv) +# Simplified Chinese (zh-CN) - Spanish (es) - Traditional Chinese (zh-TW) +# +AddLanguage ca .ca +AddLanguage cs .cz .cs +AddLanguage da .dk +AddLanguage de .de +AddLanguage el .el +AddLanguage en .en +AddLanguage eo .eo +AddLanguage es .es +AddLanguage et .et +AddLanguage fr .fr +AddLanguage he .he +AddLanguage hr .hr +AddLanguage it .it +AddLanguage ja .ja +AddLanguage ko .ko +AddLanguage ltz .ltz +AddLanguage nl .nl +AddLanguage nn .nn +AddLanguage no .no +AddLanguage pl .po +AddLanguage pt .pt +AddLanguage pt-BR .pt-br +AddLanguage ru .ru +AddLanguage sv .sv +AddLanguage zh-CN .zh-cn +AddLanguage zh-TW .zh-tw + +# +# LanguagePriority allows you to give precedence to some languages +# in case of a tie during content negotiation. +# +# Just list the languages in decreasing order of preference. We have +# more or less alphabetized them here. You probably want to change this. +# +LanguagePriority en ca cs da de el eo es et fr he hr it ja ko ltz nl nn no pl pt pt-BR ru sv zh-CN zh-TW + +# +# ForceLanguagePriority allows you to serve a result page rather than +# MULTIPLE CHOICES (Prefer) [in case of a tie] or NOT ACCEPTABLE (Fallback) +# [in case no accepted languages matched the available variants] +# +ForceLanguagePriority Prefer Fallback + +# +# Specify a default charset for all content served; this enables +# interpretation of all content as UTF-8 by default. To use the +# default browser choice (ISO-8859-1), or to allow the META tags +# in HTML content to override this choice, comment out this +# directive: +# +AddDefaultCharset UTF-8 + +# +# AddType allows you to add to or override the MIME configuration +# file mime.types for specific file types. +# +#AddType application/x-tar .tgz + +# +# AddEncoding allows you to have certain browsers uncompress +# information on the fly. Note: Not all browsers support this. +# Despite the name similarity, the following Add* directives have nothing +# to do with the FancyIndexing customization directives above. +# +#AddEncoding x-compress .Z +#AddEncoding x-gzip .gz .tgz + +# If the AddEncoding directives above are commented-out, then you +# probably should define those extensions to indicate media types: +# +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz + +# +# MIME-types for downloading Certificates and CRLs +# +AddType application/x-x509-ca-cert .crt +AddType application/x-pkcs7-crl .crl + +# +# AddHandler allows you to map certain file extensions to "handlers": +# actions unrelated to filetype. These can be either built into the server +# or added with the Action directive (see below) +# +# To use CGI scripts outside of ScriptAliased directories: +# (You will also need to add "ExecCGI" to the "Options" directive.) +# +#AddHandler cgi-script .cgi + +# +# For files that include their own HTTP headers: +# +#AddHandler send-as-is asis + +# +# For type maps (negotiated resources): +# (This is enabled by default to allow the Apache "It Worked" page +# to be distributed in multiple languages.) +# +AddHandler type-map var + +# +# Filters allow you to process content before it is sent to the client. +# +# To parse .shtml files for server-side includes (SSI): +# (You will also need to add "Includes" to the "Options" directive.) +# +AddType text/html .shtml +AddOutputFilter INCLUDES .shtml + +# +# Action lets you define media types that will execute a script whenever +# a matching file is called. This eliminates the need for repeated URL +# pathnames for oft-used CGI file processors. +# Format: Action media/type /cgi-script/location +# Format: Action handler-name /cgi-script/location +# + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# Putting this all together, we can internationalize error responses. +# +# We use Alias to redirect any /error/HTTP_.html.var response to +# our collection of by-error message multi-language collections. We use +# includes to substitute the appropriate text. +# +# You can modify the messages' appearance without changing any of the +# default HTTP_.html.var files by adding the line: +# +# Alias /error/include/ "/your/include/path/" +# +# which allows you to create your own set of files by starting with the +# /var/www/error/include/ files and +# copying them to /your/include/path/, even on a per-VirtualHost basis. +# + +Alias /error/ "/var/www/error/" + + + + + AllowOverride None + Options IncludesNoExec + AddOutputFilter Includes html + AddHandler type-map var + Order allow,deny + Allow from all + LanguagePriority en es de fr + ForceLanguagePriority Prefer Fallback + + +# ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var +# ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var +# ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var +# ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var +# ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var +# ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var +# ErrorDocument 410 /error/HTTP_GONE.html.var +# ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var +# ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var +# ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var +# ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var +# ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var +# ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var +# ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var +# ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var +# ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var +# ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var + + + + +# +# The following directives modify normal HTTP response behavior to +# handle known problems with browser implementations. +# +BrowserMatch "Mozilla/2" nokeepalive +BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 +BrowserMatch "RealPlayer 4\.0" force-response-1.0 +BrowserMatch "Java/1\.0" force-response-1.0 +BrowserMatch "JDK/1\.0" force-response-1.0 + +# +# The following directive disables redirects on non-GET requests for +# a directory that does not include the trailing slash. This fixes a +# problem with Microsoft WebFolders which does not appropriately handle +# redirects for folders with DAV methods. +# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV. +# +BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully +BrowserMatch "MS FrontPage" redirect-carefully +BrowserMatch "^WebDrive" redirect-carefully +BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully +BrowserMatch "^gnome-vfs/1.0" redirect-carefully +BrowserMatch "^XML Spy" redirect-carefully +BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully + +# +# Allow server status reports generated by mod_status, +# with the URL of http://servername/server-status +# Change the ".example.com" to match your domain to enable. +# +# +# SetHandler server-status +# Order deny,allow +# Deny from all +# Allow from .example.com +# + +# +# Allow remote server configuration reports, with the URL of +# http://servername/server-info (requires that mod_info.c be loaded). +# Change the ".example.com" to match your domain to enable. +# +# +# SetHandler server-info +# Order deny,allow +# Deny from all +# Allow from .example.com +# + +# +# Proxy Server directives. Uncomment the following lines to +# enable the proxy server: +# +# +#ProxyRequests On +# +# +# Order deny,allow +# Deny from all +# Allow from .example.com +# + +# +# Enable/disable the handling of HTTP/1.1 "Via:" headers. +# ("Full" adds the server version; "Block" removes all outgoing Via: headers) +# Set to one of: Off | On | Full | Block +# +#ProxyVia On + +# +# To enable a cache of proxied content, uncomment the following lines. +# See http://httpd.apache.org/docs/2.2/mod/mod_cache.html for more details. +# +# +# CacheEnable disk / +# CacheRoot "/var/cache/mod_proxy" +# +# + +# +# End of proxy directives. + +### Section 3: Virtual Hosts +# +# VirtualHost: If you want to maintain multiple domains/hostnames on your +# machine you can setup VirtualHost containers for them. Most configurations +# use only name-based virtual hosts so the server doesn't need to worry about +# IP addresses. This is indicated by the asterisks in the directives below. +# +# Please see the documentation at +# +# for further details before you try to setup virtual hosts. +# +# You may use the command line option '-S' to verify your virtual host +# configuration. + +# +# Use name-based virtual hosting. +# +#NameVirtualHost *:80 +# +# NOTE: NameVirtualHost cannot be used without a port specifier +# (e.g. :80) if mod_ssl is being used, due to the nature of the +# SSL protocol. +# + +# +# VirtualHost example: +# Almost any Apache directive may go into a VirtualHost container. +# The first VirtualHost section is used for requests without a known +# server name. +# +# +# ServerAdmin webmaster@dummy-host.example.com +# DocumentRoot /www/docs/dummy-host.example.com +# ServerName dummy-host.example.com +# ErrorLog logs/dummy-host.example.com-error_log +# CustomLog logs/dummy-host.example.com-access_log common +#