From bda9a49a6ab8d779e5da86e19b01b8b320650687 Mon Sep 17 00:00:00 2001 From: Philipp Spitzer Date: Wed, 6 Jan 2016 20:42:07 +0100 Subject: [PATCH 01/64] Added apache2 dav and dav_svn modules to debian_apache_2_4 two_vhost_80 test data. Also added symlinks in mods-enabled to the newly added files to include the modules in the test. After doing so, many tests fail with "AttributeError: 'NoneType' object has no attribute 'lower'" in letsencrypt_apache/parser.py", line 311. --- .../apache2/mods-available/authz_svn.load | 5 ++ .../apache2/mods-available/dav.load | 3 + .../apache2/mods-available/dav_svn.conf | 56 +++++++++++++++++++ .../apache2/mods-available/dav_svn.load | 7 +++ .../apache2/mods-enabled/authz_svn.load | 1 + .../apache2/mods-enabled/dav.load | 1 + .../apache2/mods-enabled/dav_svn.conf | 1 + .../apache2/mods-enabled/dav_svn.load | 1 + 8 files changed, 75 insertions(+) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/authz_svn.load create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav.load create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.conf create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.load create mode 120000 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/authz_svn.load create mode 120000 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav.load create mode 120000 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.conf create mode 120000 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.load diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/authz_svn.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/authz_svn.load new file mode 100644 index 000000000..c6df2733b --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/authz_svn.load @@ -0,0 +1,5 @@ +# Depends: dav_svn + + Include mods-enabled/dav_svn.load + +LoadModule authz_svn_module /usr/lib/apache2/modules/mod_authz_svn.so diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav.load new file mode 100644 index 000000000..a5867fff3 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav.load @@ -0,0 +1,3 @@ + + LoadModule dav_module /usr/lib/apache2/modules/mod_dav.so + diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.conf new file mode 100644 index 000000000..801cbd6bd --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.conf @@ -0,0 +1,56 @@ +# dav_svn.conf - Example Subversion/Apache configuration +# +# For details and further options see the Apache user manual and +# the Subversion book. +# +# NOTE: for a setup with multiple vhosts, you will want to do this +# configuration in /etc/apache2/sites-available/*, not here. + +# ... +# URL controls how the repository appears to the outside world. +# In this example clients access the repository as http://hostname/svn/ +# Note, a literal /svn should NOT exist in your document root. +# + + # Uncomment this to enable the repository + #DAV svn + + # Set this to the path to your repository + #SVNPath /var/lib/svn + # Alternatively, use SVNParentPath if you have multiple repositories under + # under a single directory (/var/lib/svn/repo1, /var/lib/svn/repo2, ...). + # You need either SVNPath and SVNParentPath, but not both. + #SVNParentPath /var/lib/svn + + # Access control is done at 3 levels: (1) Apache authentication, via + # any of several methods. A "Basic Auth" section is commented out + # below. (2) Apache and , also commented out + # below. (3) mod_authz_svn is a svn-specific authorization module + # which offers fine-grained read/write access control for paths + # within a repository. (The first two layers are coarse-grained; you + # can only enable/disable access to an entire repository.) Note that + # mod_authz_svn is noticeably slower than the other two layers, so if + # you don't need the fine-grained control, don't configure it. + + # Basic Authentication is repository-wide. It is not secure unless + # you are using https. See the 'htpasswd' command to create and + # manage the password file - and the documentation for the + # 'auth_basic' and 'authn_file' modules, which you will need for this + # (enable them with 'a2enmod'). + #AuthType Basic + #AuthName "Subversion Repository" + #AuthUserFile /etc/apache2/dav_svn.passwd + + # To enable authorization via mod_authz_svn (enable that module separately): + # + #AuthzSVNAccessFile /etc/apache2/dav_svn.authz + # + + # The following three lines allow anonymous read, but make + # committers authenticate themselves. It requires the 'authz_user' + # module (enable it with 'a2enmod'). + # + #Require valid-user + # + +# diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.load new file mode 100644 index 000000000..e41e1581a --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/dav_svn.load @@ -0,0 +1,7 @@ +# Depends: dav + + + Include mods-enabled/dav.load + + LoadModule dav_svn_module /usr/lib/apache2/modules/mod_dav_svn.so + diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/authz_svn.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/authz_svn.load new file mode 120000 index 000000000..7ac0725dd --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/authz_svn.load @@ -0,0 +1 @@ +../mods-available/authz_svn.load \ No newline at end of file diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav.load new file mode 120000 index 000000000..9dcfef6da --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav.load @@ -0,0 +1 @@ +../mods-available/dav.load \ No newline at end of file diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.conf new file mode 120000 index 000000000..964c7bb0b --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.conf @@ -0,0 +1 @@ +../mods-available/dav_svn.conf \ No newline at end of file diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.load new file mode 120000 index 000000000..4094e4173 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/dav_svn.load @@ -0,0 +1 @@ +../mods-available/dav_svn.load \ No newline at end of file From df383ee6e408f3be4bc3beb59aa33abc8e90f268 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 11 Feb 2016 15:40:31 -0800 Subject: [PATCH 02/64] Remove werkzeug dependency by parsing Retry-After ourselves Fixes #2409 Progress on #1301 --- acme/acme/client.py | 27 ++++++++++++++++++++------- acme/setup.py | 1 - 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 478536ecc..7c366df90 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -11,13 +11,16 @@ from six.moves import http_client # pylint: disable=import-error import OpenSSL import requests import sys -import werkzeug from acme import errors from acme import jose from acme import jws from acme import messages +try: + from email.utils import parsedate_tz +except ImportError: # pragma: no cover + from email.Utils import parsedate_tz logger = logging.getLogger(__name__) @@ -245,7 +248,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes @classmethod def retry_after(cls, response, default): - """Compute next `poll` time based on response ``Retry-After`` header. + """Compute next `poll` time based on response ``Retry-After`` header, + per https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37 :param requests.Response response: Response from `poll`. :param int default: Default value (in seconds), used when @@ -256,17 +260,26 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ retry_after = response.headers.get('Retry-After', str(default)) + now = datetime.datetime.now() + year_now = now[0] try: seconds = int(retry_after) except ValueError: # pylint: disable=no-member - decoded = werkzeug.parse_date(retry_after) # RFC1123 - if decoded is None: + t = parsedate_tz(value.strip()) + try: + year = t[0] # raises TypeError if t is None + # Handle two-digit years -- but any webserver that thinks + # "retry after 99" means "come back after 1999" is.. deprecated + if year >= 0 and year < 100: + year += 2000 + t_corrected = datetime(*([year] + t[1:7])) # raises ValueError + tz = t[-1] if t[-1] else 0 + return t_corrected - timedelta(tz) # raises OverflowError + except (TypeError, ValueError, OverflowError): seconds = default - else: - return decoded - return datetime.datetime.now() + datetime.timedelta(seconds=seconds) + return now + datetime.timedelta(seconds=seconds) def poll(self, authzr): """Poll Authorization Resource for status. diff --git a/acme/setup.py b/acme/setup.py index 8b7b040e5..2daf8ad25 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -20,7 +20,6 @@ install_requires = [ 'requests', 'setuptools', # pkg_resources 'six', - 'werkzeug', ] # env markers in extras_require cause problems with older pip: #517 From 0ecaa8abca893509a8dbc0a3aeb0c265398f8d23 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 11 Feb 2016 15:45:51 -0800 Subject: [PATCH 03/64] rm unused var --- acme/acme/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 7c366df90..c545316b1 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -261,7 +261,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ retry_after = response.headers.get('Retry-After', str(default)) now = datetime.datetime.now() - year_now = now[0] try: seconds = int(retry_after) except ValueError: From ef404d49857638bb61fa739f18d11ba566ecaf4c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 11 Feb 2016 16:12:42 -0800 Subject: [PATCH 04/64] slightly simpler / more compact --- acme/acme/client.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c545316b1..c1407c464 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -265,16 +265,15 @@ class Client(object): # pylint: disable=too-many-instance-attributes seconds = int(retry_after) except ValueError: # pylint: disable=no-member - t = parsedate_tz(value.strip()) + t = list(parsedate_tz(value.strip())) # returns None on fail try: year = t[0] # raises TypeError if t is None # Handle two-digit years -- but any webserver that thinks # "retry after 99" means "come back after 1999" is.. deprecated if year >= 0 and year < 100: - year += 2000 - t_corrected = datetime(*([year] + t[1:7])) # raises ValueError + t[0] = year + 2000 tz = t[-1] if t[-1] else 0 - return t_corrected - timedelta(tz) # raises OverflowError + return datetime(*t) - timedelta(tz) # raises Value/OverflowError except (TypeError, ValueError, OverflowError): seconds = default From a34dc94b1ce67f704b7b93556491b4fdea86d7f0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 11 Feb 2016 17:28:07 -0800 Subject: [PATCH 05/64] bugfixes & minimalism --- acme/acme/client.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c1407c464..9f583fda4 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,10 +1,11 @@ """ACME client API.""" import collections -import datetime import heapq import logging import time +from datetime import datetime, timedelta + import six from six.moves import http_client # pylint: disable=import-error @@ -259,13 +260,12 @@ class Client(object): # pylint: disable=too-many-instance-attributes :rtype: `datetime.datetime` """ - retry_after = response.headers.get('Retry-After', str(default)) - now = datetime.datetime.now() + retry_after = response.headers.get('Retry-After', str(default)).strip() try: seconds = int(retry_after) except ValueError: # pylint: disable=no-member - t = list(parsedate_tz(value.strip())) # returns None on fail + t = list(parsedate_tz(retry_after)) try: year = t[0] # raises TypeError if t is None # Handle two-digit years -- but any webserver that thinks @@ -273,11 +273,12 @@ class Client(object): # pylint: disable=too-many-instance-attributes if year >= 0 and year < 100: t[0] = year + 2000 tz = t[-1] if t[-1] else 0 - return datetime(*t) - timedelta(tz) # raises Value/OverflowError + # raises ValueError/OverflowError + return datetime(*t) - timedelta(tz) except (TypeError, ValueError, OverflowError): seconds = default - return now + datetime.timedelta(seconds=seconds) + return datetime.now() + timedelta(seconds=seconds) def poll(self, authzr): """Poll Authorization Resource for status. @@ -369,7 +370,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value - waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] + waiting = [(datetime.now(), authzr) for authzr in authzrs] # mapping between original Authorization Resource and the most # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) @@ -377,7 +378,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) - now = datetime.datetime.now() + now = datetime.now() if when > now: seconds = (when - now).seconds logger.debug('Sleeping for %d seconds', seconds) From 6f99d9f3d9dbf86cdf4a3a1806fc460ea40ce75e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 11 Feb 2016 18:22:41 -0800 Subject: [PATCH 06/64] fixen --- acme/acme/client.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9f583fda4..b07009a11 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -4,7 +4,7 @@ import heapq import logging import time -from datetime import datetime, timedelta +import datetime import six from six.moves import http_client # pylint: disable=import-error @@ -21,7 +21,8 @@ from acme import messages try: from email.utils import parsedate_tz except ImportError: # pragma: no cover - from email.Utils import parsedate_tz + # pylint: disable=import-error,no-name-in-module + from email.Utils import parsedate_tz logger = logging.getLogger(__name__) @@ -257,7 +258,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes ``Retry-After`` header is not present or invalid. :returns: Time point when next `poll` should be performed. - :rtype: `datetime.datetime` + :rtype: `datetime.datetime.datetime` """ retry_after = response.headers.get('Retry-After', str(default)).strip() @@ -265,20 +266,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes seconds = int(retry_after) except ValueError: # pylint: disable=no-member - t = list(parsedate_tz(retry_after)) + when = parsedate_tz(retry_after) try: - year = t[0] # raises TypeError if t is None + year = when[0] # raises TypeError if t is None # Handle two-digit years -- but any webserver that thinks # "retry after 99" means "come back after 1999" is.. deprecated if year >= 0 and year < 100: - t[0] = year + 2000 - tz = t[-1] if t[-1] else 0 + when = [year + 2000] + when[1:] + tzone = when[-1] if when[-1] else 0 # raises ValueError/OverflowError - return datetime(*t) - timedelta(tz) + return datetime.datetime(*when[:7]) - datetime.timedelta(tzone) except (TypeError, ValueError, OverflowError): seconds = default - return datetime.now() + timedelta(seconds=seconds) + return datetime.datetime.now() + datetime.timedelta(seconds=seconds) def poll(self, authzr): """Poll Authorization Resource for status. @@ -368,9 +369,9 @@ class Client(object): # pylint: disable=too-many-instance-attributes attempts = collections.defaultdict(int) exhausted = set() - # priority queue with datetime (based on Retry-After) as key, + # priority queue with datetime.datetime (based on Retry-After) as key, # and original Authorization Resource as value - waiting = [(datetime.now(), authzr) for authzr in authzrs] + waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] # mapping between original Authorization Resource and the most # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) @@ -378,7 +379,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) - now = datetime.now() + now = datetime.datetime.now() if when > now: seconds = (when - now).seconds logger.debug('Sleeping for %d seconds', seconds) From ca7f190efc3b0899439f82cf06e03292e1280582 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 11:29:36 -0800 Subject: [PATCH 07/64] lint & cover --- acme/acme/client.py | 2 +- acme/acme/client_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index b07009a11..46384cd85 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -22,7 +22,7 @@ try: from email.utils import parsedate_tz except ImportError: # pragma: no cover # pylint: disable=import-error,no-name-in-module - from email.Utils import parsedate_tz + from email.Utils import parsedate_tz logger = logging.getLogger(__name__) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 9abc69c7c..5a43272f2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -194,6 +194,10 @@ class ClientTest(unittest.TestCase): self.assertEqual( datetime.datetime(1999, 12, 31, 23, 59, 59), self.client.retry_after(response=self.response, default=10)) + self.response.headers['Retry-After'] = 'Fri, 31 Dec 17 23:59:59 GMT' + self.assertEqual( + datetime.datetime(2017, 12, 31, 23, 59, 59), + self.client.retry_after(response=self.response, default=10)) @mock.patch('acme.client.datetime') def test_retry_after_invalid(self, dt_mock): From ae69a7446587794cbaeed2cb522c7e4378dc26aa Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 11:38:26 -0800 Subject: [PATCH 08/64] Tidy --- acme/acme/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 46384cd85..4c02fde94 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,11 +1,10 @@ """ACME client API.""" import collections +import datetime import heapq import logging import time -import datetime - import six from six.moves import http_client # pylint: disable=import-error From 0afb4241734ddaafdbc46b522dc224fef63a2e1a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 12:29:13 -0800 Subject: [PATCH 09/64] py26 doesn't like adding lists & tuples --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 4c02fde94..c1bdcb69f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -271,7 +271,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # Handle two-digit years -- but any webserver that thinks # "retry after 99" means "come back after 1999" is.. deprecated if year >= 0 and year < 100: - when = [year + 2000] + when[1:] + when = [year + 2000] + list(when[1:]) tzone = when[-1] if when[-1] else 0 # raises ValueError/OverflowError return datetime.datetime(*when[:7]) - datetime.timedelta(tzone) From 043273960e2f6351c0493693666d108548a066e6 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 12 Feb 2016 15:47:24 -0500 Subject: [PATCH 10/64] Always install the homebrew version of Python. Fix #1437. Otherwise, we sometimes end up using the system Python, for which we'd need to use sudo to install virtualenv. Brew complicates this by yelling at you if you do use sudo. So let's simplify things by always using the homebrew python, which is more up to date anyway. --- letsencrypt-auto-source/letsencrypt-auto | 4 ++-- letsencrypt-auto-source/pieces/bootstrappers/mac.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ea3f7a4bb..2e12315ff 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -359,8 +359,8 @@ BootstrapMac() { brew install dialog fi - if ! hash pip 2>/dev/null; then - echo "pip not installed.\nInstalling python from Homebrew..." + if [ -z "$(brew list --versions python)" ]; then + echo "python not installed.\nInstalling python from Homebrew..." brew install python fi diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh index 23c12eec3..4bdf34116 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh @@ -14,8 +14,8 @@ BootstrapMac() { brew install dialog fi - if ! hash pip 2>/dev/null; then - echo "pip not installed.\nInstalling python from Homebrew..." + if [ -z "$(brew list --versions python)" ]; then + echo "python not installed.\nInstalling python from Homebrew..." brew install python fi From f1faedaa72e7833da49e6f33ecb2e3ce6a12cf6d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 12:48:20 -0800 Subject: [PATCH 11/64] This two digit year case is hard to trigger --- acme/acme/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 5a43272f2..dee78910f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -194,7 +194,7 @@ class ClientTest(unittest.TestCase): self.assertEqual( datetime.datetime(1999, 12, 31, 23, 59, 59), self.client.retry_after(response=self.response, default=10)) - self.response.headers['Retry-After'] = 'Fri, 31 Dec 17 23:59:59 GMT' + self.response.headers['Retry-After'] = 'Fri, 31-Dec-17 23:59:59 GMT' self.assertEqual( datetime.datetime(2017, 12, 31, 23, 59, 59), self.client.retry_after(response=self.response, default=10)) From c3ddb47cfa02d06f347430dd2fb0642845023f9d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 12:59:53 -0800 Subject: [PATCH 12/64] All this import voodoo is not required for py2.6+ --- acme/acme/client.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c1bdcb69f..1d1ea406f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -17,11 +17,7 @@ from acme import jose from acme import jws from acme import messages -try: - from email.utils import parsedate_tz -except ImportError: # pragma: no cover - # pylint: disable=import-error,no-name-in-module - from email.Utils import parsedate_tz +from email.utils import parsedate_tz logger = logging.getLogger(__name__) From 6fd3dba737a8de2c2f2d43d52d932418762762bb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 12 Feb 2016 15:08:32 -0800 Subject: [PATCH 13/64] Two digit years are used/tested in py26 only --- acme/acme/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1d1ea406f..a00ff5be4 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -264,10 +264,10 @@ class Client(object): # pylint: disable=too-many-instance-attributes when = parsedate_tz(retry_after) try: year = when[0] # raises TypeError if t is None - # Handle two-digit years -- but any webserver that thinks + # py26: Handle two-digit years -- but any server that thinks # "retry after 99" means "come back after 1999" is.. deprecated if year >= 0 and year < 100: - when = [year + 2000] + list(when[1:]) + when = [year + 2000] + list(when[1:]) # pragma: no cover tzone = when[-1] if when[-1] else 0 # raises ValueError/OverflowError return datetime.datetime(*when[:7]) - datetime.timedelta(tzone) From c83517c6f1f635e650319771a98e1643cb8d55fb Mon Sep 17 00:00:00 2001 From: David Date: Mon, 15 Feb 2016 09:29:29 +0100 Subject: [PATCH 14/64] sudo: not found Executed as root git clone https://github.com/letsencrypt/letsencrypt cd letsencrypt/ ./letsencrypt-auto --help failed with message ./letsencrypt-auto: 171: ./letsencrypt-auto: sudo: not found --- letsencrypt-auto-source/letsencrypt-auto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0630c649a..19d834490 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -168,7 +168,7 @@ BootstrapDebCommon() { /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' fi - sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" $SUDO apt-get update fi fi From 49aeffdebb03ba6165e93c4486ec7c89b8a5e626 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 15 Feb 2016 12:53:10 -0800 Subject: [PATCH 15/64] Address some review comments --- acme/acme/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index a00ff5be4..6a4457430 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -245,15 +245,17 @@ class Client(object): # pylint: disable=too-many-instance-attributes @classmethod def retry_after(cls, response, default): - """Compute next `poll` time based on response ``Retry-After`` header, - per https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37 + """Compute next `poll` time based on response ``Retry-After`` header. + + Handles integers and various datestring formats per + https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37 :param requests.Response response: Response from `poll`. :param int default: Default value (in seconds), used when ``Retry-After`` header is not present or invalid. :returns: Time point when next `poll` should be performed. - :rtype: `datetime.datetime.datetime` + :rtype: `datetime.datetime` """ retry_after = response.headers.get('Retry-After', str(default)).strip() From 7c8638f108552647c468f176e1ba6e4bcd3fc36e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 16 Feb 2016 11:04:52 -0800 Subject: [PATCH 16/64] Life is simpler if we don't support HTTP/1.0 ACME servers (Though in practice with py27+ we still support them) --- acme/acme/client.py | 19 +++++++------------ acme/acme/client_test.py | 4 ---- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6a4457430..470ffc41f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -262,19 +262,14 @@ class Client(object): # pylint: disable=too-many-instance-attributes try: seconds = int(retry_after) except ValueError: - # pylint: disable=no-member when = parsedate_tz(retry_after) - try: - year = when[0] # raises TypeError if t is None - # py26: Handle two-digit years -- but any server that thinks - # "retry after 99" means "come back after 1999" is.. deprecated - if year >= 0 and year < 100: - when = [year + 2000] + list(when[1:]) # pragma: no cover - tzone = when[-1] if when[-1] else 0 - # raises ValueError/OverflowError - return datetime.datetime(*when[:7]) - datetime.timedelta(tzone) - except (TypeError, ValueError, OverflowError): - seconds = default + if when is not None: + try: + tz_secs = datetime.timedelta(when[-1] if when[-1] else 0) + return datetime.datetime(*when[:7]) - tz_secs + except (ValueError, OverflowError): + pass + seconds = default return datetime.datetime.now() + datetime.timedelta(seconds=seconds) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index dee78910f..9abc69c7c 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -194,10 +194,6 @@ class ClientTest(unittest.TestCase): self.assertEqual( datetime.datetime(1999, 12, 31, 23, 59, 59), self.client.retry_after(response=self.response, default=10)) - self.response.headers['Retry-After'] = 'Fri, 31-Dec-17 23:59:59 GMT' - self.assertEqual( - datetime.datetime(2017, 12, 31, 23, 59, 59), - self.client.retry_after(response=self.response, default=10)) @mock.patch('acme.client.datetime') def test_retry_after_invalid(self, dt_mock): From 7f2ca5d065cffa83645d84aa13e8a4a56cd28d95 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 16 Feb 2016 11:13:10 -0800 Subject: [PATCH 17/64] Document use of email.utils parser --- acme/acme/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 470ffc41f..0f4286def 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -262,6 +262,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes try: seconds = int(retry_after) except ValueError: + # The RFC 2822 parser handles all of RFC 2616's cases in modern + # environments (primarily HTTP 1.1+ but also py27+) when = parsedate_tz(retry_after) if when is not None: try: From a9780c2ddcee8aee7d9ed33205e512a23c3c4a49 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 16 Feb 2016 11:13:25 -0800 Subject: [PATCH 18/64] Test trailing whitespace in headers --- acme/acme/client_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 9abc69c7c..00fb1c73c 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -205,6 +205,11 @@ class ClientTest(unittest.TestCase): datetime.datetime(2015, 3, 27, 0, 0, 10), self.client.retry_after(response=self.response, default=10)) + self.response.headers['Retry-After'] = '20 ' + self.assertEqual( + datetime.datetime(2015, 3, 27, 0, 0, 20), + self.client.retry_after(response=self.response, default=10)) + @mock.patch('acme.client.datetime') def test_retry_after_seconds(self, dt_mock): dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27) From 9fc723f3166131bcd583df9ed63fc66450228f8d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 16 Feb 2016 11:52:48 -0800 Subject: [PATCH 19/64] Exceptional coverage --- acme/acme/client_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 00fb1c73c..79000190d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -210,6 +210,13 @@ class ClientTest(unittest.TestCase): datetime.datetime(2015, 3, 27, 0, 0, 20), self.client.retry_after(response=self.response, default=10)) + # wrong date -> ValueError + dt_mock.datetime.side_effect = datetime.datetime + self.response.headers['Retry-After'] = "Tue, 116 Feb 2016 11:50:00 MST" + self.assertEqual( + datetime.datetime(2015, 3, 27, 0, 0, 10), + self.client.retry_after(response=self.response, default=10)) + @mock.patch('acme.client.datetime') def test_retry_after_seconds(self, dt_mock): dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27) From 95efab93b7616a8bd7d836b0f59334b678ecfd78 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Feb 2016 13:59:28 -0800 Subject: [PATCH 20/64] Remove quotes around $SUDO --- letsencrypt-auto-source/letsencrypt-auto | 4 ++-- letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0630c649a..255ba2793 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -307,10 +307,10 @@ BootstrapArchCommon() { pkg-config " - missing=$("$SUDO" pacman -T $deps) + missing=$($SUDO pacman -T $deps) if [ "$missing" ]; then - "$SUDO" pacman -S --needed $missing + $SUDO pacman -S --needed $missing fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 90a9d43c4..2e11e0ae9 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -18,9 +18,9 @@ BootstrapArchCommon() { pkg-config " - missing=$("$SUDO" pacman -T $deps) + missing=$($SUDO pacman -T $deps) if [ "$missing" ]; then - "$SUDO" pacman -S --needed $missing + $SUDO pacman -S --needed $missing fi } From 4b25d6543fe6df6d33d610637c49f5594a17463c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Feb 2016 14:19:27 -0800 Subject: [PATCH 21/64] Don't exit without installing packages --- letsencrypt-auto-source/letsencrypt-auto | 3 ++- letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 255ba2793..59767a4d0 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -307,7 +307,8 @@ BootstrapArchCommon() { pkg-config " - missing=$($SUDO pacman -T $deps) + # pacman -T exits with 127 if there are missing dependencies + missing=$($SUDO pacman -T $deps) || true if [ "$missing" ]; then $SUDO pacman -S --needed $missing diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 2e11e0ae9..b2fc01a14 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -18,7 +18,8 @@ BootstrapArchCommon() { pkg-config " - missing=$($SUDO pacman -T $deps) + # pacman -T exits with 127 if there are missing dependencies + missing=$($SUDO pacman -T $deps) || true if [ "$missing" ]; then $SUDO pacman -S --needed $missing From 55228e2df4a98a5b2214c77fbd04c4d8d878815b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Feb 2016 14:30:35 -0800 Subject: [PATCH 22/64] Remove quotes around SUDO in other bootstrap scripts --- letsencrypt-auto-source/letsencrypt-auto | 8 ++++---- letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh | 2 +- .../pieces/bootstrappers/gentoo_common.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 59767a4d0..58534ee32 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -328,19 +328,19 @@ BootstrapGentooCommon() { case "$PACKAGE_MANAGER" in (paludis) - "$SUDO" cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x + $SUDO cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x ;; (pkgcore) - "$SUDO" pmerge --noreplace --oneshot $PACKAGES + $SUDO pmerge --noreplace --oneshot $PACKAGES ;; (portage|*) - "$SUDO" emerge --noreplace --oneshot $PACKAGES + $SUDO emerge --noreplace --oneshot $PACKAGES ;; esac } BootstrapFreeBsd() { - "$SUDO" pkg install -Ay \ + $SUDO pkg install -Ay \ python \ py27-virtualenv \ augeas \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh index c9abd22eb..deb2e2115 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh @@ -1,5 +1,5 @@ BootstrapFreeBsd() { - "$SUDO" pkg install -Ay \ + $SUDO pkg install -Ay \ python \ py27-virtualenv \ augeas \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh index 1d8211df4..580b69a0d 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh @@ -11,13 +11,13 @@ BootstrapGentooCommon() { case "$PACKAGE_MANAGER" in (paludis) - "$SUDO" cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x + $SUDO cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x ;; (pkgcore) - "$SUDO" pmerge --noreplace --oneshot $PACKAGES + $SUDO pmerge --noreplace --oneshot $PACKAGES ;; (portage|*) - "$SUDO" emerge --noreplace --oneshot $PACKAGES + $SUDO emerge --noreplace --oneshot $PACKAGES ;; esac } From c71fa444569da3ffefefe3396306b9fe52b9c94c Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 16 Feb 2016 17:51:08 -0500 Subject: [PATCH 23/64] Upgrade peep to 3.1.1. Fix bad LE experience reported at https://github.com/erikrose/peep/issues/119. --- letsencrypt-auto-source/letsencrypt-auto | 23 ++++++++++++++++------- letsencrypt-auto-source/pieces/peep.py | 23 ++++++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 743770f35..08390c0c4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -756,6 +756,7 @@ except ImportError: from pip.util import url_to_path # 0.7.0 except ImportError: from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.exceptions import InstallationError from pip.index import PackageFinder, Link try: from pip.log import logger @@ -774,7 +775,7 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar -__version__ = 3, 0, 0 +__version__ = 3, 1, 1 try: from pip.index import FormatControl # noqa @@ -792,6 +793,7 @@ ITS_FINE_ITS_FINE = 0 SOMETHING_WENT_WRONG = 1 # "Traditional" for command-line errors according to optparse docs: COMMAND_LINE_ERROR = 2 +UNHANDLED_EXCEPTION = 3 ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') @@ -1554,7 +1556,7 @@ def peep_install(argv): first_every_last(buckets[SatisfiedReq], *printers) return ITS_FINE_ITS_FINE - except (UnsupportedRequirementError, DownloadError) as exc: + except (UnsupportedRequirementError, InstallationError, DownloadError) as exc: out(str(exc)) return SOMETHING_WENT_WRONG finally: @@ -1574,16 +1576,23 @@ def peep_port(paths): print('Please specify one or more requirements files so I have ' 'something to port.\n') return COMMAND_LINE_ERROR + + comes_from = None for req in chain.from_iterable( _parse_requirements(path, package_finder(argv)) for path in paths): + req_path, req_line = path_and_line(req) hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') - for hash in hashes_above(*path_and_line(req))] + for hash in hashes_above(req_path, req_line)] + if req_path != comes_from: + print() + print('# from %s' % req_path) + print() + comes_from = req_path + if not hashes: print(req.req) - elif len(hashes) == 1: - print('%s --hash=sha256:%s' % (req.req, hashes[0])) else: - print('%s' % req.req, end='') + print('%s' % (req.link if getattr(req, 'link', None) else req.req), end='') for hash in hashes: print(' \\') print(' --hash=sha256:%s' % hash, end='') @@ -1628,7 +1637,7 @@ if __name__ == '__main__': exit(main()) except Exception: exception_handler(*sys.exc_info()) - exit(SOMETHING_WENT_WRONG) + exit(UNHANDLED_EXCEPTION) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/pieces/peep.py b/letsencrypt-auto-source/pieces/peep.py index c4e51f483..eee823ff2 100755 --- a/letsencrypt-auto-source/pieces/peep.py +++ b/letsencrypt-auto-source/pieces/peep.py @@ -86,6 +86,7 @@ except ImportError: from pip.util import url_to_path # 0.7.0 except ImportError: from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.exceptions import InstallationError from pip.index import PackageFinder, Link try: from pip.log import logger @@ -104,7 +105,7 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar -__version__ = 3, 0, 0 +__version__ = 3, 1, 1 try: from pip.index import FormatControl # noqa @@ -122,6 +123,7 @@ ITS_FINE_ITS_FINE = 0 SOMETHING_WENT_WRONG = 1 # "Traditional" for command-line errors according to optparse docs: COMMAND_LINE_ERROR = 2 +UNHANDLED_EXCEPTION = 3 ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') @@ -884,7 +886,7 @@ def peep_install(argv): first_every_last(buckets[SatisfiedReq], *printers) return ITS_FINE_ITS_FINE - except (UnsupportedRequirementError, DownloadError) as exc: + except (UnsupportedRequirementError, InstallationError, DownloadError) as exc: out(str(exc)) return SOMETHING_WENT_WRONG finally: @@ -904,16 +906,23 @@ def peep_port(paths): print('Please specify one or more requirements files so I have ' 'something to port.\n') return COMMAND_LINE_ERROR + + comes_from = None for req in chain.from_iterable( _parse_requirements(path, package_finder(argv)) for path in paths): + req_path, req_line = path_and_line(req) hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') - for hash in hashes_above(*path_and_line(req))] + for hash in hashes_above(req_path, req_line)] + if req_path != comes_from: + print() + print('# from %s' % req_path) + print() + comes_from = req_path + if not hashes: print(req.req) - elif len(hashes) == 1: - print('%s --hash=sha256:%s' % (req.req, hashes[0])) else: - print('%s' % req.req, end='') + print('%s' % (req.link if getattr(req, 'link', None) else req.req), end='') for hash in hashes: print(' \\') print(' --hash=sha256:%s' % hash, end='') @@ -958,4 +967,4 @@ if __name__ == '__main__': exit(main()) except Exception: exception_handler(*sys.exc_info()) - exit(SOMETHING_WENT_WRONG) + exit(UNHANDLED_EXCEPTION) From 3964357eb3417b4bb3e3c61b611852983f8f08d4 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 16 Feb 2016 15:48:36 -0800 Subject: [PATCH 24/64] rewrite generic files --- .../letsencrypt_apache/configurator.py | 7 +++++++ letsencrypt-apache/letsencrypt_apache/obj.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index cbc451ac9..694767f2a 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1074,6 +1074,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self._is_rewrite_engine_on(general_vh): self.parser.add_dir(general_vh.path, "RewriteEngine", "on") + for name in ssl_vhost.get_names(): + self.parser.add_dir(general_vh.path, "RewriteCond", + ["%{SERVER_NAME}", "={0}".format(name), "[OR]"] if self.get_version() >= (2, 3, 9): self.parser.add_dir(general_vh.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS_WITH_END) @@ -1243,6 +1246,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for http_vh in candidate_http_vhs: if http_vh.same_server(ssl_vhost): return http_vh + # Third filter - if none with same names, return generic + for http_vh in candidate_http_vhs: + if http_vh.same_server(ssl_vhost, generic=True): + return http_vh return None diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index 175ce3f92..c98ca4b99 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -189,7 +189,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return True return False - def same_server(self, vhost): + def same_server(self, vhost, generic=False): """Determines if the vhost is the same 'server'. Used in redirection - indicates whether or not the two virtual hosts @@ -199,12 +199,17 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods """ - if vhost.get_names() != self.get_names(): - return False + if not generic: + if vhost.get_names() != self.get_names(): + return False - # If equal and set is not empty... assume same server - if self.name is not None or self.aliases: - return True + # If equal and set is not empty... assume same server + if self.name is not None or self.aliases: + return True + # If we're looking for a generic vhost, don't return one with a ServerName + else: + if self.name: + return False # Both sets of names are empty. From bf30e54a32f91178d8d84305e2db4fb11592bb70 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 16 Feb 2016 15:58:53 -0800 Subject: [PATCH 25/64] fix syntax and don't have unneeded ors --- letsencrypt-apache/letsencrypt_apache/configurator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 694767f2a..2198cbdfb 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1073,10 +1073,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # even with save() and load() if not self._is_rewrite_engine_on(general_vh): self.parser.add_dir(general_vh.path, "RewriteEngine", "on") - - for name in ssl_vhost.get_names(): + cond = "[OR]" + names = ssl_vhost.get_names() + for idx, name in enumerate(names): + if idx == len(names) - 1: + cond = "" self.parser.add_dir(general_vh.path, "RewriteCond", - ["%{SERVER_NAME}", "={0}".format(name), "[OR]"] + ["%{SERVER_NAME}", "={0}".format(name), cond]) if self.get_version() >= (2, 3, 9): self.parser.add_dir(general_vh.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS_WITH_END) From 109d6caf6525e1673e4d126bd1a73530645be9bf Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 16 Feb 2016 16:07:03 -0800 Subject: [PATCH 26/64] fix how OR is added --- letsencrypt-apache/letsencrypt_apache/configurator.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 2198cbdfb..9cff002f1 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1073,13 +1073,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # even with save() and load() if not self._is_rewrite_engine_on(general_vh): self.parser.add_dir(general_vh.path, "RewriteEngine", "on") - cond = "[OR]" names = ssl_vhost.get_names() for idx, name in enumerate(names): + args = ["%{SERVER_NAME}", "={0}".format(name), "[OR]"] if idx == len(names) - 1: - cond = "" - self.parser.add_dir(general_vh.path, "RewriteCond", - ["%{SERVER_NAME}", "={0}".format(name), cond]) + args.pop() + self.parser.add_dir(general_vh.path, "RewriteCond", args) if self.get_version() >= (2, 3, 9): self.parser.add_dir(general_vh.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS_WITH_END) From c8a9da844294829d8f7c3f5a882f7d9ce5f2a4e1 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 16 Feb 2016 16:47:29 -0800 Subject: [PATCH 27/64] update test to hit new line in configurator --- .../letsencrypt_apache/tests/configurator_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 5b15a20d1..58ec3fe21 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -746,9 +746,9 @@ class TwoVhost80Test(util.ApacheTest): self.config.parser.modules.add("rewrite_module") mock_exe.return_value = True ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("satoshi.com",))]), + "fp", "ap", set([obj.Addr(("*", "443"))]), True, False) + ssl_vh.name = "satoshi.com" self.config.vhosts.append(ssl_vh) self.assertRaises( errors.PluginError, From fe1ab15f4b5f69fb7a773f37512c8eb7f18d03d0 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Wed, 17 Feb 2016 05:33:00 +0200 Subject: [PATCH 28/64] Adding test for unsupported MX error --- letsencrypt/tests/client_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index f712ea94c..1f53fdc35 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -83,6 +83,14 @@ class RegisterTest(unittest.TestCase): self._call() mock_logger.warn.assert_called_once_with(mock.ANY) + def test_unsupported_error(self): + from acme import messages + msg = "Test" + mx_err = messages.Error(detail=msg, typ="malformed", title="title") + with mock.patch("letsencrypt.client.acme_client.Client") as mock_client: + mock_client().register.side_effect = [mx_err, mock.MagicMock()] + self.assertRaises(messages.Error, self._call) + class ClientTest(unittest.TestCase): """Tests for letsencrypt.client.Client.""" From 8b7f72b5bc2db2bd8dd922b27048383e89e1e2d9 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Wed, 17 Feb 2016 06:22:50 +0200 Subject: [PATCH 29/64] Adding test for obtain_certificate_from_csr with auth_handler set to None --- letsencrypt/tests/client_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 1f53fdc35..14f4340f0 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -127,8 +127,9 @@ class ClientTest(unittest.TestCase): self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) # FIXME move parts of this to test_cli.py... + @mock.patch("letsencrypt.client.logger") @mock.patch("letsencrypt.cli._process_domain") - def test_obtain_certificate_from_csr(self, mock_process_domain): + def test_obtain_certificate_from_csr(self, mock_process_domain, mock_logger): self._mock_obtain_certificate() from letsencrypt import cli test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) @@ -153,6 +154,14 @@ class ClientTest(unittest.TestCase): # and that the cert was obtained correctly self._check_obtain_certificate() + # Test for no auth_handler + self.client.auth_handler = None + self.assertRaises( + errors.Error, + self.client.obtain_certificate_from_csr, + self.eg_domains, + test_csr) + mock_logger.warning.assert_called_once_with(mock.ANY) @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): From f72bcb5ea40b3aaa9f94ce4ddded6032c5bed448 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 17 Feb 2016 15:57:14 -0800 Subject: [PATCH 30/64] print only challenge changes to configs --- letsencrypt-apache/letsencrypt_apache/configurator.py | 1 - letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index cbc451ac9..b2dc2e974 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1429,7 +1429,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self.config_test() - logger.debug(self.reverter.view_config_changes(for_logging=True)) self._reload() def _reload(self): diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index cc1d749a0..d78910149 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -108,7 +108,7 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.reverter.register_file_creation( True, self.challenge_conf) - logger.debug("writing a config file with text: %s", config_text) + logger.debug("writing a config file with text:\n %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) @@ -144,6 +144,8 @@ class ApacheTlsSni01(common.TLSSNI01): if len(self.configurator.parser.find_dir( parser.case_i("Include"), self.challenge_conf)) == 0: # print "Including challenge virtual host(s)" + logger.debug("Adding Include {0} to {1}".format( + self.challenge_conf, parser.get_aug_path(main_config))) self.configurator.parser.add_dir( parser.get_aug_path(main_config), "Include", self.challenge_conf) From 8d61c86c8c4934539ca74dc31661c07e3b3b17ee Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 17 Feb 2016 16:11:50 -0800 Subject: [PATCH 31/64] Well actually We don't need stripping after all. --- acme/acme/client.py | 2 +- acme/acme/client_test.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 0f4286def..eaca6dc7d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -258,7 +258,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :rtype: `datetime.datetime` """ - retry_after = response.headers.get('Retry-After', str(default)).strip() + retry_after = response.headers.get('Retry-After', str(default)) try: seconds = int(retry_after) except ValueError: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 79000190d..29f60c25d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -205,11 +205,6 @@ class ClientTest(unittest.TestCase): datetime.datetime(2015, 3, 27, 0, 0, 10), self.client.retry_after(response=self.response, default=10)) - self.response.headers['Retry-After'] = '20 ' - self.assertEqual( - datetime.datetime(2015, 3, 27, 0, 0, 20), - self.client.retry_after(response=self.response, default=10)) - # wrong date -> ValueError dt_mock.datetime.side_effect = datetime.datetime self.response.headers['Retry-After'] = "Tue, 116 Feb 2016 11:50:00 MST" From 812d6e7ae99e55c35854d551748f7540982ab165 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 17 Feb 2016 16:30:55 -0800 Subject: [PATCH 32/64] fix linting --- letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index d78910149..671686433 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -144,8 +144,8 @@ class ApacheTlsSni01(common.TLSSNI01): if len(self.configurator.parser.find_dir( parser.case_i("Include"), self.challenge_conf)) == 0: # print "Including challenge virtual host(s)" - logger.debug("Adding Include {0} to {1}".format( - self.challenge_conf, parser.get_aug_path(main_config))) + logger.debug("Adding Include %s to %s", + self.challenge_conf, parser.get_aug_path(main_config)) self.configurator.parser.add_dir( parser.get_aug_path(main_config), "Include", self.challenge_conf) From eec6287d12ce70410abbe5f77de960c723af7651 Mon Sep 17 00:00:00 2001 From: dave-cz Date: Thu, 18 Feb 2016 09:57:06 +0100 Subject: [PATCH 33/64] change in the source file --- letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index b45e43ba9..bbafb39d7 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -51,7 +51,7 @@ BootstrapDebCommon() { /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' fi - sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" $SUDO apt-get update fi fi From 1de66b3d7d2f8b7681a28d4cd7ed60ad30a2c3f9 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 18 Feb 2016 16:02:07 -0800 Subject: [PATCH 34/64] Explicit error message for #2206 --- letsencrypt/cli.py | 5 +++++ letsencrypt/tests/cli_test.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 855c7a467..d245f096d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -649,6 +649,11 @@ def record_chosen_plugins(config, plugins, auth, inst): # Possible difficulties: config.csr was hacked into auth def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install.""" + if config.csr is not None: + raise errors.Error("Currently, the default 'run' verb cannot be used " + "when specifying a CSR file. Please try the " + "certonly command instead.") + try: installer, authenticator = choose_configurator_plugins(config, plugins, "run") except errors.PluginSelectionError as e: diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 77a4b5892..60fa3ebec 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -356,6 +356,15 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._call, ['-d', '204.11.231.35']) + def test_run_with_csr(self): + # This is an error because you can only use --csr with certonly + try: + self._call(['--csr', CSR]) + except errors.Error as e: + assert "Please try the certonly" in e.message + return + assert False, "Expected supplying --csr to fail with default verb" + def _get_argument_parser(self): plugins = disco.PluginsRegistry.find_all() return functools.partial(cli.prepare_and_parse_args, plugins) From 5eba011f8e57487b35cfab6fcc36a85233aee29a Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 18 Feb 2016 18:35:45 -0800 Subject: [PATCH 35/64] Generalize and move check inside handle_csr --- letsencrypt/cli.py | 15 ++++++--------- letsencrypt/tests/client_test.py | 3 +++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index d245f096d..74084692d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -649,11 +649,6 @@ def record_chosen_plugins(config, plugins, auth, inst): # Possible difficulties: config.csr was hacked into auth def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install.""" - if config.csr is not None: - raise errors.Error("Currently, the default 'run' verb cannot be used " - "when specifying a CSR file. Please try the " - "certonly command instead.") - try: installer, authenticator = choose_configurator_plugins(config, plugins, "run") except errors.PluginSelectionError as e: @@ -988,10 +983,6 @@ def renew(config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") - if config.csr is not None: - raise errors.Error("Currently, the renew verb cannot be used when " - "specifying a CSR file. Please try the certonly " - "command instead.") renewer_config = configuration.RenewerConfiguration(config) renew_successes = [] renew_failures = [] @@ -1249,6 +1240,12 @@ class HelpfulArgumentParser(object): Process a --csr flag. This needs to happen early enough that the webroot plugin can know about the calls to _process_domain """ + if parsed_args.verb != "certonly": + raise errors.Error("Currently, a CSR file may only be specified " + "when obtaining a new or replacement " + "via the certonly command. Please try the " + "certonly command instead.") + try: csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") typ = OpenSSL.crypto.FILETYPE_ASN1 diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index f712ea94c..daaea4f97 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -125,6 +125,9 @@ class ClientTest(unittest.TestCase): from letsencrypt import cli test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) mock_parsed_args = mock.MagicMock() + # The CLI should believe that this is a certonly request, because + # a CSR would not be allowed with other kinds of requests! + mock_parsed_args.verb = "certonly" with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr mock_parsed_args.domains = self.eg_domains[:] From 0e12b1fd86027cc026134d3c4ae975d3a68b0989 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Fri, 19 Feb 2016 05:00:47 +0200 Subject: [PATCH 36/64] Enabling apache-conf-test in Travis --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e253081d9..2dd7a1b30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,10 +30,9 @@ matrix: env: TOXENV=py26 BOULDER_INTEGRATION=1 - python: "2.6" env: TOXENV=py26-oldest BOULDER_INTEGRATION=1 -# Disabled for now due to requiring sudo -> causing more boulder integration -# DNS timeouts :( -# - python: "2.7" -# env: TOXENV=apacheconftest + - python: "2.7" + env: TOXENV=apacheconftest + sudo: required - python: "2.7" env: TOXENV=py27 BOULDER_INTEGRATION=1 - python: "2.7" From 4ce926315dd6461f8910aee3bdfe23f3a80c3401 Mon Sep 17 00:00:00 2001 From: Nikos Roussos Date: Fri, 19 Feb 2016 12:15:26 +0200 Subject: [PATCH 37/64] Add Fedora package installation on docs --- docs/using.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 37fca2c57..fd736c2f9 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -429,6 +429,12 @@ If you don't want to use the Apache plugin, you can omit the Packages for Debian Jessie are coming in the next few weeks. +**Fedora** + +.. code-block:: shell + + sudo dnf install letsencrypt + **Gentoo** The official Let's Encrypt client is available in Gentoo Portage. If you From b95a01a15cb20cfd5249180d3a1e9e7da2110d36 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 19 Feb 2016 12:36:11 -0500 Subject: [PATCH 38/64] Turn the root-level letsencrypt-auto symlink into a regular file. Close #2501. It will always be a copy of the latest release version, 0.4 in this case. (Modify the release script to make that so.) This way, people using the old method of running le-auto from a git checkout will not end up using a bleeding-edge version, letting us work on the tip-of-tree version more freely. --- letsencrypt-auto | 1810 +++++++++++++++++++++++++++++++++++++++++++++- tools/release.sh | 5 +- 2 files changed, 1813 insertions(+), 2 deletions(-) mode change 120000 => 100755 letsencrypt-auto diff --git a/letsencrypt-auto b/letsencrypt-auto deleted file mode 120000 index af7e83a70..000000000 --- a/letsencrypt-auto +++ /dev/null @@ -1 +0,0 @@ -letsencrypt-auto-source/letsencrypt-auto \ No newline at end of file diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 100755 index 000000000..9218bdc52 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1,1809 @@ +#!/bin/sh +# +# Download and run the latest release version of the Let's Encrypt client. +# +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* + +set -e # Work even if somebody does "sh thisscript.sh". + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN=${VENV_PATH}/bin +LE_AUTO_VERSION="0.4.0" + +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then + VERBOSE=1 + elif [ "$arg" = "--no-self-upgrade" ] ; then + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1 + elif [ "$arg" = "--os-packages-only" ] ; then + OS_PACKAGES_ONLY=1 + elif [ "$arg" = "--debug" ]; then + DEBUG=1 + fi +done + +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + if command -v python2.7 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2.7} + elif command -v python27 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python27} + elif command -v python2 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2} + elif command -v python > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python} + else + echo "Cannot find any Pythons... please install one!" + exit 1 + fi + + PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ $PYVER -lt 26 ]; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null 2>&1; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null 2>&1; then + virtualenv="$virtualenv python-virtualenv" + fi + + augeas_pkg="libaugeas0 augeas-lenses" + AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + + } + + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs + fi + + $SUDO apt-get install -y --no-install-recommends \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + $augeas_pkg \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if ! $SUDO $tool install -y \ + python \ + python-devel \ + python-virtualenv \ + python-tools \ + python-pip + then + if ! $SUDO $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv \ + python27-tools \ + python27-pip + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + fi + + if ! $SUDO $tool install -y \ + gcc \ + dialog \ + augeas-libs \ + openssl \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi + + + if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then + if ! $SUDO $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi + fi +} + +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + $SUDO zypper -nq in -l \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates +} + +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./tools/_venv_common.sh + + deps=" + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + missing=$("$SUDO" pacman -T $deps) + + if [ "$missing" ]; then + "$SUDO" pacman -S --needed $missing + fi +} + +BootstrapGentooCommon() { + PACKAGES=" + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + "$SUDO" pmerge --noreplace $PACKAGES + ;; + (portage|*) + "$SUDO" emerge --noreplace $PACKAGES + ;; + esac +} + +BootstrapFreeBsd() { + "$SUDO" pkg install -Ay \ + python \ + py27-virtualenv \ + augeas \ + libffi +} + +BootstrapMac() { + if ! hash brew 2>/dev/null; then + echo "Homebrew Not Installed\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + brew install augeas + brew install dialog + + if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv + fi +} + + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ]; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/redhat-release ]; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchCommon + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ]; then + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon + elif [ -f /etc/gentoo-release ]; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a peep install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + fi +} + +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + + + +if [ "$NO_SELF_UPGRADE" = 1 ]; then + # Phase 2: Create venv, install LE, and run. + + if [ -f "$VENV_BIN/letsencrypt" ]; then + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) + else + INSTALLED_VERSION="none" + fi + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then + echo "Creating virtual environment..." + DeterminePythonVersion + rm -rf "$VENV_PATH" + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi + + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + # There is no $ interpolation due to quotes on starting heredoc delimiter. + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" +# This is the flattened list of packages letsencrypt-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# then use `hashin` or a more secure method to gather the hashes. + +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: U8HJ3bMEMVE-t_PN7wo-BrDxJSGIqqd0SvD1pM1F268 +# sha256: pWj0nfyhKo2fNwGHJX78WKOBCeHu5xTZKFYdegGKZPg +# sha256: gJxsqM-8ruv71DK0V2ABtA04_yRjdzy1dXfXXhoCC8M +# sha256: hs3KLNnLpBQiIwOQ3xff6qnzRKkR45dci-naV7NVSOk +# sha256: JLE9uErsOFyiPHuN7YPvi7QXe8GB0UdY-fl1vl0CDYY +# sha256: lprv_XwOCX9r4e_WgsFWriJlkaB5OpS2wtXkKT9MjU4 +# sha256: AA81jUsPokn-qrnBzn1bL-fgLnvfaAbCZBhQX8aF4mg +# sha256: qdhvRgu9g1ii1ROtd54_P8h447k6ALUAL66_YW_-a5w +# sha256: MSezqzPrI8ysBx-aCAJ0jlz3xcvNAkgrsGPjW0HbsLA +# sha256: 4rLUIjZGmkAiTTnntsYFdfOIsvQj81TH7pClt_WMgGU +# sha256: jC3Mr-6JsbQksL7GrS3ZYiyUnSAk6Sn12h7YAerHXx0 +# sha256: pN56TRGu1Ii6tPsU9JiFh6gpvs5aIEM_eA1uM7CAg8s +# sha256: XKj-MEJSZaSSdOSwITobyY9LE0Sa5elvmEdx5dg-WME +# sha256: pP04gC9Z5xTrqBoCT2LbcQsn2-J6fqEukRU3MnqoTTA +# sha256: hs1pErvIPpQF1Kc81_S07oNTZS0tvHyCAQbtW00bqzo +# sha256: jx0XfTZOo1kAQVriTKPkcb49UzTtBBkpQGjEn0WROZg +cffi==1.4.2 + +# sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc +ConfigArgParse==0.10.0 + +# sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI +configobj==5.0.6 + +# sha256: 1U_hszrB4J8cEj4vl0948z6V1h1PSALdISIKXD6MEX0 +# sha256: B1X2aE4RhSAFs2MTdh7ctbqEOmTNAizhrC3L1JqTYG0 +# sha256: zjhNo4lZlluh90VKJfVp737yqxRd8ueiml4pS3TgRnc +# sha256: GvQDkV3LmWHDB2iuZRr6tpKC0dpaut-mN1IhrBGHdQM +# sha256: ag08d91PH-W8ZfJ--3fsjQSjiNpesl66DiBAwJgZ30o +# sha256: KdelgcO6_wTh--IAaltHjZ7cfPmib8ijWUkkf09lA3k +# sha256: IPAWEKpAh_bVadjMIMR4uB8DhIYnWqqx3Dx12VAsZ-A +# sha256: l9hGUIulDVomml82OK4cFmWbNTFaH0B_oVF2cH2j0Jc +# sha256: djfqRMLL1NsvLKccsmtmPRczORqnafi8g2xZVilbd5g +# sha256: gR-eqJVbPquzLgQGU0XDB4Ui5rPuPZLz0n08fNcWpjM +# sha256: DXCMjYz97Qm4fCoLqHY856ZjWG4EPmrEL9eDHpKQHLY +# sha256: Efnq11YqPgATWGytM5o_em9Yg8zhw7S5jhrGnft3p_Y +# sha256: dNhnm55-0ePs-wq1NNyTUruxz3PTYsmQkJTAlyivqJY +# sha256: z1Hd-123eBaiB1OKZgEUuC4w4IAD_uhJmwILi4SA2sU +# sha256: 47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU +# sha256: dITvgYGUFB3_eUdf-74vd6-FHiw7v-Lk1ZEjEi-KTjM +# sha256: 7gLB6J7l7pUBV6VK1YTXN8Ec83putMCFPozz8n6WLcA +# sha256: pfGPaxhQpVVKV9v2YsrSUSpGBW5paHJqmFjngN1bnQo +# sha256: 26GA8xrb5xi6qdbPirY0hJSwlLK4GAL_8zvVDSfRPnM +# sha256: 5RinlLjzjoOC9_B3kUGBPOtIE6z9MRVBwNsOGJ69eN4 +# sha256: f1FFn4TWcERCdeYVg59FQsk1R6Euk4oKSQba_l994VM +cryptography==1.1.2 + +# sha256: JHXX_N31lR6S_1RpcnWIAt5SYL9Akxmp8ZNOa7yLHcc +# sha256: NZB977D5krdat3iPZf7cHPIP-iJojg5vbxKvwGs-pQE +enum34==1.1.2 + +# sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 +# sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM +funcsigs==0.4 + +# sha256: my_FC9PEujBrllG2lBHvIgJtTYM1uTr8IhTO8SRs5wc +# sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs +idna==2.0 + +# sha256: k1cSgAzkdgcB2JrWd2Zs1SaR_S9vCzQMi0I5o8F5iKU +# sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA +ipaddress==1.0.16 + +# sha256: 54vpwKDfy6xxL-BPv5K5bN2ugLG4QvJCSCFMhJbwBu8 +# sha256: Syb_TnEQ23butvWntkqCYjg51ZXCA47tpmLyott46Xw +linecache2==1.0.0 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + +# sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs +# sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs +parsedatetime==1.5 + +# sha256: Rsjbda51oFa9HMB_ohc0_i5gPRGgeDPswe63TDXHLgw +# sha256: 4hJ2JqkebIhduJZol22zECDwry2nKJJLVkgPx8zwlkk +pbr==1.8.1 + +# sha256: WE8LKfzF1SO0M8uJGLL8dNZ-MO4LRKlbrwMVKPQkYZ8 +# sha256: KMoLbp2Zqo3Chuh0ekRxNitpgSolKR3im2qNcKFUWg0 +# sha256: FnrV__UqZyxN3BwaCyUUbWgT67CKmqsKOsRfiltmnDs +# sha256: 5t6mFzqYhye7Ij00lzSa1c3vXAsoLv8tg-X5BlxT-F8 +# sha256: KvXgpKrWYEmVXQc0qk49yMqhep6vi0waJ6Xx7m5A9vw +# sha256: 2YhNwNwuVeJEjklXeNyYmcHIvzeusvQ0wb6nSvk8JoM +# sha256: 4nwv5t_Mhzi-PSxaAi94XrcpcQV-Gp4eNPunO86KcaY +# sha256: Za_W_syPOu0J7kvmNYO8jrRy8GzqpP4kxNHVoaPA4T8 +# sha256: uhxVj7_N-UUVwjlLEVXB3FbivCqcF9MDSYJ8ntimfkY +# sha256: upXqACLctk028MEzXAYF-uNb3z4P6o2S9dD2RWo15Vs +# sha256: QhtlkdFrUJqqjYwVgh1mu5TLSo3EOFytXFG4XUoJbYU +# sha256: MmswXL22-U2vv-LCaxHaiLCrB7igf4GIq511_wxuhBo +# sha256: mu3lsrb-RrN0jqjlIURDiQ0WNAJ77z0zt9rRZVaDAng +# sha256: c77R24lNGqnDx-YR0wLN6reuig3A7q92cnh42xrFzYc +# sha256: k1td1tVYr1EvQlAafAj0HXr_E5rxuzlZ2qOruFkjTWw +# sha256: TKARHPFX3MDy9poyPFtUeHGNaNRfyUNdhL4OwPGGIVs +# sha256: tvE8lTmKP88CJsTc-kSFYLpYZSWc2W7CgQZYZR6TIYk +# sha256: 7mvjDRY1u96kxDJdUH3IoNu95-HBmL1i3bn0MZi54hQ +# sha256: 36eGhYwmjX-74bYXXgAewCc418-uCnzne_m2Ua9nZyk +# sha256: qnf53nKvnBbMKIzUokz1iCQ4j1fXqB5ADEYWRXYphw4 +# sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus +psutil==3.3.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + +# sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M +pycparser==2.14 + +# sha256: iORea7Jd_tJyoe8ucoRh1EtjTCzWiemJtuVqNJxaOuU +# sha256: 8KJgcNbbCIHei8x4RpNLfDyTDY-cedRYg-5ImEvA1nI +pyOpenSSL==0.15.1 + +# sha256: 7qMYNcVuIJavQ2OldFp4SHimHQQ-JH06bWoKMql0H1Y +# sha256: jfvGxFi42rocDzYgqMeACLMjomiye3NZ6SpK5BMl9TU +pyRFC3339==1.0 + +# sha256: Z9WdZs26jWJOA4m4eyqDoXbyHxaodVO1D1cDsj8pusI +python-augeas==0.5.0 + +# sha256: BOk_JJlcQ92Q8zjV2GXKcs4_taU1jU2qSWVXHbNfw-w +# sha256: Pm9ZP-rZj4pSa8PjBpM1MyNuM3KfVS9SiW6lBPVTE_o +python2-pythondialog==3.3.0 + +# sha256: Or5qbT_C-75MYBRCEfRdou2-MYKm9lEa9ru6BZix-ZI +# sha256: k575weEiTZgEBWial__PeCjFbRUXsx1zRkNWwfK3dp4 +# sha256: 6tSu-nAHJJ4F5RsBCVcZ1ajdlXYAifVzCqxWmLGTKRg +# sha256: PMoN8IvQ7ZhDI5BJTOPe0AP15mGqRgvnpzS__jWYNgU +# sha256: Pt5HDT0XujwHY436DRBFK8G25a0yYSemW6d-aq6xG-w +# sha256: aMR5ZPcYbuwwaxNilidyK5B5zURH7Z5eyuzU6shMpzQ +# sha256: 3V05kZUKrkCmyB3hV4lC5z1imAjO_FHRLNFXmA5s_Bg +# sha256: p3xSBiwH63x7MFRdvHPjKZW34Rfup1Axe1y1x6RhjxQ +# sha256: ga-a7EvJYKmgEnxIjxh3La5GNGiSM_BvZUQ-exHr61E +# sha256: 4Hmx2txcBiRswbtv4bI6ULHRFz8u3VEE79QLtzoo9AY +# sha256: -9JnRncsJMuTyLl8va1cueRshrvbG52KdD7gDi-x_F0 +# sha256: mSZu8wo35Dky3uwrfKc-g8jbw7n_cD7HPsprHa5r7-o +# sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM +pytz==2015.7 + +# sha256: ET-7pVManjSUW302szoIToul0GZLcDyBp8Vy2RkZpbg +# sha256: xXeBXdAPE5QgP8ROuXlySwmPiCZKnviY7kW45enPWH8 +requests==2.9.1 + +# sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE +# sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo +six==1.10.0 + +# sha256: glPOvsSxkJTWfMXtWvmb8duhKFKSIm6Yoxkp-HpdayM +# sha256: BazGegmYDC7P7dNCP3rgEEg57MtV_GRXc-HKoJUcMDA +traceback2==1.4.0 + +# sha256: E_d9CHXbbZtDXh1PQedK1MwutuHVyCSZYJKzQw8Ii7g +# sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk +unittest2==1.1.0 + +# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms +# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI +Werkzeug==0.11.3 + +# sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo +zope.component==4.2.2 + +# sha256: 3HpZov2Rcw03kxMaXSYbKek-xOKpfxvEh86N7-4v54Y +zope.event==4.1.0 + +# sha256: 8HtjH3pgHNjL0zMtVPQxQscIioMpn4WTVvCNHU1CWbM +# sha256: 3lzKCDuUOdgAL7drvmtJmMWlpyH6sluEKYln8ALfTJQ +# sha256: Z4hBb36n9bipe-lIJTd6ol6L3HNGPge6r5hYsp5zcHc +# sha256: bzIw9yVFGCAeWjcIy7LemMhIME8G497Yv7OeWCXLouE +# sha256: X6V1pSQPBCAMMIhCfQ1Le3N_bpAYgYpR2ND5J6aiUXo +# sha256: UiGUrWpUVzXt11yKg_SNZdGvBk5DKn0yDWT1a6_BLpk +# sha256: 6Mey1AlD9xyZFIyX9myqf1E0FH9XQj-NtbSCUJnOmgk +# sha256: J5Ak8CCGAcPKqQfFOHbjetiGJffq8cs4QtvjYLIocBc +# sha256: LiIanux8zFiImieOoT3P7V75OdgLB4Gamos8scaBSE8 +# sha256: aRGJZUEOyG1E3GuQF-4929WC4MCr7vYrOhnb9sitEys +# sha256: 0E34aG7IZNDK3ozxmff4OuzUFhCaIINNVo-DEN7RLeo +# sha256: 51qUfhXul-fnHgLqMC_rL8YtOiu0Zov5377UOlBqx-c +# sha256: TkXSL7iDIipaufKCoRb-xe4ujRpWjM_2otdbvQ62vPw +# sha256: vOkzm7PHpV4IA7Y9IcWDno5Hm8hcSt9CrkFbcvlPrLI +# sha256: koE4NlJFoOiGmlmZ-8wqRUdaCm7VKklNYNvcVAM1_t0 +# sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro +# sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I +zope.interface==4.1.3 + +# sha256: ilvjjTWOS86xchl0WBZ0YOAw_0rmqdnjNsxb1hq2RD8 +# sha256: T37KMj0TnsuvHIzCCmoww2fpfpOBTj7cd4NAqucXcpw +acme==0.4.0 + +# sha256: 33BQiANlNLGqGpirTfdCEElTF9YbpaKiYpTbK4zeGD8 +# sha256: lwsV1OdEzzlMeb08C_PRxaCXZ2vOk_1AI2755rZHmPM +letsencrypt==0.4.0 + +# sha256: D3YDaVFjLsMSEfjI5B5D5tn5FeWUtNHYXCObw3ih2tg +# sha256: VTgvsePYGRmI4IOSAnxoYFHd8KciD73bxIuIHtbVFd8 +letsencrypt-apache==0.4.0 + +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +mock==1.0.1 + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" +#!/usr/bin/env python +"""peep ("prudently examine every package") verifies that packages conform to a +trusted, locally stored hash and only then installs them:: + + peep install -r requirements.txt + +This makes your deployments verifiably repeatable without having to maintain a +local PyPI mirror or use a vendor lib. Just update the version numbers and +hashes in requirements.txt, and you're all set. + +""" +# This is here so embedded copies of peep.py are MIT-compliant: +# Copyright (c) 2013 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +try: + xrange = xrange +except NameError: + xrange = range +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify +import cgi +from collections import defaultdict +from functools import wraps +from hashlib import sha256 +from itertools import chain, islice +import mimetypes +from optparse import OptionParser +from os.path import join, basename, splitext, isdir +from pickle import dumps, loads +import re +import sys +from shutil import rmtree, copy +from sys import argv, exit +from tempfile import mkdtemp +import traceback +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 +# TODO: Probably use six to make urllib stuff work across 2/3. + +from pkg_resources import require, VersionConflict, DistributionNotFound + +# We don't admit our dependency on pip in setup.py, lest a naive user simply +# say `pip install peep.tar.gz` and thus pull down an untrusted copy of pip +# from PyPI. Instead, we make sure it's installed and new enough here and spit +# out an error message if not: + + +def activate(specifier): + """Make a compatible version of pip importable. Raise a RuntimeError if we + couldn't.""" + try: + for distro in require(specifier): + distro.activate() + except (VersionConflict, DistributionNotFound): + raise RuntimeError('The installed version of pip is too old; peep ' + 'requires ' + specifier) + +# Before 0.6.2, the log module wasn't there, so some +# of our monkeypatching fails. It probably wouldn't be +# much work to support even earlier, though. +activate('pip>=0.6.2') + +import pip +from pip.commands.install import InstallCommand +try: + from pip.download import url_to_path # 1.5.6 +except ImportError: + try: + from pip.util import url_to_path # 0.7.0 + except ImportError: + from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.index import PackageFinder, Link +try: + from pip.log import logger +except ImportError: + from pip import logger # 6.0 +from pip.req import parse_requirements +try: + from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner +except ImportError: + class NullProgressBar(object): + def __init__(self, *args, **kwargs): + pass + + def iter(self, ret, *args, **kwargs): + return ret + + DownloadProgressBar = DownloadProgressSpinner = NullProgressBar + +__version__ = 3, 0, 0 + +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True + + +ITS_FINE_ITS_FINE = 0 +SOMETHING_WENT_WRONG = 1 +# "Traditional" for command-line errors according to optparse docs: +COMMAND_LINE_ERROR = 2 + +ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') + +MARKER = object() + + +class PipException(Exception): + """When I delegated to pip, it exited with an error.""" + + def __init__(self, error_code): + self.error_code = error_code + + +class UnsupportedRequirementError(Exception): + """An unsupported line was encountered in a requirements file.""" + + +class DownloadError(Exception): + def __init__(self, link, exc): + self.link = link + self.reason = str(exc) + + def __str__(self): + return 'Downloading %s failed: %s' % (self.link, self.reason) + + +def encoded_hash(sha): + """Return a short, 7-bit-safe representation of a hash. + + If you pass a sha256, this results in the hash algorithm that the Wheel + format (PEP 427) uses, except here it's intended to be run across the + downloaded archive before unpacking. + + """ + return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') + + +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + +def run_pip(initial_args): + """Delegate to pip the given args (starting with the subcommand), and raise + ``PipException`` if something goes wrong.""" + status_code = pip.main(initial_args) + + # Clear out the registrations in the pip "logger" singleton. Otherwise, + # loggers keep getting appended to it with every run. Pip assumes only one + # command invocation will happen per interpreter lifetime. + logger.consumers = [] + + if status_code: + raise PipException(status_code) + + +def hash_of_file(path): + """Return the hash of a downloaded file.""" + with open(path, 'rb') as archive: + sha = sha256() + while True: + data = archive.read(2 ** 20) + if not data: + break + sha.update(data) + return encoded_hash(sha) + + +def is_git_sha(text): + """Return whether this is probably a git sha""" + # Handle both the full sha as well as the 7-character abbreviation + if len(text) in (40, 7): + try: + int(text, 16) + return True + except ValueError: + pass + return False + + +def filename_from_url(url): + parsed = urlparse(url) + path = parsed.path + return path.split('/')[-1] + + +def requirement_args(argv, want_paths=False, want_other=False): + """Return an iterable of filtered arguments. + + :arg argv: Arguments, starting after the subcommand + :arg want_paths: If True, the returned iterable includes the paths to any + requirements files following a ``-r`` or ``--requirement`` option. + :arg want_other: If True, the returned iterable includes the args that are + not a requirement-file path or a ``-r`` or ``--requirement`` flag. + + """ + was_r = False + for arg in argv: + # Allow for requirements files named "-r", don't freak out if there's a + # trailing "-r", etc. + if was_r: + if want_paths: + yield arg + was_r = False + elif arg in ['-r', '--requirement']: + was_r = True + else: + if want_other: + yield arg + +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') + +HASH_COMMENT_RE = re.compile( + r""" + \s*\#\s+ # Lines that start with a '#' + (?Psha256):\s+ # Hash type is hardcoded to be sha256 for now. + (?P[^\s]+) # Hashes can be anything except '#' or spaces. + \s* # Suck up whitespace before the comment or + # just trailing whitespace if there is no + # comment. Also strip trailing newlines. + (?:\#(?P.*))? # Comments can be anything after a whitespace+# + # and are optional. + $""", re.X) + + +def peep_hash(argv): + """Return the peep hash of one or more files, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + parser = OptionParser( + usage='usage: %prog hash file [file ...]', + description='Print a peep hash line for one or more files: for ' + 'example, "# sha256: ' + 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY".') + _, paths = parser.parse_args(args=argv) + if paths: + for path in paths: + print('# sha256:', hash_of_file(path)) + return ITS_FINE_ITS_FINE + else: + parser.print_usage() + return COMMAND_LINE_ERROR + + +class EmptyOptions(object): + """Fake optparse options for compatibility with pip<1.2 + + pip<1.2 had a bug in parse_requirements() in which the ``options`` kwarg + was required. We work around that by passing it a mock object. + + """ + default_vcs = None + skip_requirements_regex = None + isolated_mode = False + + +def memoize(func): + """Memoize a method that should return the same result every time on a + given instance. + + """ + @wraps(func) + def memoizer(self): + if not hasattr(self, '_cache'): + self._cache = {} + if func.__name__ not in self._cache: + self._cache[func.__name__] = func(self) + return self._cache[func.__name__] + return memoizer + + +def package_finder(argv): + """Return a PackageFinder respecting command-line options. + + :arg argv: Everything after the subcommand + + """ + # We instantiate an InstallCommand and then use some of its private + # machinery--its arg parser--for our own purposes, like a virus. This + # approach is portable across many pip versions, where more fine-grained + # ones are not. Ignoring options that don't exist on the parser (for + # instance, --use-wheel) gives us a straightforward method of backward + # compatibility. + try: + command = InstallCommand() + except TypeError: + # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 + # given)" error. In that version, InstallCommand takes a top=level + # parser passed in from outside. + from pip.baseparser import create_main_parser + command = InstallCommand(create_main_parser()) + # The downside is that it essentially ruins the InstallCommand class for + # further use. Calling out to pip.main() within the same interpreter, for + # example, would result in arguments parsed this time turning up there. + # Thus, we deepcopy the arg parser so we don't trash its singletons. Of + # course, deepcopy doesn't work on these objects, because they contain + # uncopyable regex patterns, so we pickle and unpickle instead. Fun! + options, _ = loads(dumps(command.parser)).parse_args(argv) + + # Carry over PackageFinder kwargs that have [about] the same names as + # options attr names: + possible_options = [ + 'find_links', + FORMAT_CONTROL_ARG, + ('allow_all_prereleases', 'pre'), + 'process_dependency_links' + ] + kwargs = {} + for option in possible_options: + kw, attr = option if isinstance(option, tuple) else (option, option) + value = getattr(options, attr, MARKER) + if value is not MARKER: + kwargs[kw] = value + + # Figure out index_urls: + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + index_urls = [] + index_urls += getattr(options, 'mirrors', []) + + # If pip is new enough to have a PipSession, initialize one, since + # PackageFinder requires it: + if hasattr(command, '_build_session'): + kwargs['session'] = command._build_session(options) + + return PackageFinder(index_urls=index_urls, **kwargs) + + +class DownloadedReq(object): + """A wrapper around InstallRequirement which offers additional information + based on downloading and examining a corresponding package archive + + These are conceptually immutable, so we can get away with memoizing + expensive things. + + """ + def __init__(self, req, argv, finder): + """Download a requirement, compare its hashes, and return a subclass + of DownloadedReq depending on its state. + + :arg req: The InstallRequirement I am based on + :arg argv: The args, starting after the subcommand + + """ + self._req = req + self._argv = argv + self._finder = finder + + # We use a separate temp dir for each requirement so requirements + # (from different indices) that happen to have the same archive names + # don't overwrite each other, leading to a security hole in which the + # latter is a hash mismatch, the former has already passed the + # comparison, and the latter gets installed. + self._temp_path = mkdtemp(prefix='peep-') + # Think of DownloadedReq as a one-shot state machine. It's an abstract + # class that ratchets forward to being one of its own subclasses, + # depending on its package status. Then it doesn't move again. + self.__class__ = self._class() + + def dispose(self): + """Delete temp files and dirs I've made. Render myself useless. + + Do not call further methods on me after calling dispose(). + + """ + rmtree(self._temp_path) + + def _version(self): + """Deduce the version number of the downloaded package from its filename.""" + # TODO: Can we delete this method and just print the line from the + # reqs file verbatim instead? + def version_of_archive(filename, package_name): + # Since we know the project_name, we can strip that off the left, strip + # any archive extensions off the right, and take the rest as the + # version. + for ext in ARCHIVE_EXTENSIONS: + if filename.endswith(ext): + filename = filename[:-len(ext)] + break + # Handle github sha tarball downloads. + if is_git_sha(filename): + filename = package_name + '-' + filename + if not filename.lower().replace('_', '-').startswith(package_name.lower()): + # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -? + give_up(filename, package_name) + return filename[len(package_name) + 1:] # Strip off '-' before version. + + def version_of_wheel(filename, package_name): + # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file- + # name-convention) we know the format bits are '-' separated. + whl_package_name, version, _rest = filename.split('-', 2) + # Do the alteration to package_name from PEP 427: + our_package_name = re.sub(r'[^\w\d.]+', '_', package_name, re.UNICODE) + if whl_package_name != our_package_name: + give_up(filename, whl_package_name) + return version + + def give_up(filename, package_name): + raise RuntimeError("The archive '%s' didn't start with the package name " + "'%s', so I couldn't figure out the version number. " + "My bad; improve me." % + (filename, package_name)) + + get_version = (version_of_wheel + if self._downloaded_filename().endswith('.whl') + else version_of_archive) + return get_version(self._downloaded_filename(), self._project_name()) + + def _is_always_unsatisfied(self): + """Returns whether this requirement is always unsatisfied + + This would happen in cases where we can't determine the version + from the filename. + + """ + # If this is a github sha tarball, then it is always unsatisfied + # because the url has a commit sha in it and not the version + # number. + url = self._url() + if url: + filename = filename_from_url(url) + if filename.endswith(ARCHIVE_EXTENSIONS): + filename, ext = splitext(filename) + if is_git_sha(filename): + return True + return False + + @memoize # Avoid hitting the file[cache] over and over. + def _expected_hashes(self): + """Return a list of known-good hashes for this package.""" + return hashes_above(*path_and_line(self._req)) + + def _download(self, link): + """Download a file, and return its name within my temp dir. + + This does no verification of HTTPS certs, but our checking hashes + makes that largely unimportant. It would be nice to be able to use the + requests lib, which can verify certs, but it is guaranteed to be + available only in pip >= 1.5. + + This also drops support for proxies and basic auth, though those could + be added back in. + + """ + # Based on pip 1.4.1's URLOpener but with cert verification removed + def opener(is_https): + if is_https: + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + else: + opener = build_opener() + return opener + + # Descended from unpack_http_url() in pip 1.4.1 + def best_filename(link, response): + """Return the most informative possible filename for a download, + ideally with a proper extension. + + """ + content_type = response.info().get('content-type', '') + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess: + content_disposition = response.info().get('content-disposition') + if content_disposition: + type, params = cgi.parse_header(content_disposition) + # We use ``or`` here because we don't want to use an "empty" value + # from the filename param: + filename = params.get('filename') or filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != response.geturl(): + ext = splitext(response.geturl())[1] + if ext: + filename += ext + return filename + + # Descended from _download_url() in pip 1.4.1 + def pipe_to_file(response, path, size=0): + """Pull the data off an HTTP response, shove it in a new file, and + show progress. + + :arg response: A file-like object to read from + :arg path: The path of the new file + :arg size: The expected size, in bytes, of the download. 0 for + unknown or to suppress progress indication (as for cached + downloads) + + """ + def response_chunks(chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + print('Downloading %s%s...' % ( + self._req.req, + (' (%sK)' % (size / 1000)) if size > 1000 else '')) + progress_indicator = (DownloadProgressBar(max=size).iter if size + else DownloadProgressSpinner().iter) + with open(path, 'wb') as file: + for chunk in progress_indicator(response_chunks(4096), 4096): + file.write(chunk) + + url = link.url.split('#', 1)[0] + try: + response = opener(urlparse(url).scheme != 'http').open(url) + except (HTTPError, IOError) as exc: + raise DownloadError(link, exc) + filename = best_filename(link, response) + try: + size = int(response.headers['content-length']) + except (ValueError, KeyError, TypeError): + size = 0 + pipe_to_file(response, join(self._temp_path, filename), size=size) + return filename + + # Based on req_set.prepare_files() in pip bb2a8428d4aebc8d313d05d590f386fa3f0bbd0f + @memoize # Avoid re-downloading. + def _downloaded_filename(self): + """Download the package's archive if necessary, and return its + filename. + + --no-deps is implied, as we have reimplemented the bits that would + ordinarily do dependency resolution. + + """ + # Peep doesn't support requirements that don't come down as a single + # file, because it can't hash them. Thus, it doesn't support editable + # requirements, because pip itself doesn't support editable + # requirements except for "local projects or a VCS url". Nor does it + # support VCS requirements yet, because we haven't yet come up with a + # portable, deterministic way to hash them. In summary, all we support + # is == requirements and tarballs/zips/etc. + + # TODO: Stop on reqs that are editable or aren't ==. + + # If the requirement isn't already specified as a URL, get a URL + # from an index: + link = self._link() or self._finder.find_requirement(self._req, upgrade=False) + + if link: + lower_scheme = link.scheme.lower() # pip lower()s it for some reason. + if lower_scheme == 'http' or lower_scheme == 'https': + file_path = self._download(link) + return basename(file_path) + elif lower_scheme == 'file': + # The following is inspired by pip's unpack_file_url(): + link_path = url_to_path(link.url_without_fragment) + if isdir(link_path): + raise UnsupportedRequirementError( + "%s: %s is a directory. So that it can compute " + "a hash, peep supports only filesystem paths which " + "point to files" % + (self._req, link.url_without_fragment)) + else: + copy(link_path, self._temp_path) + return basename(link_path) + else: + raise UnsupportedRequirementError( + "%s: The download link, %s, would not result in a file " + "that can be hashed. Peep supports only == requirements, " + "file:// URLs pointing to files (not folders), and " + "http:// and https:// URLs pointing to tarballs, zips, " + "etc." % (self._req, link.url)) + else: + raise UnsupportedRequirementError( + "%s: couldn't determine where to download this requirement from." + % (self._req,)) + + def install(self): + """Install the package I represent, without dependencies. + + Obey typical pip-install options passed in on the command line. + + """ + other_args = list(requirement_args(self._argv, want_other=True)) + archive_path = join(self._temp_path, self._downloaded_filename()) + # -U so it installs whether pip deems the requirement "satisfied" or + # not. This is necessary for GitHub-sourced zips, which change without + # their version numbers changing. + run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path]) + + @memoize + def _actual_hash(self): + """Download the package's archive if necessary, and return its hash.""" + return hash_of_file(join(self._temp_path, self._downloaded_filename())) + + def _project_name(self): + """Return the inner Requirement's "unsafe name". + + Raise ValueError if there is no name. + + """ + name = getattr(self._req.req, 'project_name', '') + if name: + return name + raise ValueError('Requirement has no project_name.') + + def _name(self): + return self._req.name + + def _link(self): + try: + return self._req.link + except AttributeError: + # The link attribute isn't available prior to pip 6.1.0, so fall + # back to the now deprecated 'url' attribute. + return Link(self._req.url) if self._req.url else None + + def _url(self): + link = self._link() + return link.url if link else None + + @memoize # Avoid re-running expensive check_if_exists(). + def _is_satisfied(self): + self._req.check_if_exists() + return (self._req.satisfied_by and + not self._is_always_unsatisfied()) + + def _class(self): + """Return the class I should be, spanning a continuum of goodness.""" + try: + self._project_name() + except ValueError: + return MalformedReq + if self._is_satisfied(): + return SatisfiedReq + if not self._expected_hashes(): + return MissingReq + if self._actual_hash() not in self._expected_hashes(): + return MismatchedReq + return InstallableReq + + @classmethod + def foot(cls): + """Return the text to be printed once, after all of the errors from + classes of my type are printed. + + """ + return '' + + +class MalformedReq(DownloadedReq): + """A requirement whose package name could not be determined""" + + @classmethod + def head(cls): + return 'The following requirements could not be processed:\n' + + def error(self): + return '* Unable to determine package name from URL %s; add #egg=' % self._url() + + +class MissingReq(DownloadedReq): + """A requirement for which no hashes were specified in the requirements file""" + + @classmethod + def head(cls): + return ('The following packages had no hashes specified in the requirements file, which\n' + 'leaves them open to tampering. Vet these packages to your satisfaction, then\n' + 'add these "sha256" lines like so:\n\n') + + def error(self): + if self._url(): + # _url() always contains an #egg= part, or this would be a + # MalformedRequest. + line = self._url() + else: + line = '%s==%s' % (self._name(), self._version()) + return '# sha256: %s\n%s\n' % (self._actual_hash(), line) + + +class MismatchedReq(DownloadedReq): + """A requirement for which the downloaded file didn't match any of my hashes.""" + @classmethod + def head(cls): + return ("THE FOLLOWING PACKAGES DIDN'T MATCH THE HASHES SPECIFIED IN THE REQUIREMENTS\n" + "FILE. If you have updated the package versions, update the hashes. If not,\n" + "freak out, because someone has tampered with the packages.\n\n") + + def error(self): + preamble = ' %s: expected' % self._project_name() + if len(self._expected_hashes()) > 1: + preamble += ' one of' + padding = '\n' + ' ' * (len(preamble) + 1) + return '%s %s\n%s got %s' % (preamble, + padding.join(self._expected_hashes()), + ' ' * (len(preamble) - 4), + self._actual_hash()) + + @classmethod + def foot(cls): + return '\n' + + +class SatisfiedReq(DownloadedReq): + """A requirement which turned out to be already installed""" + + @classmethod + def head(cls): + return ("These packages were already installed, so we didn't need to download or build\n" + "them again. If you installed them with peep in the first place, you should be\n" + "safe. If not, uninstall them, then re-attempt your install with peep.\n") + + def error(self): + return ' %s' % (self._req,) + + +class InstallableReq(DownloadedReq): + """A requirement whose hash matched and can be safely installed""" + + +# DownloadedReq subclasses that indicate an error that should keep us from +# going forward with installation, in the order in which their errors should +# be reported: +ERROR_CLASSES = [MismatchedReq, MissingReq, MalformedReq] + + +def bucket(things, key): + """Return a map of key -> list of things.""" + ret = defaultdict(list) + for thing in things: + ret[key(thing)].append(thing) + return ret + + +def first_every_last(iterable, first, every, last): + """Execute something before the first item of iter, something else for each + item, and a third thing after the last. + + If there are no items in the iterable, don't execute anything. + + """ + did_first = False + for item in iterable: + if not did_first: + did_first = True + first(item) + every(item) + if did_first: + last(item) + + +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + +def downloaded_reqs_from_path(path, argv): + """Return a list of DownloadedReqs representing the requirements parsed + out of a given requirements file. + + :arg path: The path to the requirements file + :arg argv: The commandline args, starting after the subcommand + + """ + finder = package_finder(argv) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] + + +def peep_install(argv): + """Perform the ``peep install`` subcommand, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + output = [] + out = output.append + reqs = [] + try: + req_paths = list(requirement_args(argv, want_paths=True)) + if not req_paths: + out("You have to specify one or more requirements files with the -r option, because\n" + "otherwise there's nowhere for peep to look up the hashes.\n") + return COMMAND_LINE_ERROR + + # We're a "peep install" command, and we have some requirement paths. + reqs = list(chain.from_iterable( + downloaded_reqs_from_path(path, argv) + for path in req_paths)) + buckets = bucket(reqs, lambda r: r.__class__) + + # Skip a line after pip's "Cleaning up..." so the important stuff + # stands out: + if any(buckets[b] for b in ERROR_CLASSES): + out('\n') + + printers = (lambda r: out(r.head()), + lambda r: out(r.error() + '\n'), + lambda r: out(r.foot())) + for c in ERROR_CLASSES: + first_every_last(buckets[c], *printers) + + if any(buckets[b] for b in ERROR_CLASSES): + out('-------------------------------\n' + 'Not proceeding to installation.\n') + return SOMETHING_WENT_WRONG + else: + for req in buckets[InstallableReq]: + req.install() + + first_every_last(buckets[SatisfiedReq], *printers) + + return ITS_FINE_ITS_FINE + except (UnsupportedRequirementError, DownloadError) as exc: + out(str(exc)) + return SOMETHING_WENT_WRONG + finally: + for req in reqs: + req.dispose() + print(''.join(output)) + + +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + +def main(): + """Be the top-level entrypoint. Return a shell status code.""" + commands = {'hash': peep_hash, + 'install': peep_install, + 'port': peep_port} + try: + if len(argv) >= 2 and argv[1] in commands: + return commands[argv[1]](argv[2:]) + else: + # Fall through to top-level pip main() for everything else: + return pip.main() + except PipException as exc: + return exc.error_code + + +def exception_handler(exc_type, exc_value, exc_tb): + print('Oh no! Peep had a problem while trying to do stuff. Please write up a bug report') + print('with the specifics so we can fix it:') + print() + print('https://github.com/erikrose/peep/issues/new') + print() + print('Here are some particulars you can copy and paste into the bug report:') + print() + print('---') + print('peep:', repr(__version__)) + print('python:', repr(sys.version)) + print('pip:', repr(getattr(pip, '__version__', 'no __version__ attr'))) + print('Command line: ', repr(sys.argv)) + print( + ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))) + print('---') + + +if __name__ == '__main__': + try: + exit(main()) + except Exception: + exception_handler(*sys.exc_info()) + exit(SOMETHING_WENT_WRONG) + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + set +e + PEEP_OUT=`"$VENV_BIN/python" "$TEMP_DIR/peep.py" install -r "$TEMP_DIR/letsencrypt-auto-requirements.txt"` + PEEP_STATUS=$? + set -e + rm -rf "$TEMP_DIR" + if [ "$PEEP_STATUS" != 0 ]; then + # Report error. (Otherwise, be quiet.) + echo "Had a problem while downloading and verifying Python packages:" + echo "$PEEP_OUT" + exit 1 + fi + fi + echo "Requesting root privileges to run letsencrypt..." + echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" + $SUDO "$VENV_BIN/letsencrypt" "$@" +else + # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # letsencrypt-auto (which is always the same as that of the letsencrypt + # package). Phase 2 checks the version of the locally installed letsencrypt. + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$OS_PACKAGES_ONLY" = 1 ]; then + echo "OS packages installed." + exit 0 + fi + + echo "Checking for new version..." + TEMP_DIR=$(TempDir) + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 + +On failure, return non-zero. + +""" +from distutils.version import LooseVersion +from json import loads +from os import devnull, environ +from os.path import dirname, join +import re +from subprocess import check_call, CalledProcessError +from sys import argv, exit +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB +-----END PUBLIC KEY----- +""") + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/letsencrypt/json'))) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp_dir): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. + + """ + le_auto_dir = environ.get( + 'LE_AUTO_DIR_TEMPLATE', + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'letsencrypt-auto-source/') % tag + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp_dir, 'public_key.pem'), + '-signature', + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + + +def main(): + get = HttpsGetter().get + flag = argv[1] + try: + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) + +UNLIKELY_EOF + # --------------------------------------------------------------------------- + DeterminePythonVersion + REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` + if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Replacing letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + rm -rf "$TEMP_DIR" + fi # should upgrade + "$0" --no-self-upgrade "$@" +fi diff --git a/tools/release.sh b/tools/release.sh index 6ec83053f..02e3d00b8 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -171,7 +171,10 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" done -git add letsencrypt-auto-source +# copy leauto to the root, overwriting the previous release version +cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto + +git add letsencrypt-auto letsencrypt-auto-source git diff --cached git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" From 0062678f7d4c5e7ced4c0c1e9e18c3b155bf7208 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 19 Feb 2016 12:20:32 -0800 Subject: [PATCH 39/64] A temporary save is a good save --- letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index cc1d749a0..30a3d3ef5 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -62,7 +62,7 @@ class ApacheTlsSni01(common.TLSSNI01): return [] # Save any changes to the configuration as a precaution # About to make temporary changes to the config - self.configurator.save() + self.configurator.save("Changes before challenge setup", True) # Prepare the server for HTTPS self.configurator.prepare_server_https( From f9a3abeeae43e286cdf94f3198c709a594f9171f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 19 Feb 2016 17:58:37 -0800 Subject: [PATCH 40/64] fixes #2247 --- letsencrypt-apache/letsencrypt_apache/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 3c13aae5f..f49ac0acc 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -477,7 +477,7 @@ class ApacheParser(object): # Note: This works for augeas globs, ie. *.conf if use_new: inc_test = self.aug.match( - "/augeas/load/Httpd/incl [. ='%s']" % filepath) + "/augeas/load/Httpd['%s' =~ glob(incl)]" % filepath) if not inc_test: # Load up files # This doesn't seem to work on TravisCI From b6142c13d65e45c9b7e624fd8c416d604dc95c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roy=20Wellington=20=E2=85=A3?= Date: Fri, 19 Feb 2016 22:08:40 -0800 Subject: [PATCH 41/64] Change zope's implements to be a class decorator. When attempting to import any module that uses zope.interface.implements in Python 3, a TypeError is raised; it reads: TypeError: Class advice impossible in Python3. Use the @implementer class decorator instead. Following the listed advice seems to function in Python 3. --- examples/plugins/letsencrypt_example_plugins.py | 4 ++-- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- .../configurators/apache/apache24.py | 3 +-- .../configurators/apache/common.py | 3 +-- .../letsencrypt_compatibility_test/validator.py | 2 +- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 2 +- letsencrypt/configuration.py | 2 +- letsencrypt/continuity_auth.py | 2 +- letsencrypt/display/util.py | 9 +++------ letsencrypt/plugins/common.py | 2 +- letsencrypt/plugins/manual.py | 2 +- letsencrypt/plugins/null.py | 2 +- letsencrypt/plugins/standalone.py | 2 +- letsencrypt/plugins/webroot.py | 2 +- letsencrypt/reporter.py | 2 +- 15 files changed, 18 insertions(+), 23 deletions(-) diff --git a/examples/plugins/letsencrypt_example_plugins.py b/examples/plugins/letsencrypt_example_plugins.py index 2810d0d40..990d7787c 100644 --- a/examples/plugins/letsencrypt_example_plugins.py +++ b/examples/plugins/letsencrypt_example_plugins.py @@ -9,9 +9,9 @@ from letsencrypt import interfaces from letsencrypt.plugins import common +@zope.interface.implementer(interfaces.IAuthenticator) class Authenticator(common.Plugin): """Example Authenticator.""" - zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) description = "Example Authenticator plugin" @@ -20,9 +20,9 @@ class Authenticator(common.Plugin): # "self" as first argument, e.g. def prepare(self)... +@zope.interface.implementer(interfaces.IInstaller) class Installer(common.Plugin): """Example Installer.""" - zope.interface.implements(interfaces.IInstaller) zope.interface.classProvides(interfaces.IPluginFactory) description = "Example Installer plugin" diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f4a407974..20e73eb1e 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -60,6 +60,7 @@ logger = logging.getLogger(__name__) # sites-available doesn't allow immediate find_dir search even with save() # and load() +@zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Apache configurator. @@ -80,7 +81,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :ivar dict assoc: Mapping between domains and vhosts """ - zope.interface.implements(interfaces.IAuthenticator, interfaces.IInstaller) zope.interface.classProvides(interfaces.IPluginFactory) description = "Apache Web Server - Alpha" diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py index 3cc6fdf8e..a68f53689 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py @@ -34,11 +34,10 @@ SHARED_MODULES = { "vhost_alias"} +@zope.interface.implementer(interfaces.IConfiguratorProxy) class Proxy(apache_common.Proxy): """Wraps the ApacheConfigurator for Apache 2.4 tests""" - zope.interface.implements(interfaces.IConfiguratorProxy) - def __init__(self, args): """Initializes the plugin with the given command line args""" super(Proxy, self).__init__(args) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py index 5fef8c47f..d383963a3 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py @@ -19,12 +19,11 @@ APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) APACHE_COMMANDS = ["apachectl", "a2enmod", "a2dismod"] +@zope.interface.implementer(interfaces.IConfiguratorProxy) class Proxy(configurators_common.Proxy): # pylint: disable=too-many-instance-attributes """A common base for Apache test configurators""" - zope.interface.implements(interfaces.IConfiguratorProxy) - def __init__(self, args): """Initializes the plugin with the given command line args""" super(Proxy, self).__init__(args) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py index e5386f290..90ce108c0 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py @@ -12,10 +12,10 @@ from letsencrypt import interfaces logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IValidator) class Validator(object): # pylint: disable=no-self-use """Collection of functions to test a live webserver's configuration""" - zope.interface.implements(interfaces.IValidator) def certificate(self, cert, name, alt_host=None, port=443): """Verifies the certificate presented at name is cert""" diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index d2f45a8c2..876d843f5 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -31,6 +31,7 @@ from letsencrypt_nginx import parser logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) class NginxConfigurator(common.Plugin): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Nginx configurator. @@ -52,7 +53,6 @@ class NginxConfigurator(common.Plugin): :ivar tup version: version of Nginx """ - zope.interface.implements(interfaces.IAuthenticator, interfaces.IInstaller) zope.interface.classProvides(interfaces.IPluginFactory) description = "Nginx Web Server - currently doesn't work" diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index c49751a6c..062722346 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -11,6 +11,7 @@ from letsencrypt import interfaces from letsencrypt import le_util +@zope.interface.implementer(interfaces.IConfig) class NamespaceConfig(object): """Configuration wrapper around :class:`argparse.Namespace`. @@ -32,7 +33,6 @@ class NamespaceConfig(object): :type namespace: :class:`argparse.Namespace` """ - zope.interface.implements(interfaces.IConfig) def __init__(self, namespace): self.namespace = namespace diff --git a/letsencrypt/continuity_auth.py b/letsencrypt/continuity_auth.py index 52d0cee8e..28612bb17 100644 --- a/letsencrypt/continuity_auth.py +++ b/letsencrypt/continuity_auth.py @@ -9,6 +9,7 @@ from letsencrypt import interfaces from letsencrypt import proof_of_possession +@zope.interface.implementer(interfaces.IAuthenticator) class ContinuityAuthenticator(object): """IAuthenticator for :const:`~acme.challenges.ContinuityChallenge` class challenges. @@ -18,7 +19,6 @@ class ContinuityAuthenticator(object): :class:`letsencrypt.proof_of_possession.Proof_of_Possession` """ - zope.interface.implements(interfaces.IAuthenticator) # This will have an installer soon for get_key/cert purposes def __init__(self, config, installer): # pylint: disable=unused-argument diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 976a2afdf..84049c47c 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -36,11 +36,10 @@ def _wrap_lines(msg): fixed_l.append(textwrap.fill(line, 80)) return os.linesep.join(fixed_l) +@zope.interface.implementer(interfaces.IDisplay) class NcursesDisplay(object): """Ncurses-based display.""" - zope.interface.implements(interfaces.IDisplay) - def __init__(self, width=WIDTH, height=HEIGHT): super(NcursesDisplay, self).__init__() self.dialog = dialog.Dialog() @@ -176,11 +175,10 @@ class NcursesDisplay(object): message, width=self.width, height=self.height, choices=choices) +@zope.interface.implementer(interfaces.IDisplay) class FileDisplay(object): """File-based display.""" - zope.interface.implements(interfaces.IDisplay) - def __init__(self, outfile): super(FileDisplay, self).__init__() self.outfile = outfile @@ -411,11 +409,10 @@ class FileDisplay(object): return OK, selection +@zope.interface.implementer(interfaces.IDisplay) class NoninteractiveDisplay(object): """An iDisplay implementation that never asks for interactive user input""" - zope.interface.implements(interfaces.IDisplay) - def __init__(self, outfile): super(NoninteractiveDisplay, self).__init__() self.outfile = outfile diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 37738f5c0..2a32df96e 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -31,9 +31,9 @@ hostname_regex = re.compile( r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$", re.IGNORECASE) +@zope.interface.implementer(interfaces.IPlugin) class Plugin(object): """Generic plugin.""" - zope.interface.implements(interfaces.IPlugin) # classProvides is not inherited, subclasses must define it on their own #zope.interface.classProvides(interfaces.IPluginFactory) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 0e516b5b0..248b4ca58 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -23,6 +23,7 @@ from letsencrypt.plugins import common logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IAuthenticator) class Authenticator(common.Plugin): """Manual Authenticator. @@ -34,7 +35,6 @@ class Authenticator(common.Plugin): .. todo:: Support for `~.challenges.TLSSNI01`. """ - zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) hidden = True diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index cdb96a116..55734a16d 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -11,9 +11,9 @@ from letsencrypt.plugins import common logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IInstaller) class Installer(common.Plugin): """Null installer.""" - zope.interface.implements(interfaces.IInstaller) zope.interface.classProvides(interfaces.IPluginFactory) description = "Null Installer" diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index a27b9f5c8..1bb7da658 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -135,6 +135,7 @@ def supported_challenges_validator(data): return data +@zope.interface.implementer(interfaces.IAuthenticator) class Authenticator(common.Plugin): """Standalone Authenticator. @@ -143,7 +144,6 @@ class Authenticator(common.Plugin): challenges from the certificate authority. Therefore, it does not rely on any existing server program. """ - zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) description = "Automatically use a temporary webserver" diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index 49f779bb8..67dd36686 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -17,9 +17,9 @@ from letsencrypt.plugins import common logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IAuthenticator) class Authenticator(common.Plugin): """Webroot Authenticator.""" - zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) description = "Webroot Authenticator" diff --git a/letsencrypt/reporter.py b/letsencrypt/reporter.py index c0c7856a7..81106be34 100644 --- a/letsencrypt/reporter.py +++ b/letsencrypt/reporter.py @@ -17,6 +17,7 @@ from letsencrypt import le_util logger = logging.getLogger(__name__) +@zope.interface.implementer(interfaces.IReporter) class Reporter(object): """Collects and displays information to the user. @@ -24,7 +25,6 @@ class Reporter(object): the user. """ - zope.interface.implements(interfaces.IReporter) HIGH_PRIORITY = 0 """High priority constant. See `add_message`.""" From e9d981acebd0c0d4ba4b5e504fc4687dbc6610e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roy=20Wellington=20=E2=85=A3?= Date: Fri, 19 Feb 2016 22:08:40 -0800 Subject: [PATCH 42/64] Change zope's classProvides to be a class decorator. When attempting to import any module that uses zope.interface.classProvides in Python 3, a TypeError is raised; it reads: TypeError: Class advice impossible in Python3. Use the @provider class decorator instead. Following the listed advice seems to function in Python 3. --- examples/plugins/letsencrypt_example_plugins.py | 4 ++-- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 2 +- letsencrypt/plugins/common.py | 4 ++-- letsencrypt/plugins/manual.py | 2 +- letsencrypt/plugins/null.py | 2 +- letsencrypt/plugins/standalone.py | 2 +- letsencrypt/plugins/webroot.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/plugins/letsencrypt_example_plugins.py b/examples/plugins/letsencrypt_example_plugins.py index 990d7787c..5c22ca7ff 100644 --- a/examples/plugins/letsencrypt_example_plugins.py +++ b/examples/plugins/letsencrypt_example_plugins.py @@ -10,9 +10,9 @@ from letsencrypt.plugins import common @zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): """Example Authenticator.""" - zope.interface.classProvides(interfaces.IPluginFactory) description = "Example Authenticator plugin" @@ -21,9 +21,9 @@ class Authenticator(common.Plugin): @zope.interface.implementer(interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) class Installer(common.Plugin): """Example Installer.""" - zope.interface.classProvides(interfaces.IPluginFactory) description = "Example Installer plugin" diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 20e73eb1e..6f03ce4ee 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -61,6 +61,7 @@ logger = logging.getLogger(__name__) # and load() @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Apache configurator. @@ -81,7 +82,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :ivar dict assoc: Mapping between domains and vhosts """ - zope.interface.classProvides(interfaces.IPluginFactory) description = "Apache Web Server - Alpha" diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 876d843f5..3a45a2e0e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) class NginxConfigurator(common.Plugin): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Nginx configurator. @@ -53,7 +54,6 @@ class NginxConfigurator(common.Plugin): :ivar tup version: version of Nginx """ - zope.interface.classProvides(interfaces.IPluginFactory) description = "Nginx Web Server - currently doesn't work" diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 2a32df96e..319692344 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -34,8 +34,8 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) class Plugin(object): """Generic plugin.""" - # classProvides is not inherited, subclasses must define it on their own - #zope.interface.classProvides(interfaces.IPluginFactory) + # provider is not inherited, subclasses must define it on their own + # @zope.interface.provider(interfaces.IPluginFactory) def __init__(self, config, name): self.config = config diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 248b4ca58..47c8ff6e4 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): """Manual Authenticator. @@ -35,7 +36,6 @@ class Authenticator(common.Plugin): .. todo:: Support for `~.challenges.TLSSNI01`. """ - zope.interface.classProvides(interfaces.IPluginFactory) hidden = True description = "Manually configure an HTTP server" diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index 55734a16d..2c643d495 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -12,9 +12,9 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IInstaller) +@zope.interface.provider(interfaces.IPluginFactory) class Installer(common.Plugin): """Null installer.""" - zope.interface.classProvides(interfaces.IPluginFactory) description = "Null Installer" hidden = True diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 1bb7da658..acc253bca 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -136,6 +136,7 @@ def supported_challenges_validator(data): @zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): """Standalone Authenticator. @@ -144,7 +145,6 @@ class Authenticator(common.Plugin): challenges from the certificate authority. Therefore, it does not rely on any existing server program. """ - zope.interface.classProvides(interfaces.IPluginFactory) description = "Automatically use a temporary webserver" diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index 67dd36686..0e3ebe1a7 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -18,9 +18,9 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): """Webroot Authenticator.""" - zope.interface.classProvides(interfaces.IPluginFactory) description = "Webroot Authenticator" From 9f372bfa38d9594ea924042a10379b47818fb130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roy=20Wellington=20=E2=85=A3?= Date: Sat, 20 Feb 2016 01:38:45 -0800 Subject: [PATCH 43/64] Don't mix tabs & spaces in Python. It's a bit silly, and might cause someone a lot of grief to debug. --- .../letsencrypt_nginx/tests/parser_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index b64f1dee3..b597fcad5 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -230,23 +230,23 @@ class NginxParserTest(util.NginxTest): def test_parse_server_ssl(self): server = parser.parse_server([ - ['listen', '443'] - ]) + ['listen', '443'] + ]) self.assertFalse(server['ssl']) server = parser.parse_server([ - ['listen', '443 ssl'] - ]) + ['listen', '443 ssl'] + ]) self.assertTrue(server['ssl']) server = parser.parse_server([ - ['listen', '443'], ['ssl', 'off'] - ]) + ['listen', '443'], ['ssl', 'off'] + ]) self.assertFalse(server['ssl']) server = parser.parse_server([ - ['listen', '443'], ['ssl', 'on'] - ]) + ['listen', '443'], ['ssl', 'on'] + ]) self.assertTrue(server['ssl']) if __name__ == "__main__": From 9a36439e1b738c7ead76752572b091cc413c560d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 22 Feb 2016 17:26:55 -0800 Subject: [PATCH 44/64] Tweaks per review --- acme/acme/client.py | 2 +- acme/acme/client_test.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index eaca6dc7d..79d7c5df4 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -4,6 +4,7 @@ import datetime import heapq import logging import time +from email.utils import parsedate_tz import six from six.moves import http_client # pylint: disable=import-error @@ -17,7 +18,6 @@ from acme import jose from acme import jws from acme import messages -from email.utils import parsedate_tz logger = logging.getLogger(__name__) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 29f60c25d..93c86862d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -205,8 +205,12 @@ class ClientTest(unittest.TestCase): datetime.datetime(2015, 3, 27, 0, 0, 10), self.client.retry_after(response=self.response, default=10)) - # wrong date -> ValueError + @mock.patch('acme.client.datetime') + def test_retry_after_overflow(self, dt_mock): + dt_mock.datetime.now.return_value = datetime.datetime(2015, 3, 27) + dt_mock.timedelta = datetime.timedelta dt_mock.datetime.side_effect = datetime.datetime + self.response.headers['Retry-After'] = "Tue, 116 Feb 2016 11:50:00 MST" self.assertEqual( datetime.datetime(2015, 3, 27, 0, 0, 10), From b81079be3b373bb49304b7a61c04f0ab8835433b Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Mon, 22 Feb 2016 18:08:06 -0800 Subject: [PATCH 45/64] brad nits --- letsencrypt-apache/letsencrypt_apache/obj.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index c98ca4b99..b2a21ef5d 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -194,6 +194,8 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods Used in redirection - indicates whether or not the two virtual hosts serve on the exact same IP combinations, but different ports. + The generic flag indicates that that we're trying to match to a + default or generic vhost .. todo:: Handle _default_ @@ -207,8 +209,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods if self.name is not None or self.aliases: return True # If we're looking for a generic vhost, don't return one with a ServerName - else: - if self.name: + elif self.name: return False # Both sets of names are empty. From 7aa5edb2121da9280ee3a6f979c7802459fea2b7 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Mon, 22 Feb 2016 21:31:14 -0800 Subject: [PATCH 46/64] Set CSR version in make_csr --- letsencrypt/crypto_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 76265a739..5fdcba843 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -118,6 +118,7 @@ def make_csr(key_str, domains): value=", ".join("DNS:%s" % d for d in domains) ), ]) + req.set_version(2) req.set_pubkey(pkey) req.sign(pkey, "sha256") return tuple(OpenSSL.crypto.dump_certificate_request(method, req) From 6d1b0298acf0a433c5e29b21958fc808fcf3e5f4 Mon Sep 17 00:00:00 2001 From: Dominic Cleal Date: Tue, 23 Feb 2016 09:35:29 +0000 Subject: [PATCH 47/64] Add failing test from ticket #2525 Augeas fails to parse the last new line/continuation between an IP in a VirtualHost block and the closing `>` of the section. --- .../failing/section-continuations-2525.conf | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf new file mode 100644 index 000000000..6840b71d6 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf @@ -0,0 +1,284 @@ + +NameVirtualHost 0.0.0.0:7080 +NameVirtualHost [00000:000:000:000::0]:7080 +NameVirtualHost 0.0.0.0:7080 + +NameVirtualHost 127.0.0.1:7080 +NameVirtualHost 0.0.0.0:7081 +NameVirtualHost [0000:000:000:000::2]:7081 +NameVirtualHost 0.0.0.0:7081 + +NameVirtualHost 127.0.0.1:7081 + +ServerName "example.com" +ServerAdmin "srv@example.com" + +DocumentRoot "/var/www/vhosts/default/htdocs" + + + LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + +TraceEnable off + +ServerTokens ProductOnly + + + AllowOverride "All" + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + + php_admin_flag engine off + + + + php_admin_flag engine off + + + + + + AllowOverride "All" + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + php_admin_flag engine off + + + php_admin_flag engine off + + + + + Header add X-Powered-By PleskLin + + + + SecRuleEngine DetectionOnly + SecRequestBodyAccess On + SecRequestBodyLimit 134217728 + SecResponseBodyAccess Off + SecResponseBodyLimit 524288 + SecAuditEngine On + SecAuditLog "/var/log/modsec_audit.log" + SecAuditLogType serial + + +Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" + + + ServerName "default" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + + SSLEngine off + + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + + + + ServerName "default-0_0_0_0" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + ServerName "default-0000_000_000_00000__2" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + ServerName "default-0_0_0_0" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + + SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + + + + DocumentRoot "/var/www/vhosts/default/htdocs" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + + SSLEngine off + + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + DocumentRoot "/var/www/vhosts/default/htdocs" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + RPAFproxy_ips 0.0.0.0 [00000:000:000:00000::2] 0.0.0.0 + + + RPAFproxy_ips 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 + + + RemoteIPInternalProxy 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 + RemoteIPHeader X-Forwarded-For + \ No newline at end of file From 40bca477a5bafb94bd667ebdc4fd3c745ce1dabd Mon Sep 17 00:00:00 2001 From: Dominic Cleal Date: Tue, 23 Feb 2016 09:37:35 +0000 Subject: [PATCH 48/64] Merge Augeas lens fix for continuations in section headings From https://github.com/hercules-team/augeas/commit/0b22176535809f6e8aba3191a809f122e08bc7d0 Closes: #2525 --- .../letsencrypt_apache/augeas_lens/httpd.aug | 7 +++---- .../{failing => passing}/section-continuations-2525.conf | 0 2 files changed, 3 insertions(+), 4 deletions(-) rename letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/{failing => passing}/section-continuations-2525.conf (100%) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index edaca3fef..f3e688e05 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -45,9 +45,8 @@ autoload xfm let dels (s:string) = del s s (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " - -let sep_osp = Sep.opt_space +let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " +let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)/ "" let sep_eq = del /[ \t]*=[ \t]*/ "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ @@ -60,7 +59,7 @@ let indent = Util.indent (* borrowed from shellvars.aug *) let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ -let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ +let char_arg_sec = /[^\\ '"\t\r\n>]|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ let cdot = /\\\\./ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf similarity index 100% rename from letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/section-continuations-2525.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf From f4c33656a2131923dba8b678b0701674041aeb31 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 23 Feb 2016 12:08:01 -0800 Subject: [PATCH 49/64] Try to finally prevent dangling AWS volumes --- tests/letstest/multitester.py | 228 ++++++++++++++++++---------------- 1 file changed, 119 insertions(+), 109 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index e27385002..876b7807f 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -333,6 +333,55 @@ def create_client_instances(targetlist): print() return instances + +def test_client_process(inqueue, outqueue): + cur_proc = mp.current_process() + for inreq in iter(inqueue.get, SENTINEL): + ii, target = inreq + + #save all stdout to log file + sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w') + + print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) + instances[ii] = block_until_instance_ready(instances[ii]) + print("server %s at %s"%(instances[ii], instances[ii].public_ip_address)) + env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address) + print(env.host_string) + + try: + install_and_launch_letsencrypt(instances[ii], boulder_url, target) + outqueue.put((ii, target, 'pass')) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, 'fail')) + print("%s - %s FAIL"%(target['ami'], target['name'])) + pass + + # append server letsencrypt.log to each per-machine output log + print("\n\nletsencrypt.log\n" + "-"*80 + "\n") + try: + execute(grab_letsencrypt_log) + except: + print("log fail\n") + pass + + +def cleanup(cl_args, instances, targetlist): + print('Logs in ', LOGDIR) + if not cl_args.saveinstances: + print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') + if cl_args.killboulder: + boulder_server.terminate() + terminate_and_clean(instances) + else: + # print login information for the boxes for debugging + for ii, target in enumerate(targetlist): + print(target['name'], + target['ami'], + "%s@%s"%(target['user'], instances[ii].public_ip_address)) + + + #------------------------------------------------------------------------------- # SCRIPT BEGINS #------------------------------------------------------------------------------- @@ -413,124 +462,85 @@ else: #machine_type='t2.medium', security_groups=['letsencrypt_test']) -if not cl_args.boulderonly: - instances = create_client_instances(targetlist) +try: + if not cl_args.boulderonly: + instances = create_client_instances(targetlist) -# Configure and launch boulder server -#------------------------------------------------------------------------------- -print("Waiting on Boulder Server") -boulder_server = block_until_instance_ready(boulder_server) -print(" server %s"%boulder_server) + # Configure and launch boulder server + #------------------------------------------------------------------------------- + print("Waiting on Boulder Server") + boulder_server = block_until_instance_ready(boulder_server) + print(" server %s"%boulder_server) -# env.host_string defines the ssh user and host for connection -env.host_string = "ubuntu@%s"%boulder_server.public_ip_address -print("Boulder Server at (SSH):", env.host_string) -if not boulder_preexists: - print("Configuring and Launching Boulder") - config_and_launch_boulder(boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) + # env.host_string defines the ssh user and host for connection + env.host_string = "ubuntu@%s"%boulder_server.public_ip_address + print("Boulder Server at (SSH):", env.host_string) + if not boulder_preexists: + print("Configuring and Launching Boulder") + config_and_launch_boulder(boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) -boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address -print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) -print("Boulder Server at (EC2 private ip): %s"%boulder_url) + boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address + print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) + print("Boulder Server at (EC2 private ip): %s"%boulder_url) -if cl_args.boulderonly: - sys.exit(0) + if cl_args.boulderonly: + sys.exit(0) -# Install and launch client scripts in parallel -#------------------------------------------------------------------------------- -print("Uploading and running test script in parallel: %s"%cl_args.test_script) -print("Output routed to log files in %s"%LOGDIR) -# (Advice: always use Manager.Queue, never regular multiprocessing.Queue -# the latter has implementation flaws that deadlock it in some circumstances) -manager = Manager() -outqueue = manager.Queue() -inqueue = manager.Queue() -SENTINEL = None #queue kill signal + # Install and launch client scripts in parallel + #------------------------------------------------------------------------------- + print("Uploading and running test script in parallel: %s"%cl_args.test_script) + print("Output routed to log files in %s"%LOGDIR) + # (Advice: always use Manager.Queue, never regular multiprocessing.Queue + # the latter has implementation flaws that deadlock it in some circumstances) + manager = Manager() + outqueue = manager.Queue() + inqueue = manager.Queue() + SENTINEL = None #queue kill signal -# launch as many processes as clients to test -num_processes = len(targetlist) -jobs = [] #keep a reference to current procs + # launch as many processes as clients to test + num_processes = len(targetlist) + jobs = [] #keep a reference to current procs -def test_client_process(inqueue, outqueue): - cur_proc = mp.current_process() - for inreq in iter(inqueue.get, SENTINEL): - ii, target = inreq - #save all stdout to log file - sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w') + # initiate process execution + for i in range(num_processes): + p = mp.Process(target=test_client_process, args=(inqueue, outqueue)) + jobs.append(p) + p.daemon = True # kills subprocesses if parent is killed + p.start() - print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) - instances[ii] = block_until_instance_ready(instances[ii]) - print("server %s at %s"%(instances[ii], instances[ii].public_ip_address)) - env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address) - print(env.host_string) - - try: - install_and_launch_letsencrypt(instances[ii], boulder_url, target) - outqueue.put((ii, target, 'pass')) - print("%s - %s SUCCESS"%(target['ami'], target['name'])) - except: - outqueue.put((ii, target, 'fail')) - print("%s - %s FAIL"%(target['ami'], target['name'])) - pass - - # append server letsencrypt.log to each per-machine output log - print("\n\nletsencrypt.log\n" + "-"*80 + "\n") - try: - execute(grab_letsencrypt_log) - except: - print("log fail\n") - pass - -# initiate process execution -for i in range(num_processes): - p = mp.Process(target=test_client_process, args=(inqueue, outqueue)) - jobs.append(p) - p.daemon = True # kills subprocesses if parent is killed - p.start() - -# fill up work queue -for ii, target in enumerate(targetlist): - inqueue.put((ii, target)) - -# add SENTINELs to end client processes -for i in range(num_processes): - inqueue.put(SENTINEL) -# wait on termination of client processes -for p in jobs: - p.join() -# add SENTINEL to output queue -outqueue.put(SENTINEL) - -# clean up -execute(local_repo_clean) - -# print and save summary results -results_file = open(LOGDIR+'/results', 'w') -outputs = [outq for outq in iter(outqueue.get, SENTINEL)] -outputs.sort(key=lambda x: x[0]) -for outq in outputs: - ii, target, status = outq - print('%d %s %s'%(ii, target['name'], status)) - results_file.write('%d %s %s\n'%(ii, target['name'], status)) -results_file.close() - -if not cl_args.saveinstances: - print('Logs in ', LOGDIR) - print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') - if cl_args.killboulder: - boulder_server.terminate() - terminate_and_clean(instances) -else: - # print login information for the boxes for debugging + # fill up work queue for ii, target in enumerate(targetlist): - print(target['name'], - target['ami'], - "%s@%s"%(target['user'], instances[ii].public_ip_address)) + inqueue.put((ii, target)) -# kill any connections -fabric.network.disconnect_all() + # add SENTINELs to end client processes + for i in range(num_processes): + inqueue.put(SENTINEL) + # wait on termination of client processes + for p in jobs: + p.join() + # add SENTINEL to output queue + outqueue.put(SENTINEL) + + # clean up + execute(local_repo_clean) + + # print and save summary results + results_file = open(LOGDIR+'/results', 'w') + outputs = [outq for outq in iter(outqueue.get, SENTINEL)] + outputs.sort(key=lambda x: x[0]) + for outq in outputs: + ii, target, status = outq + print('%d %s %s'%(ii, target['name'], status)) + results_file.write('%d %s %s\n'%(ii, target['name'], status)) + results_file.close() + +finally: + cleanup(cl_args, instances, targetlist) + + # kill any connections + fabric.network.disconnect_all() From c86b602edeae8e1fbfc36bd4d34784cddc88a862 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 23 Feb 2016 12:46:18 -0800 Subject: [PATCH 50/64] Return an error code if any renewals fail --- letsencrypt/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 855c7a467..3fc5c4829 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1030,6 +1030,12 @@ def renew(config, unused_plugins): _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures) + if renew_failures or parse_failures: + raise errors.Error("{0} renew failure(s), {1} parse failure(s)".format( + len(renew_failures), len(parse_failures))) + else: + logger.debug("no renewal failures") + def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" From bf0e20bfa6dc5521a43cf09db9293259d12fa3de Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 23 Feb 2016 12:47:21 -0800 Subject: [PATCH 51/64] Test renewal erroring For the new case and a lot of previous ones... --- letsencrypt/tests/cli_test.py | 66 +++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 77a4b5892..d47815de5 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -540,7 +540,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._certonly_new_request_common, mock_client) def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, - args=None, renew=True): + args=None, renew=True, error_expected=False): # pylint: disable=too-many-locals cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -567,11 +567,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: args += extra_args - self._call(args) - - if log_out: - with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(log_out in lf.read()) + try: + ret, _, _, _ = self._call(args) + if ret: + print "Returned", ret + raise AssertionError(ret) + assert not error_expected, "renewal should have errored" + except: + assert error_expected, "renewal should not have errored" + traceback.format_exc() if renew: mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) @@ -580,6 +583,10 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods except: self._dump_log() raise + finally: + if log_out: + with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: + self.assertTrue(log_out in lf.read()) return mock_lineage, mock_get_utility @@ -624,12 +631,25 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, renew=True) + def test_renew_verb_empty_config(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'empty.conf'), 'w'): + rd = os.path.join(self.config_dir, 'renewal') + if not os.path.exists(rd): + os.makedirs(rd) + with open(os.path.join(rd, 'empty.conf'), 'w'): pass # leave the file empty - self.test_renew_verb() + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(False, [], args=args, renew=False, error_expected=True) + + def _unused_test(self): + with open(rc, "w") as dest: + dest.write("BLOBOFRANDOM\nJUNK") + + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, renew=True, + log_out="1 parse failure", error_expected=True) + + assert False, "Failed to raise SystemExit" def _make_dummy_renewal_config(self): renewer_configs_dir = os.path.join(self.config_dir, 'renewal') @@ -637,7 +657,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: f.write("My contents don't matter") - def _test_renew_common(self, renewalparams=None, + def _test_renew_common(self, renewalparams=None, error_expected=False, names=None, assert_oc_called=None): self._make_dummy_renewal_config() with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: @@ -649,7 +669,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods mock_lineage.names.return_value = names mock_rc.return_value = mock_lineage with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, + self._test_renewal_common(True, None, error_expected=error_expected, args=['renew'], renew=False) if assert_oc_called is not None: if assert_oc_called: @@ -658,21 +678,22 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertFalse(mock_obtain_cert.called) def test_renew_no_renewalparams(self): - self._test_renew_common(assert_oc_called=False) + self._test_renew_common(assert_oc_called=False, error_expected=True) def test_renew_no_authenticator(self): - self._test_renew_common(renewalparams={}, assert_oc_called=False) + self._test_renew_common(renewalparams={}, assert_oc_called=False, + error_expected=True) def test_renew_with_bad_int(self): renewalparams = {'authenticator': 'webroot', 'rsa_key_size': 'over 9000'} - self._test_renew_common(renewalparams=renewalparams, + self._test_renew_common(renewalparams=renewalparams, error_expected=True, assert_oc_called=False) def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} names = ['*.example.com'] - self._test_renew_common(renewalparams=renewalparams, + self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) def test_renew_plugin_config_restoration(self): @@ -686,7 +707,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods # pylint: disable=protected-access with mock.patch('letsencrypt.cli._reconstitute') as mock_reconstitute: mock_reconstitute.side_effect = Exception - self._test_renew_common(assert_oc_called=False) + self._test_renew_common(assert_oc_called=False, error_expected=True) def test_renew_obtain_cert_error(self): self._make_dummy_renewal_config() @@ -698,15 +719,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods 'renewalparams': {'authenticator': 'webroot'}} with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: mock_obtain_cert.side_effect = Exception - self._test_renewal_common(True, None, + self._test_renewal_common(True, None, error_expected=True, args=['renew'], renew=False) def test_renew_with_bad_cli_args(self): - self.assertRaises(errors.Error, self._test_renewal_common, True, None, - args='renew -d example.com'.split(), renew=False) - self.assertRaises(errors.Error, self._test_renewal_common, True, None, - args='renew --csr {0}'.format(CSR).split(), - renew=False) + self._test_renewal_common(True, None, args='renew -d example.com'.split(), + renew=False, error_expected=True) + self._test_renewal_common(True, None, args='renew --csr {0}'.format(CSR).split(), + renew=False, error_expected=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From ba88ea1b310a7960f7611ebb81c465e69d9469af Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 23 Feb 2016 12:53:31 -0800 Subject: [PATCH 52/64] Cleanup & lint --- letsencrypt/tests/cli_test.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index d47815de5..b3119e2fa 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -541,7 +541,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, renew=True, error_expected=False): - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals,too-many-arguments cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) @@ -573,8 +573,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods print "Returned", ret raise AssertionError(ret) assert not error_expected, "renewal should have errored" - except: - assert error_expected, "renewal should not have errored" + traceback.format_exc() + except: # pylint: disable=bare-except + if not error_expected: + raise AssertionError( + "Unexpected renewal error:\n" + + traceback.format_exc()) if renew: mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) @@ -631,7 +634,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, renew=True) - def test_renew_verb_empty_config(self): rd = os.path.join(self.config_dir, 'renewal') if not os.path.exists(rd): @@ -641,16 +643,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(False, [], args=args, renew=False, error_expected=True) - def _unused_test(self): - with open(rc, "w") as dest: - dest.write("BLOBOFRANDOM\nJUNK") - - args = ["renew", "--dry-run", "-tvv"] - self._test_renewal_common(True, [], args=args, renew=True, - log_out="1 parse failure", error_expected=True) - - assert False, "Failed to raise SystemExit" - def _make_dummy_renewal_config(self): renewer_configs_dir = os.path.join(self.config_dir, 'renewal') os.makedirs(renewer_configs_dir) From 34685a855869f2d6eafc8b7f9f8983c1176a81e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Feb 2016 15:53:56 -0800 Subject: [PATCH 53/64] Fix indendation --- letsencrypt-apache/letsencrypt_apache/obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index b2a21ef5d..80a49b6a6 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -210,7 +210,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return True # If we're looking for a generic vhost, don't return one with a ServerName elif self.name: - return False + return False # Both sets of names are empty. From b7a53541c5911c3619b851a5d57db4fdc8ee4161 Mon Sep 17 00:00:00 2001 From: bmw Date: Tue, 23 Feb 2016 16:51:11 -0800 Subject: [PATCH 54/64] Revert "Merge Augeas lens fix for continuations in section headings" --- .../letsencrypt_apache/augeas_lens/httpd.aug | 7 +- .../passing/section-continuations-2525.conf | 284 ------------------ 2 files changed, 4 insertions(+), 287 deletions(-) delete mode 100644 letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index f3e688e05..edaca3fef 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -45,8 +45,9 @@ autoload xfm let dels (s:string) = del s s (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " -let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)/ "" +let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " + +let sep_osp = Sep.opt_space let sep_eq = del /[ \t]*=[ \t]*/ "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ @@ -59,7 +60,7 @@ let indent = Util.indent (* borrowed from shellvars.aug *) let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ -let char_arg_sec = /[^\\ '"\t\r\n>]|\\\\"|\\\\'/ +let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ let cdot = /\\\\./ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf deleted file mode 100644 index 6840b71d6..000000000 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf +++ /dev/null @@ -1,284 +0,0 @@ - -NameVirtualHost 0.0.0.0:7080 -NameVirtualHost [00000:000:000:000::0]:7080 -NameVirtualHost 0.0.0.0:7080 - -NameVirtualHost 127.0.0.1:7080 -NameVirtualHost 0.0.0.0:7081 -NameVirtualHost [0000:000:000:000::2]:7081 -NameVirtualHost 0.0.0.0:7081 - -NameVirtualHost 127.0.0.1:7081 - -ServerName "example.com" -ServerAdmin "srv@example.com" - -DocumentRoot "/var/www/vhosts/default/htdocs" - - - LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog - - - LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" plesklog - - -TraceEnable off - -ServerTokens ProductOnly - - - AllowOverride "All" - Options SymLinksIfOwnerMatch - Order allow,deny - Allow from all - - - php_admin_flag engine off - - - - php_admin_flag engine off - - - - - - AllowOverride "All" - Options SymLinksIfOwnerMatch - Order allow,deny - Allow from all - - php_admin_flag engine off - - - php_admin_flag engine off - - - - - Header add X-Powered-By PleskLin - - - - SecRuleEngine DetectionOnly - SecRequestBodyAccess On - SecRequestBodyLimit 134217728 - SecResponseBodyAccess Off - SecResponseBodyLimit 524288 - SecAuditEngine On - SecAuditLog "/var/log/modsec_audit.log" - SecAuditLogType serial - - -Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" - - - ServerName "default" - UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" - ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" - - - SSLEngine off - - - - AllowOverride None - Options None - Order allow,deny - Allow from all - - - - - - php_admin_flag engine on - - - - php_admin_flag engine on - - - - - - - - - - ServerName "default-0_0_0_0" - UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" - ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" - - SSLEngine on - SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" - - - AllowOverride None - Options None - Order allow,deny - Allow from all - - - - - - php_admin_flag engine on - - - - php_admin_flag engine on - - - - - - - ServerName "default-0000_000_000_00000__2" - UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" - ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" - - SSLEngine on - SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" - - - AllowOverride None - Options None - Order allow,deny - Allow from all - - - - - - php_admin_flag engine on - - - - php_admin_flag engine on - - - - - - - ServerName "default-0_0_0_0" - UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" - ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" - - SSLEngine on - SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" - - SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" - - - AllowOverride None - Options None - Order allow,deny - Allow from all - - - - - - php_admin_flag engine on - - - - php_admin_flag engine on - - - - - - - - - - DocumentRoot "/var/www/vhosts/default/htdocs" - ServerName lists - ServerAlias lists.* - UseCanonicalName Off - - ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" - - Alias "/icons/" "/var/www/icons/" - Alias "/pipermail/" "/var/lib/mailman/archives/public/" - - - SSLEngine off - - - - Options FollowSymLinks - Order allow,deny - Allow from all - - - - - - - DocumentRoot "/var/www/vhosts/default/htdocs" - ServerName lists - ServerAlias lists.* - UseCanonicalName Off - - ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" - - Alias "/icons/" "/var/www/icons/" - Alias "/pipermail/" "/var/lib/mailman/archives/public/" - - SSLEngine on - SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" - - - Options FollowSymLinks - Order allow,deny - Allow from all - - - - - - - RPAFproxy_ips 0.0.0.0 [00000:000:000:00000::2] 0.0.0.0 - - - RPAFproxy_ips 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 - - - RemoteIPInternalProxy 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 - RemoteIPHeader X-Forwarded-For - \ No newline at end of file From d9534cefb503be0254360e3072bceff35f7e7b3a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Feb 2016 17:38:33 -0800 Subject: [PATCH 55/64] Revert "Revert "Merge Augeas lens fix for continuations in section headings"" This reverts commit b7a53541c5911c3619b851a5d57db4fdc8ee4161. --- .../letsencrypt_apache/augeas_lens/httpd.aug | 7 +- .../passing/section-continuations-2525.conf | 284 ++++++++++++++++++ 2 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index edaca3fef..f3e688e05 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -45,9 +45,8 @@ autoload xfm let dels (s:string) = del s s (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " - -let sep_osp = Sep.opt_space +let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " +let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)/ "" let sep_eq = del /[ \t]*=[ \t]*/ "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ @@ -60,7 +59,7 @@ let indent = Util.indent (* borrowed from shellvars.aug *) let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ -let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ +let char_arg_sec = /[^\\ '"\t\r\n>]|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ let cdot = /\\\\./ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf new file mode 100644 index 000000000..6840b71d6 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf @@ -0,0 +1,284 @@ + +NameVirtualHost 0.0.0.0:7080 +NameVirtualHost [00000:000:000:000::0]:7080 +NameVirtualHost 0.0.0.0:7080 + +NameVirtualHost 127.0.0.1:7080 +NameVirtualHost 0.0.0.0:7081 +NameVirtualHost [0000:000:000:000::2]:7081 +NameVirtualHost 0.0.0.0:7081 + +NameVirtualHost 127.0.0.1:7081 + +ServerName "example.com" +ServerAdmin "srv@example.com" + +DocumentRoot "/var/www/vhosts/default/htdocs" + + + LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + +TraceEnable off + +ServerTokens ProductOnly + + + AllowOverride "All" + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + + php_admin_flag engine off + + + + php_admin_flag engine off + + + + + + AllowOverride "All" + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + php_admin_flag engine off + + + php_admin_flag engine off + + + + + Header add X-Powered-By PleskLin + + + + SecRuleEngine DetectionOnly + SecRequestBodyAccess On + SecRequestBodyLimit 134217728 + SecResponseBodyAccess Off + SecResponseBodyLimit 524288 + SecAuditEngine On + SecAuditLog "/var/log/modsec_audit.log" + SecAuditLogType serial + + +Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" + + + ServerName "default" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + + SSLEngine off + + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + + + + ServerName "default-0_0_0_0" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + ServerName "default-0000_000_000_00000__2" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + ServerName "default-0_0_0_0" + UseCanonicalName Off + DocumentRoot "/var/www/vhosts/default/htdocs" + ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + + SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + + php_admin_flag engine on + + + + php_admin_flag engine on + + + + + + + + + + DocumentRoot "/var/www/vhosts/default/htdocs" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + + SSLEngine off + + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + DocumentRoot "/var/www/vhosts/default/htdocs" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + SSLEngine on + SSLVerifyClient none + SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + RPAFproxy_ips 0.0.0.0 [00000:000:000:00000::2] 0.0.0.0 + + + RPAFproxy_ips 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 + + + RemoteIPInternalProxy 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 + RemoteIPHeader X-Forwarded-For + \ No newline at end of file From d4804fd9e6f361433d3192b74718920e754a6d38 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 25 Feb 2016 00:08:20 -0500 Subject: [PATCH 56/64] Use a new file for the updated le-auto script. Fix #2456. I prefer to err toward simplicity here. Yes, there's an assumption necessary for this to work--that the shell doesn't do multiple open() calls to the script path throughout the life of the interpreter--but I think it's reasonable. The alternative of exec-ing out to a dedicated update script which then execs back to le-auto has more moving parts (like extra files that we have to clean up) and is longer. --- letsencrypt-auto-source/letsencrypt-auto | 17 +++++++++++++---- .../letsencrypt-auto.template | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 23a9d93f0..85e05cae6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1818,12 +1818,21 @@ UNLIKELY_EOF # future Windows compatibility. "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - # Install new copy of letsencrypt-auto. This preserves permissions and - # ownership from the old copy. + # Install new copy of letsencrypt-auto. # TODO: Deal with quotes in pathnames. echo "Replacing letsencrypt-auto..." - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # Clone permissions with cp. chmod and chown don't have a --reference + # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: + echo " " $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" + $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" + # Using mv rather than cp leaves the old file descriptor pointing to the + # original copy so the shell can continue to read it unmolested. mv across + # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the + # cp is unlikely to fail (esp. under sudo) if the rm doesn't. + echo " " $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" + $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. rm -rf "$TEMP_DIR" fi # should upgrade diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index bccd9e2c9..353fd2129 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -252,12 +252,21 @@ UNLIKELY_EOF # future Windows compatibility. "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - # Install new copy of letsencrypt-auto. This preserves permissions and - # ownership from the old copy. + # Install new copy of letsencrypt-auto. # TODO: Deal with quotes in pathnames. echo "Replacing letsencrypt-auto..." - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # Clone permissions with cp. chmod and chown don't have a --reference + # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: + echo " " $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" + $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" + # Using mv rather than cp leaves the old file descriptor pointing to the + # original copy so the shell can continue to read it unmolested. mv across + # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the + # cp is unlikely to fail (esp. under sudo) if the rm doesn't. + echo " " $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" + $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. rm -rf "$TEMP_DIR" fi # should upgrade From ac26a931472c621bde6ea4524c5963d66d7b6d54 Mon Sep 17 00:00:00 2001 From: Dominic Cleal Date: Thu, 25 Feb 2016 14:44:19 +0000 Subject: [PATCH 57/64] Merge Augeas lens fix for backslashes in section headings From https://github.com/hercules-team/augeas/commit/1cd33e52110e7c85befc00d93c867ec89cc12628 --- letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index f3e688e05..697d5de89 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -59,7 +59,7 @@ let indent = Util.indent (* borrowed from shellvars.aug *) let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ -let char_arg_sec = /[^\\ '"\t\r\n>]|\\\\"|\\\\'/ +let char_arg_sec = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ let cdot = /\\\\./ From 66e09fbf2fd1647565b2271f58a5a72c077f1637 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2016 12:11:38 -0800 Subject: [PATCH 58/64] Fix path problems in section-continuations-2525.conf --- .../passing/section-continuations-2525.conf | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf index 6840b71d6..e1bfeee27 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf @@ -13,7 +13,7 @@ NameVirtualHost 127.0.0.1:7081 ServerName "example.com" ServerAdmin "srv@example.com" -DocumentRoot "/var/www/vhosts/default/htdocs" +DocumentRoot "/var/www/html" LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog @@ -70,7 +70,7 @@ ServerTokens ProductOnly SecAuditLogType serial -Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" +#Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" ServerName "default" UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" @@ -116,12 +116,12 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem AllowOverride None @@ -149,12 +149,12 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" > ServerName "default-0000_000_000_00000__2" UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/certGXCBZ4r" + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem AllowOverride None @@ -182,14 +182,14 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem - SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" + #SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" AllowOverride None @@ -220,7 +220,7 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" 0.0.0.0:7080 \ 127.0.0.1:7080 \ > - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ServerName lists ServerAlias lists.* UseCanonicalName Off @@ -249,7 +249,7 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" 0.0.0.0:7081 \ 127.0.0.1:7081 \ > - DocumentRoot "/var/www/vhosts/default/htdocs" + DocumentRoot "/var/www/html" ServerName lists ServerAlias lists.* UseCanonicalName Off @@ -261,7 +261,7 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" SSLEngine on SSLVerifyClient none - SSLCertificateFile "/usr/local/psa/var/certificates/cert-Nv4Tz5" + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem Options FollowSymLinks @@ -281,4 +281,4 @@ Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" RemoteIPInternalProxy 0.0.0.0 [0000:000:000:0000::2] 0.0.0.0 RemoteIPHeader X-Forwarded-For - \ No newline at end of file + From 73870ac9b63b5759486c6c6ebdfa24683d18b7bb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2016 12:20:54 -0800 Subject: [PATCH 59/64] tabs + spaces = headaches --- .../passing/section-continuations-2525.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf index e1bfeee27..035a05c7e 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf @@ -121,7 +121,7 @@ ServerTokens ProductOnly SSLEngine on SSLVerifyClient none - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem AllowOverride None @@ -154,7 +154,7 @@ ServerTokens ProductOnly SSLEngine on SSLVerifyClient none - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem AllowOverride None @@ -187,7 +187,7 @@ ServerTokens ProductOnly SSLEngine on SSLVerifyClient none - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem #SSLCACertificateFile "/usr/local/psa/var/certificates/cert-nLy6Z1" @@ -261,7 +261,7 @@ ServerTokens ProductOnly SSLEngine on SSLVerifyClient none - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem Options FollowSymLinks From 03ee5a01b7bb3cb83c75d1a86fc9339439a33276 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2016 12:49:35 -0800 Subject: [PATCH 60/64] Does someone not like quotes? --- .../passing/section-continuations-2525.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf index 035a05c7e..bc403e4cf 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf @@ -13,7 +13,7 @@ NameVirtualHost 127.0.0.1:7081 ServerName "example.com" ServerAdmin "srv@example.com" -DocumentRoot "/var/www/html" +DocumentRoot /var/www/html LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog @@ -80,7 +80,7 @@ ServerTokens ProductOnly > ServerName "default" UseCanonicalName Off - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" @@ -116,7 +116,7 @@ ServerTokens ProductOnly > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -149,7 +149,7 @@ ServerTokens ProductOnly > ServerName "default-0000_000_000_00000__2" UseCanonicalName Off - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -182,7 +182,7 @@ ServerTokens ProductOnly > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -220,7 +220,7 @@ ServerTokens ProductOnly 0.0.0.0:7080 \ 127.0.0.1:7080 \ > - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ServerName lists ServerAlias lists.* UseCanonicalName Off @@ -249,7 +249,7 @@ ServerTokens ProductOnly 0.0.0.0:7081 \ 127.0.0.1:7081 \ > - DocumentRoot "/var/www/html" + DocumentRoot /var/www/html ServerName lists ServerAlias lists.* UseCanonicalName Off From 13a4089ee65e608f49669e282cbfe23af38e0366 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2016 13:10:43 -0800 Subject: [PATCH 61/64] I promise /tmp is a directory --- .../passing/section-continuations-2525.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf index bc403e4cf..8f65e4773 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/section-continuations-2525.conf @@ -13,7 +13,7 @@ NameVirtualHost 127.0.0.1:7081 ServerName "example.com" ServerAdmin "srv@example.com" -DocumentRoot /var/www/html +DocumentRoot /tmp LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog @@ -80,7 +80,7 @@ ServerTokens ProductOnly > ServerName "default" UseCanonicalName Off - DocumentRoot /var/www/html + DocumentRoot /tmp ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" @@ -116,7 +116,7 @@ ServerTokens ProductOnly > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot /var/www/html + DocumentRoot /tmp ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -149,7 +149,7 @@ ServerTokens ProductOnly > ServerName "default-0000_000_000_00000__2" UseCanonicalName Off - DocumentRoot /var/www/html + DocumentRoot /tmp ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -182,7 +182,7 @@ ServerTokens ProductOnly > ServerName "default-0_0_0_0" UseCanonicalName Off - DocumentRoot /var/www/html + DocumentRoot /tmp ScriptAlias "/cgi-bin/" "/var/www/vhosts/default/cgi-bin" SSLEngine on @@ -220,7 +220,7 @@ ServerTokens ProductOnly 0.0.0.0:7080 \ 127.0.0.1:7080 \ > - DocumentRoot /var/www/html + DocumentRoot /tmp ServerName lists ServerAlias lists.* UseCanonicalName Off @@ -249,7 +249,7 @@ ServerTokens ProductOnly 0.0.0.0:7081 \ 127.0.0.1:7081 \ > - DocumentRoot /var/www/html + DocumentRoot /tmp ServerName lists ServerAlias lists.* UseCanonicalName Off From bcb40a890beafe6f7a86c7b2b81d421a51280e19 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 25 Feb 2016 16:52:29 -0800 Subject: [PATCH 62/64] Remove werkzeug from leauto requirements --- .../pieces/letsencrypt-auto-requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index 574e567c3..b258fcfad 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -172,10 +172,6 @@ traceback2==1.4.0 # sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk unittest2==1.1.0 -# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms -# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI -Werkzeug==0.11.3 - # sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo zope.component==4.2.2 From 593cb3a03855c0ec41479b16d34209ee168ac74d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 25 Feb 2016 16:58:54 -0800 Subject: [PATCH 63/64] alphabetanit --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 79d7c5df4..e67de35c5 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,10 +1,10 @@ """ACME client API.""" import collections import datetime +from email.utils import parsedate_tz import heapq import logging import time -from email.utils import parsedate_tz import six from six.moves import http_client # pylint: disable=import-error From 556e9f2123535f3aaaae751f3e99c646e0b1cd39 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 25 Feb 2016 17:03:48 -0800 Subject: [PATCH 64/64] Rebuild leauto --- letsencrypt-auto-source/letsencrypt-auto | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9218bdc52..ba0827cf6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.4.0" +LE_AUTO_VERSION="0.5.0.dev0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -609,10 +609,6 @@ traceback2==1.4.0 # sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk unittest2==1.1.0 -# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms -# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI -Werkzeug==0.11.3 - # sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo zope.component==4.2.2